Merge pull request #25581 from vespa-engine/revert-25580-revert-25577-revert-25547-revert-25532-hmusum/support-zstd-for-compressed-file-references

Support zstd as compression type for file references, take 3

3 files changed, 18 insertions, 14 deletions

diff --git a/configserver/pom.xml b/configserver/pom.xml
index 0708d1fc46d..ef0fde3e57e 100644
--- a/configserver/pom.xml
+++ b/configserver/pom.xml
@@ -112,6 +112,11 @@
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
+      <artifactId>airlift-zstd</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
       <artifactId>defaults</artifactId>
       <version>${project.version}</version>
       <scope>provided</scope>
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
index 2ea41a9fd8c..8d6f9ea1af3 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.filedistribution;
 
+import ai.vespa.airlift.zstd.ZstdInputStream;
+import com.yahoo.compress.ZstdOutputStream;
 import net.jpountz.lz4.LZ4BlockInputStream;
 import net.jpountz.lz4.LZ4BlockOutputStream;
 import org.apache.commons.compress.archivers.ArchiveEntry;
@@ -21,7 +23,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -124,7 +125,7 @@ public class FileReferenceCompressor {
                 return switch (compressionType) {
                     case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile));
                     case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile));
-                    default -> throw new RuntimeException("Unknown compression type " + compressionType);
+                    case zstd -> new ZstdOutputStream(new FileOutputStream(outputFile));
                 };
             case file:
                 return new FileOutputStream(outputFile);
@@ -140,7 +141,7 @@ public class FileReferenceCompressor {
                 return switch (compressionType) {
                     case gzip -> new GZIPInputStream(new FileInputStream(inputFile));
                     case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile));
-                    default -> throw new RuntimeException("Unknown compression type " + compressionType);
+                    case zstd -> new ZstdInputStream(new FileInputStream(inputFile));
                 };
             case file:
                 return new FileInputStream(inputFile);
@@ -150,4 +151,3 @@ public class FileReferenceCompressor {
     }
 
 }
-
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
index 84e7a07340e..59ebd571431 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
@@ -19,6 +19,7 @@ import java.nio.file.Files;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.file;
 import static org.junit.Assert.assertEquals;
@@ -61,20 +62,18 @@ public class FileReceiverTest {
         writerB.write("2");
         writerB.close();
 
+        testWithCompression(dirWithFiles, gzip);
+        testWithCompression(dirWithFiles, lz4);
+        testWithCompression(dirWithFiles, zstd);
+    }
+
+    private void testWithCompression(File dirWithFiles, CompressionType compressionType) throws IOException {
         File tempFile = temporaryFolder.newFile();
-        File file = new FileReferenceCompressor(compressed, gzip).compress(dirWithFiles, tempFile);
-        transferCompressedData(gzip, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
+        File file = new FileReferenceCompressor(compressed, compressionType).compress(dirWithFiles, tempFile);
+        transferCompressedData(compressionType, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
         File downloadDir = new File(root, "ref");
         assertEquals("1", IOUtils.readFile(new File(downloadDir, "a")));
         assertEquals("2", IOUtils.readFile(new File(downloadDir, "b")));
-
-        tempFile = temporaryFolder.newFile();
-        FileReferenceCompressor compressor = new FileReferenceCompressor(compressed, lz4);
-        file = compressor.compress(dirWithFiles, tempFile);
-        transferCompressedData(lz4, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
-        downloadDir = new File(root, "ref");
-        assertEquals("1", IOUtils.readFile(new File(downloadDir, "a")));
-        assertEquals("2", IOUtils.readFile(new File(downloadDir, "b")));
     }
 
     private void transferPartsAndAssert(FileReference ref, String fileName, String all, int numParts) throws IOException {