summaryrefslogtreecommitdiffstats
path: root/filedistribution
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2023-01-16 13:21:14 +0100
committerGitHub <noreply@github.com>2023-01-16 13:21:14 +0100
commit34df2a872569812a45c2b0d43cb3ac6f2662986e (patch)
tree7b3f523f6355eacf7ffc2b9ef842242d6c3567ec /filedistribution
parent5b93a02564c303b136591ad0cb59bc612456e523 (diff)
Revert "Support zstd as compression type for file references, take 2"
Diffstat (limited to 'filedistribution')
-rw-r--r--filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java8
-rw-r--r--filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java19
2 files changed, 14 insertions, 13 deletions
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
index 8d6f9ea1af3..2ea41a9fd8c 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
@@ -1,8 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.filedistribution;
-import ai.vespa.airlift.zstd.ZstdInputStream;
-import com.yahoo.compress.ZstdOutputStream;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import org.apache.commons.compress.archivers.ArchiveEntry;
@@ -23,6 +21,7 @@ import java.util.List;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -125,7 +124,7 @@ public class FileReferenceCompressor {
return switch (compressionType) {
case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile));
case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile));
- case zstd -> new ZstdOutputStream(new FileOutputStream(outputFile));
+ default -> throw new RuntimeException("Unknown compression type " + compressionType);
};
case file:
return new FileOutputStream(outputFile);
@@ -141,7 +140,7 @@ public class FileReferenceCompressor {
return switch (compressionType) {
case gzip -> new GZIPInputStream(new FileInputStream(inputFile));
case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile));
- case zstd -> new ZstdInputStream(new FileInputStream(inputFile));
+ default -> throw new RuntimeException("Unknown compression type " + compressionType);
};
case file:
return new FileInputStream(inputFile);
@@ -151,3 +150,4 @@ public class FileReferenceCompressor {
}
}
+
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
index 59ebd571431..84e7a07340e 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java
@@ -19,7 +19,6 @@ import java.nio.file.Files;
import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip;
import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4;
-import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd;
import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.file;
import static org.junit.Assert.assertEquals;
@@ -62,18 +61,20 @@ public class FileReceiverTest {
writerB.write("2");
writerB.close();
- testWithCompression(dirWithFiles, gzip);
- testWithCompression(dirWithFiles, lz4);
- testWithCompression(dirWithFiles, zstd);
- }
-
- private void testWithCompression(File dirWithFiles, CompressionType compressionType) throws IOException {
File tempFile = temporaryFolder.newFile();
- File file = new FileReferenceCompressor(compressed, compressionType).compress(dirWithFiles, tempFile);
- transferCompressedData(compressionType, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
+ File file = new FileReferenceCompressor(compressed, gzip).compress(dirWithFiles, tempFile);
+ transferCompressedData(gzip, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
File downloadDir = new File(root, "ref");
assertEquals("1", IOUtils.readFile(new File(downloadDir, "a")));
assertEquals("2", IOUtils.readFile(new File(downloadDir, "b")));
+
+ tempFile = temporaryFolder.newFile();
+ FileReferenceCompressor compressor = new FileReferenceCompressor(compressed, lz4);
+ file = compressor.compress(dirWithFiles, tempFile);
+ transferCompressedData(lz4, new FileReference("ref"), "a", IOUtils.readFileBytes(file));
+ downloadDir = new File(root, "ref");
+ assertEquals("1", IOUtils.readFile(new File(downloadDir, "a")));
+ assertEquals("2", IOUtils.readFile(new File(downloadDir, "b")));
}
private void transferPartsAndAssert(FileReference ref, String fileName, String all, int numParts) throws IOException {