From 63fb4bdd7e9eb7096cab95f5b1db622872acd340 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 12 Jan 2023 20:49:15 +0100 Subject: Revert "Support zstd as compression type for file references" --- .../filedistribution/FileReferenceCompressor.java | 23 +++++++++------------- .../vespa/filedistribution/FileReceiverTest.java | 19 +++++++++--------- 2 files changed, 19 insertions(+), 23 deletions(-) (limited to 'filedistribution') diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index 80ed682025b..2ea41a9fd8c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; -import ai.vespa.airlift.zstd.ZstdInputStream; -import com.yahoo.compress.ZstdOutputStream; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.apache.commons.compress.archivers.ArchiveEntry; @@ -23,6 +21,7 @@ import java.util.List; import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -119,37 +118,33 @@ public class FileReferenceCompressor { } private OutputStream compressedOutputStream(File outputFile) throws IOException { - FileOutputStream out = new FileOutputStream(outputFile); switch (type) { case compressed: log.log(Level.FINE, () -> "Compressing with compression type " + compressionType); return switch (compressionType) { - case gzip -> new GZIPOutputStream(out); - case lz4 -> new LZ4BlockOutputStream(out); - case zstd -> new ZstdOutputStream(out); + case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile)); + case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile)); + default -> throw new RuntimeException("Unknown compression type " + compressionType); }; case file: - return out; + return new FileOutputStream(outputFile); default: - out.close(); throw new RuntimeException("Unknown file reference type " + type); } } private InputStream decompressedInputStream(File inputFile) throws IOException { - FileInputStream in = new FileInputStream(inputFile); switch (type) { case compressed: log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType); return switch (compressionType) { - case gzip -> new GZIPInputStream(in); - case lz4 -> new LZ4BlockInputStream(in); - case zstd -> new ZstdInputStream(in); + case gzip -> new GZIPInputStream(new FileInputStream(inputFile)); + case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile)); + default -> throw new RuntimeException("Unknown compression type " + compressionType); }; case file: - return in; + return new FileInputStream(inputFile); default: - in.close(); throw new RuntimeException("Unknown file reference type " + type); } } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java index 59ebd571431..84e7a07340e 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java @@ -19,7 +19,6 @@ import java.nio.file.Files; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.file; import static org.junit.Assert.assertEquals; @@ -62,18 +61,20 @@ public class FileReceiverTest { writerB.write("2"); writerB.close(); - testWithCompression(dirWithFiles, gzip); - testWithCompression(dirWithFiles, lz4); - testWithCompression(dirWithFiles, zstd); - } - - private void testWithCompression(File dirWithFiles, CompressionType compressionType) throws IOException { File tempFile = temporaryFolder.newFile(); - File file = new FileReferenceCompressor(compressed, compressionType).compress(dirWithFiles, tempFile); - transferCompressedData(compressionType, new FileReference("ref"), "a", IOUtils.readFileBytes(file)); + File file = new FileReferenceCompressor(compressed, gzip).compress(dirWithFiles, tempFile); + transferCompressedData(gzip, new FileReference("ref"), "a", IOUtils.readFileBytes(file)); File downloadDir = new File(root, "ref"); assertEquals("1", IOUtils.readFile(new File(downloadDir, "a"))); assertEquals("2", IOUtils.readFile(new File(downloadDir, "b"))); + + tempFile = temporaryFolder.newFile(); + FileReferenceCompressor compressor = new FileReferenceCompressor(compressed, lz4); + file = compressor.compress(dirWithFiles, tempFile); + transferCompressedData(lz4, new FileReference("ref"), "a", IOUtils.readFileBytes(file)); + downloadDir = new File(root, "ref"); + assertEquals("1", IOUtils.readFile(new File(downloadDir, "a"))); + assertEquals("2", IOUtils.readFile(new File(downloadDir, "b"))); } private void transferPartsAndAssert(FileReference ref, String fileName, String all, int numParts) throws IOException { -- cgit v1.2.3