diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-03-05 21:40:58 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-03-05 21:40:58 +0000 |
commit | aabd0930e6569efa85f075b054de2200d9642f27 (patch) | |
tree | bab541bcb74783eb185be824cc53920041ea5546 /config | |
parent | 395e7270e0d1ccb793d499b4957c1d810e617338 (diff) |
Use lz4 through the compressor in vespajlib.
Diffstat (limited to 'config')
-rwxr-xr-x | config/pom.xml | 4 | ||||
-rw-r--r-- | config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java | 19 | ||||
-rw-r--r-- | config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java | 73 |
3 files changed, 6 insertions, 90 deletions
diff --git a/config/pom.xml b/config/pom.xml index d4f6fd1ebf5..ad036c442d1 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -66,10 +66,6 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> </dependency> - <dependency> - <groupId>net.jpountz.lz4</groupId> - <artifactId>lz4</artifactId> - </dependency> <!-- test scope --> <dependency> diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java index 7a27902c89c..a70a2861cc8 100644 --- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; import com.yahoo.vespa.config.util.ConfigUtils; -import net.jpountz.lz4.LZ4Compressor; -import net.jpountz.lz4.LZ4Factory; /** * Wrapper for LZ4 compression that selects compression level based on properties. @@ -12,9 +12,8 @@ import net.jpountz.lz4.LZ4Factory; */ public class LZ4PayloadCompressor { - private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance(); private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL"; - private static final int compressionLevel = getCompressionLevel(); + private static final Compressor compressor = new Compressor(CompressionType.LZ4, getCompressionLevel()); private static int getCompressionLevel() { return Integer.parseInt(ConfigUtils.getEnvValue("0", @@ -24,17 +23,11 @@ public class LZ4PayloadCompressor { } public byte[] compress(byte[] input) { - return getCompressor().compress(input); + return compressor.compressUnconditionally(input); } - public void decompress(byte[] input, byte[] outputbuffer) { - if (input.length > 0) { - lz4Factory.safeDecompressor().decompress(input, outputbuffer); - } - } - - private LZ4Compressor getCompressor() { - return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor(); + public void decompress(byte[] input, byte[] output) { + compressor.decompressUnconditionally(input, output); } } diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java deleted file mode 100644 index 805efc90ade..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import net.jpountz.lz4.LZ4Compressor; -import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4SafeDecompressor; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.nio.file.FileSystems; -import java.nio.file.Files; - -/** - * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge. - * - * @author Ulf Lilleengen - */ -public class LZ4CompressionTest { - private static LZ4Factory factory = LZ4Factory.safeInstance(); - - @Test - @Ignore - public void testCompression() throws IOException { - byte[] data = getInput(); - System.out.println("High compressor"); - for (int i = 0; i < 10; i++) { - timeCompressor(factory.highCompressor(), data); - } - System.out.println("Fast compressor"); - for (int i = 0; i < 10; i++) { - timeCompressor(factory.fastCompressor(), data); - } - } - - private byte[] getInput() throws IOException { - byte[] data = Files.readAllBytes(FileSystems.getDefault().getPath("src/test/ca.json")); - System.out.println("Input size: " + data.length); - return data; - } - - private void timeCompressor(LZ4Compressor lz4Compressor, byte[] data) { - long start = System.currentTimeMillis(); - byte[] compressed = lz4Compressor.compress(data); - long end = System.currentTimeMillis(); - System.out.println("Compression took " + (end - start) + " millis, and size of data is " + compressed.length + " bytes"); - } - - @Test - @Ignore - public void testDecompression() throws IOException { - byte[] data = getInput(); - byte[] outputbuffer = new byte[data.length]; - byte[] hcCompressedData = factory.highCompressor().compress(data); - System.out.println("High compressor"); - for (int i = 0; i < 10; i++) { - timeDecompressor(hcCompressedData, factory.safeDecompressor(), outputbuffer); - } - byte[] fastCompressedData = factory.fastCompressor().compress(data); - System.out.println("Fast compressor"); - for (int i = 0; i < 10; i++) { - timeDecompressor(fastCompressedData, factory.safeDecompressor(), outputbuffer); - } - } - - private void timeDecompressor(byte[] compressedData, LZ4SafeDecompressor decompressor, byte[] outputbuffer) { - long start = System.currentTimeMillis(); - decompressor.decompress(compressedData, outputbuffer); - long end = System.currentTimeMillis(); - System.out.println("Decompression took " + (end - start) + " millis"); - } - -} |