diff options
author | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-01-19 13:29:20 +0100 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-01-20 12:23:29 +0100 |
commit | 61bda66d5b7789a1deb580992ff4adf2835def47 (patch) | |
tree | 98ad497b1053c93d30a0b95c6f947b4eda14e8ff /vespajlib/src/main/java/com/yahoo/compress | |
parent | c68838b159e51eeb6810a1d8689d09f6494a9554 (diff) |
Add zstd support to Compressor
Introduce zstandard compression using airlift aircompressor - a pure Java implementation.
Diffstat (limited to 'vespajlib/src/main/java/com/yahoo/compress')
3 files changed, 49 insertions, 1 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java index c3e42895a5e..36deb318ae8 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -11,7 +11,8 @@ public enum CompressionType { // Do not change the type->ordinal association. The gap is due to historic types no longer supported. NONE((byte) 0), INCOMPRESSIBLE((byte) 5), - LZ4((byte) 6); + LZ4((byte) 6), + ZSTD((byte) 7); private byte code; @@ -38,6 +39,8 @@ public enum CompressionType { return INCOMPRESSIBLE; case ((byte) 6): return LZ4; + case ((byte) 7): + return ZSTD; default: throw new IllegalArgumentException("Unknown compression type ordinal " + value); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index fb5da192f36..3220b81a3a9 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -18,6 +18,7 @@ import java.util.Random; */ public class Compressor { + private final ZstdCompressor zstdCompressor = new ZstdCompressor(); private final CompressionType type; private final int level; private final double compressionThresholdFactor; @@ -91,6 +92,11 @@ public class Compressor { if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); return new Compression(CompressionType.LZ4, dataSize, compressedData); + case ZSTD: + int dataLength = uncompressedSize.orElse(data.length); + if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data); + byte[] compressed = zstdCompressor.compress(data, 0, dataLength); + return new Compression(CompressionType.ZSTD, dataLength, compressed); default: throw new IllegalArgumentException(requestedCompression + " is not supported"); } @@ -130,6 +136,15 @@ public class Compressor { if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get()) throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get()); return uncompressedLZ4Data; + case ZSTD: + int compressedLength = expectedCompressedSize.orElseThrow(() -> new IllegalArgumentException("Zstd decompressor requires input size")); + byte[] decompressedData = zstdCompressor.decompress(compressedData, compressedDataOffset, compressedLength); + expectedCompressedSize.ifPresent(expectedSize -> { + if (compressedData.length != expectedSize) { + throw new IllegalStateException("Compressed size mismatch. Expected " + expectedSize + ". Got " + decompressedData.length); + } + }); + return decompressedData; default: throw new IllegalArgumentException(compression + " is not supported"); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java new file mode 100644 index 00000000000..58a01df5b09 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import java.util.Arrays; + +/** + * Frame based Zstd compressor (https://github.com/facebook/zstd) + * Implemented based on https://github.com/airlift/aircompressor - a pure Java implementation (no JNI). + * + * @author bjorncs + */ +public class ZstdCompressor { + + private static final io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor(); + private static final io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor(); + + public byte[] compress(byte[] input, int inputOffset, int inputLength) { + int maxCompressedLength = compressor.maxCompressedLength(inputLength); + byte[] output = new byte[maxCompressedLength]; + int compressedLength = compressor.compress(input, inputOffset, inputLength, output, 0, maxCompressedLength); + return Arrays.copyOf(output, compressedLength); + } + + public byte[] decompress(byte[] input, int inputOffset, int inputLength) { + int decompressedLength = (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); + byte[] output = new byte[decompressedLength]; + decompressor.decompress(input, inputOffset, inputLength, output, 0, decompressedLength); + return output; + } +} |