Add zstd support to Compressor

Introduce zstandard compression using airlift aircompressor - a pure Java implementation.

5 files changed, 98 insertions, 1 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java
index c3e42895a5e..36deb318ae8 100644
--- a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java
+++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java
@@ -11,7 +11,8 @@ public enum CompressionType {
     // Do not change the type->ordinal association. The gap is due to historic types no longer supported.
     NONE((byte) 0),
     INCOMPRESSIBLE((byte) 5),
-    LZ4((byte) 6);
+    LZ4((byte) 6),
+    ZSTD((byte) 7);
 
     private byte code;
 
@@ -38,6 +39,8 @@ public enum CompressionType {
                 return INCOMPRESSIBLE;
             case ((byte) 6):
                 return LZ4;
+            case ((byte) 7):
+                return ZSTD;
             default:
                 throw new IllegalArgumentException("Unknown compression type ordinal " + value);
         }
diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
index fb5da192f36..3220b81a3a9 100644
--- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
+++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
@@ -18,6 +18,7 @@ import java.util.Random;
  */
 public class Compressor {
 
+    private final ZstdCompressor zstdCompressor = new ZstdCompressor();
     private final CompressionType type;
     private final int level;
     private final double compressionThresholdFactor;
@@ -91,6 +92,11 @@ public class Compressor {
                 if (compressedData.length + 8 >= dataSize * compressionThresholdFactor)
                     return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
                 return new Compression(CompressionType.LZ4, dataSize, compressedData);
+            case ZSTD:
+                int dataLength = uncompressedSize.orElse(data.length);
+                if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data);
+                byte[] compressed = zstdCompressor.compress(data, 0, dataLength);
+                return new Compression(CompressionType.ZSTD, dataLength, compressed);
             default:
                 throw new IllegalArgumentException(requestedCompression + " is not supported");
         }
@@ -130,6 +136,15 @@ public class Compressor {
                 if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get())
                     throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get());
                 return uncompressedLZ4Data;
+            case ZSTD:
+                int compressedLength = expectedCompressedSize.orElseThrow(() -> new IllegalArgumentException("Zstd decompressor requires input size"));
+                byte[] decompressedData = zstdCompressor.decompress(compressedData, compressedDataOffset, compressedLength);
+                expectedCompressedSize.ifPresent(expectedSize -> {
+                    if (compressedData.length != expectedSize) {
+                        throw new IllegalStateException("Compressed size mismatch. Expected " + expectedSize + ". Got " + decompressedData.length);
+                    }
+                });
+                return decompressedData;
             default:
                 throw new IllegalArgumentException(compression + " is not supported");
         }
diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java
new file mode 100644
index 00000000000..58a01df5b09
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.compress;
+
+import java.util.Arrays;
+
+/**
+ * Frame based Zstd compressor (https://github.com/facebook/zstd)
+ * Implemented based on https://github.com/airlift/aircompressor - a pure Java implementation (no JNI).
+ *
+ * @author bjorncs
+ */
+public class ZstdCompressor {
+
+    private static final io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor();
+    private static final io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor();
+
+    public byte[] compress(byte[] input, int inputOffset, int inputLength) {
+        int maxCompressedLength = compressor.maxCompressedLength(inputLength);
+        byte[] output = new byte[maxCompressedLength];
+        int compressedLength = compressor.compress(input, inputOffset, inputLength, output, 0, maxCompressedLength);
+        return Arrays.copyOf(output, compressedLength);
+    }
+
+    public byte[] decompress(byte[] input, int inputOffset, int inputLength) {
+        int decompressedLength = (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength);
+        byte[] output = new byte[decompressedLength];
+        decompressor.decompress(input, inputOffset, inputLength, output, 0, decompressedLength);
+        return output;
+    }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java b/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java
new file mode 100644
index 00000000000..8a16e107608
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java
@@ -0,0 +1,27 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.compress;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author bjorncs
+ */
+class CompressorTest {
+
+    @Test
+    void compresses_and_decompresses_input_using_zstd() {
+        byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes();
+        Compressor compressor = new Compressor(CompressionType.ZSTD);
+        Compressor.Compression compression = compressor.compress(CompressionType.ZSTD, inputData, Optional.empty());
+        assertEquals(inputData.length, compression.uncompressedSize());
+        byte[] compressedData = compression.data();
+        byte[] decompressedData = compressor.decompress(CompressionType.ZSTD, compressedData, 0, inputData.length, Optional.of(compressedData.length));
+        assertArrayEquals(inputData, decompressedData);
+    }
+
+}
\ No newline at end of file
diff --git a/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java b/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java
new file mode 100644
index 00000000000..837482a089a
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java
@@ -0,0 +1,22 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.compress;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+
+/**
+ * @author bjorncs
+ */
+class ZstdCompressorTest {
+
+    @Test
+    void compresses_and_decompresses_input() {
+        byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes();
+        ZstdCompressor compressor = new ZstdCompressor();
+        byte[] compressedData = compressor.compress(inputData, 0, inputData.length);
+        byte[] decompressedData = compressor.decompress(compressedData, 0, compressedData.length);
+        assertArrayEquals(inputData, decompressedData);
+    }
+
+}
\ No newline at end of file