Use lz4 through the compressor in vespajlib.

4 files changed, 23 insertions, 92 deletions

diff --git a/config/pom.xml b/config/pom.xml
index d4f6fd1ebf5..ad036c442d1 100755
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -66,10 +66,6 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>net.jpountz.lz4</groupId>
-      <artifactId>lz4</artifactId>
-    </dependency>
 
     <!-- test scope -->
     <dependency>
diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
index 7a27902c89c..a70a2861cc8 100644
--- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
+++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config;
 
+import com.yahoo.compress.CompressionType;
+import com.yahoo.compress.Compressor;
 import com.yahoo.vespa.config.util.ConfigUtils;
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
 
 /**
  * Wrapper for LZ4 compression that selects compression level based on properties.
@@ -12,9 +12,8 @@ import net.jpountz.lz4.LZ4Factory;
  */
 public class LZ4PayloadCompressor {
 
-    private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance();
     private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL";
-    private static final int compressionLevel = getCompressionLevel();
+    private static final Compressor compressor = new Compressor(CompressionType.LZ4, getCompressionLevel());
 
     private static int getCompressionLevel() {
         return Integer.parseInt(ConfigUtils.getEnvValue("0",
@@ -24,17 +23,11 @@ public class LZ4PayloadCompressor {
     }
 
     public byte[] compress(byte[] input) {
-        return getCompressor().compress(input);
+        return compressor.compressUnconditionally(input);
     }
 
-    public void decompress(byte[] input, byte[] outputbuffer) {
-        if (input.length > 0) {
-            lz4Factory.safeDecompressor().decompress(input, outputbuffer);
-        }
-    }
-
-    private LZ4Compressor getCompressor() {
-        return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor();
+    public void decompress(byte[] input, byte[] output) {
+        compressor.decompressUnconditionally(input, output);
     }
 
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
deleted file mode 100644
index 805efc90ade..00000000000
--- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config;
-
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4SafeDecompressor;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-
-/**
- * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge.
- *
- * @author Ulf Lilleengen
- */
-public class LZ4CompressionTest {
-    private static LZ4Factory factory = LZ4Factory.safeInstance();
-
-    @Test
-    @Ignore
-    public void testCompression() throws IOException {
-        byte[] data = getInput();
-        System.out.println("High compressor");
-        for (int i = 0; i < 10; i++) {
-            timeCompressor(factory.highCompressor(), data);
-        }
-        System.out.println("Fast compressor");
-        for (int i = 0; i < 10; i++) {
-            timeCompressor(factory.fastCompressor(), data);
-        }
-    }
-
-    private byte[] getInput() throws IOException {
-        byte[] data = Files.readAllBytes(FileSystems.getDefault().getPath("src/test/ca.json"));
-        System.out.println("Input size: " + data.length);
-        return data;
-    }
-
-    private void timeCompressor(LZ4Compressor lz4Compressor, byte[] data) {
-        long start = System.currentTimeMillis();
-        byte[] compressed = lz4Compressor.compress(data);
-        long end = System.currentTimeMillis();
-        System.out.println("Compression took " + (end - start) + " millis, and size of data is " + compressed.length + " bytes");
-    }
-
-    @Test
-    @Ignore
-    public void testDecompression() throws IOException {
-        byte[] data = getInput();
-        byte[] outputbuffer = new byte[data.length];
-        byte[] hcCompressedData = factory.highCompressor().compress(data);
-        System.out.println("High compressor");
-        for (int i = 0; i < 10; i++) {
-            timeDecompressor(hcCompressedData, factory.safeDecompressor(), outputbuffer);
-        }
-        byte[] fastCompressedData = factory.fastCompressor().compress(data);
-        System.out.println("Fast compressor");
-        for (int i = 0; i < 10; i++) {
-            timeDecompressor(fastCompressedData, factory.safeDecompressor(), outputbuffer);
-        }
-    }
-
-    private void timeDecompressor(byte[] compressedData, LZ4SafeDecompressor decompressor, byte[] outputbuffer) {
-        long start = System.currentTimeMillis();
-        decompressor.decompress(compressedData, outputbuffer);
-        long end = System.currentTimeMillis();
-        System.out.println("Decompression took " + (end - start) + " millis");
-    }
-
-}
diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
index 9e9fac936f4..ee003734d1a 100644
--- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
+++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
@@ -31,6 +31,10 @@ public class Compressor {
         this(type, 9, 0.95, 0);
     }
 
+    public Compressor(CompressionType type, int level) {
+        this(type, level, 0.95, 0);
+    }
+
     /**
      * Creates a compressor.
      *
@@ -79,8 +83,7 @@ public class Compressor {
             case LZ4:
                 int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length;
                 if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
-                LZ4Compressor compressor = level < 7 ? factory.fastCompressor() : factory.highCompressor();
-                byte[] compressedData = compressor.compress(data, 0, dataSize);
+                byte[] compressedData = getCompressor().compress(data, 0, dataSize);
                 if (compressedData.length + 8 >= dataSize * compressionThresholdFactor)
                     return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
                 return new Compression(CompressionType.LZ4, dataSize, compressedData);
@@ -88,6 +91,9 @@ public class Compressor {
                 throw new IllegalArgumentException(requestedCompression + " is not supported");
         }
     }
+    private LZ4Compressor getCompressor() {
+        return level < 7 ? factory.fastCompressor() : factory.highCompressor();
+    }
     /** Compresses some data using the requested compression type */
     public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, Optional.empty()); }
     /** Compresses some data using the compression type of this compressor */
@@ -133,6 +139,15 @@ public class Compressor {
         return decompress(compression.type(), compression.data(), 0, compression.uncompressedSize(), Optional.empty());
     }
 
+    public byte[] compressUnconditionally(byte[] input) {
+        return getCompressor().compress(input);
+    }
+    public void decompressUnconditionally(byte[] input, byte[] output) {
+        if (input.length > 0) {
+            factory.safeDecompressor().decompress(input, output);
+        }
+    }
+
     public static class Compression {
 
         private final CompressionType compressionType;