lz4 compress blobs.

10 files changed, 63 insertions, 33 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
index 7fd36b0d142..1719ea72cb0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -38,7 +38,7 @@ public class DistributableResource {
         Objects.requireNonNull(blob, "Blob cannot be null");
         this.name = name;
         this.blob = blob;
-        path = name;
+        path = name + ".lz4";
         pathType = PathType.BLOB;
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index bde8567c34e..52d443e6998 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import com.yahoo.io.IOUtils;
+import net.jpountz.lz4.LZ4FrameOutputStream;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -70,7 +71,13 @@ public class ApplicationFileManager implements AddFileInterface {
             file = new File(path.toFile(), relativePath);
             Files.createDirectories(file.getParentFile().toPath());
             fos = new FileOutputStream(file);
-            fos.write(blob.array(), blob.arrayOffset(), blob.remaining());
+            if (relativePath.endsWith(".lz4")) {
+                LZ4FrameOutputStream lz4 = new LZ4FrameOutputStream(fos);
+                lz4.write(blob.array(), blob.arrayOffset(), blob.remaining());
+                lz4.close();
+            } else {
+                fos.write(blob.array(), blob.arrayOffset(), blob.remaining());
+            }
             return file;
         } catch (IOException e) {
             throw new IllegalArgumentException("Failed creating temp file", e);
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
index eccc236f0ca..fbfd34814ac 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -17,8 +17,10 @@ import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import net.jpountz.lz4.LZ4FrameInputStream;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
@@ -207,10 +209,16 @@ public class RankProfilesConfigImporter {
         return expressions;
     }
 
+    protected final String readExpressionFromFile(File file) throws IOException {
+        return (file.getName().endsWith(".lz4"))
+            ? Utf8.toString(IOUtils.readBytes(new LZ4FrameInputStream(new FileInputStream(file)), 65536))
+            : Utf8.toString(IOUtils.readFileBytes(file));
+    }
+
     protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException {
         try {
             File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS);
-            return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(file)));
+            return new RankingExpression(name, readExpressionFromFile(file));
         }
         catch (InterruptedException e) {
             throw new IllegalStateException("Gave up waiting for expression " + name);
@@ -245,8 +253,8 @@ public class RankProfilesConfigImporter {
         private static final Pattern valuePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.value");
         private static final Pattern  typePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.type");
 
-        private Map<String, TensorType> types = new HashMap<>();
-        private Map<String, String> values = new HashMap<>();
+        private final Map<String, TensorType> types = new HashMap<>();
+        private final Map<String, String> values = new HashMap<>();
 
         void addIfSmallConstantInfo(String key, String value) {
             tryValue(key, value);
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
index 20993583d41..250ae9a4883 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
@@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
-import com.yahoo.text.Utf8;
 
 import java.io.IOException;
 import java.util.Optional;
@@ -44,7 +43,7 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC
     @Override
     protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException {
         try {
-            return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(constantsPath.append(fileReference.value()).toFile())));
+            return new RankingExpression(name, readExpressionFromFile(constantsPath.append(fileReference.value()).toFile()));
         } catch (IOException e) {
             throw new IllegalArgumentException("Missing expression file '" + fileReference.value() + "' for expression '" + name + "'.", e);
         }
diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4
new file mode 100644
index 00000000000..30f23b963db
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4
diff --git a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
index b82aca648a8..aab15fd400f 100644
--- a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
+++ b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
@@ -84,8 +84,8 @@ rankprofile[3].fef.property[1].name "rankingExpression(firstphase).expressionNam
 rankprofile[3].fef.property[1].value "overflow.firstphase"
 rankprofile[3].fef.property[2].name "vespa.rank.secondphase"
 rankprofile[3].fef.property[2].value "rankingExpression(secondphase)"
-rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript"
-rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")"
+rankprofile[3].fef.property[3].name "rankingExpression(secondphase).expressionName"
+rankprofile[3].fef.property[3].value "overflow.secondphase"
 rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize"
 rankprofile[3].fef.property[4].value "101"
 rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize"
diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
index f7106d02ad3..8cb02567538 100644
--- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
+++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
@@ -1,2 +1,4 @@
 expression[0].name "overflow.firstphase"
 expression[0].fileref "overflow.firstphase.expr"
+expression[1].name "overflow.secondphase"
+expression[1].fileref "overflow.firstphase.expr.lz4"
diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
index ad51ac1ca38..98228292cc7 100644
--- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
+++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
@@ -82,22 +82,25 @@ public class Compressor {
      * @throws IllegalArgumentException if the compression type is not supported
      */
     public Compression compress(CompressionType requestedCompression, byte[] data, Optional<Integer> uncompressedSize) {
+        return compress(requestedCompression, data, 0, uncompressedSize.orElse(data.length));
+    }
+    public Compression compress(CompressionType requestedCompression, byte[] data, int offset, int len) {
         switch (requestedCompression) {
             case NONE:
-                data = uncompressedSize.isPresent() ? Arrays.copyOf(data, uncompressedSize.get()) : data;
+                if ((offset != 0) || (len != data.length)) {
+                    data = Arrays.copyOfRange(data, offset, offset + len);
+                }
                 return new Compression(CompressionType.NONE, data.length, data);
             case LZ4:
-                int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length;
-                if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
-                byte[] compressedData = getCompressor().compress(data, 0, dataSize);
-                if (compressedData.length + 8 >= dataSize * compressionThresholdFactor)
-                    return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
-                return new Compression(CompressionType.LZ4, dataSize, compressedData);
+                if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data);
+                byte[] compressedData = getCompressor().compress(data, offset, len);
+                if (compressedData.length + 8 >= len * compressionThresholdFactor)
+                    return new Compression(CompressionType.INCOMPRESSIBLE, len, data);
+                return new Compression(CompressionType.LZ4, len, compressedData);
             case ZSTD:
-                int dataLength = uncompressedSize.orElse(data.length);
-                if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data);
-                byte[] compressed = zstdCompressor.compress(data, 0, dataLength);
-                return new Compression(CompressionType.ZSTD, dataLength, compressed);
+                if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data);
+                byte[] compressed = zstdCompressor.compress(data, offset, len);
+                return new Compression(CompressionType.ZSTD, len, compressed);
             default:
                 throw new IllegalArgumentException(requestedCompression + " is not supported");
         }
diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
index fa32ec1ff9c..64140ab9706 100644
--- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
+++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
@@ -2,8 +2,25 @@
 package com.yahoo.io;
 
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import java.io.Writer;
 import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.util.List;
@@ -19,7 +36,7 @@ import java.nio.ByteBuffer;
  */
 public abstract class IOUtils {
 
-    static private final Charset utf8Charset = Charset.forName("utf-8");
+    static private final Charset utf8Charset = StandardCharsets.UTF_8;
 
     /** Closes a writer, or does nothing if the writer is null */
     public static void closeWriter(Writer writer) {
@@ -341,7 +358,7 @@ public abstract class IOUtils {
     public static String readFile(File file) throws IOException {
         try {
             if (file == null) return null;
-            return new String(Files.readAllBytes(file.toPath()), "utf-8");
+            return Files.readString(file.toPath(), utf8Charset);
         }
         catch (NoSuchFileException e) {
             throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'");
@@ -370,9 +387,7 @@ public abstract class IOUtils {
         if (lengthL>Integer.MAX_VALUE)
             throw new IllegalArgumentException("File too big for byte array: "+file.getCanonicalPath());
 
-        InputStream in = null;
-        try {
-            in = new FileInputStream(file);
+        try (InputStream in = new FileInputStream(file)) {
             int length = (int)lengthL;
             byte[] array = new byte[length];
             int offset = 0;
@@ -381,10 +396,6 @@ public abstract class IOUtils {
                 offset += count;
             return array;
         }
-        finally {
-            if (in != null)
-                in.close();
-        }
     }
 
     /**
diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java
index 26c76de4bc1..20ed0fc8018 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java
@@ -33,16 +33,16 @@ final class BufferedOutput {
 
     public int position() { return pos; }
 
-    final void put(byte b) {
+    void put(byte b) {
         reserve(1);
         buf[pos++] = b;
     }
 
-    final void absolutePut(int position, byte b) {
+    void absolutePut(int position, byte b) {
         buf[position] = b;
     }
 
-    final void put(byte[] bytes) {
+    void put(byte[] bytes) {
         reserve(bytes.length);
         for (byte b : bytes) {
             buf[pos++] = b;