lz4 compress blobs.

5 files changed, 16 insertions, 7 deletions

diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
index eccc236f0ca..fbfd34814ac 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -17,8 +17,10 @@ import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import net.jpountz.lz4.LZ4FrameInputStream;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
@@ -207,10 +209,16 @@ public class RankProfilesConfigImporter {
         return expressions;
     }
 
+    protected final String readExpressionFromFile(File file) throws IOException {
+        return (file.getName().endsWith(".lz4"))
+            ? Utf8.toString(IOUtils.readBytes(new LZ4FrameInputStream(new FileInputStream(file)), 65536))
+            : Utf8.toString(IOUtils.readFileBytes(file));
+    }
+
     protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException {
         try {
             File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS);
-            return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(file)));
+            return new RankingExpression(name, readExpressionFromFile(file));
         }
         catch (InterruptedException e) {
             throw new IllegalStateException("Gave up waiting for expression " + name);
@@ -245,8 +253,8 @@ public class RankProfilesConfigImporter {
         private static final Pattern valuePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.value");
         private static final Pattern  typePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.type");
 
-        private Map<String, TensorType> types = new HashMap<>();
-        private Map<String, String> values = new HashMap<>();
+        private final Map<String, TensorType> types = new HashMap<>();
+        private final Map<String, String> values = new HashMap<>();
 
         void addIfSmallConstantInfo(String key, String value) {
             tryValue(key, value);
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
index 20993583d41..250ae9a4883 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java
@@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
-import com.yahoo.text.Utf8;
 
 import java.io.IOException;
 import java.util.Optional;
@@ -44,7 +43,7 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC
     @Override
     protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException {
         try {
-            return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(constantsPath.append(fileReference.value()).toFile())));
+            return new RankingExpression(name, readExpressionFromFile(constantsPath.append(fileReference.value()).toFile()));
         } catch (IOException e) {
             throw new IllegalArgumentException("Missing expression file '" + fileReference.value() + "' for expression '" + name + "'.", e);
         }
diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4
new file mode 100644
index 00000000000..30f23b963db
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4
diff --git a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
index b82aca648a8..aab15fd400f 100644
--- a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
+++ b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg
@@ -84,8 +84,8 @@ rankprofile[3].fef.property[1].name "rankingExpression(firstphase).expressionNam
 rankprofile[3].fef.property[1].value "overflow.firstphase"
 rankprofile[3].fef.property[2].name "vespa.rank.secondphase"
 rankprofile[3].fef.property[2].value "rankingExpression(secondphase)"
-rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript"
-rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")"
+rankprofile[3].fef.property[3].name "rankingExpression(secondphase).expressionName"
+rankprofile[3].fef.property[3].value "overflow.secondphase"
 rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize"
 rankprofile[3].fef.property[4].value "101"
 rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize"
diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
index f7106d02ad3..8cb02567538 100644
--- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
+++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg
@@ -1,2 +1,4 @@
 expression[0].name "overflow.firstphase"
 expression[0].fileref "overflow.firstphase.expr"
+expression[1].name "overflow.secondphase"
+expression[1].fileref "overflow.firstphase.expr.lz4"