diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-30 12:11:35 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-30 12:11:35 +0200 |
commit | 37eaa36494321b3d3c1427fabf34ebe020f4b9ac (patch) | |
tree | 6868462525801a65eab185fa1577d891772e8546 /model-evaluation | |
parent | 11e7a365a7f6e63b109f24f3f81ff4aaee9bad72 (diff) |
lz4 compress blobs.
Diffstat (limited to 'model-evaluation')
5 files changed, 16 insertions, 7 deletions
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index eccc236f0ca..fbfd34814ac 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -17,8 +17,10 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import net.jpountz.lz4.LZ4FrameInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -207,10 +209,16 @@ public class RankProfilesConfigImporter { return expressions; } + protected final String readExpressionFromFile(File file) throws IOException { + return (file.getName().endsWith(".lz4")) + ? Utf8.toString(IOUtils.readBytes(new LZ4FrameInputStream(new FileInputStream(file)), 65536)) + : Utf8.toString(IOUtils.readFileBytes(file)); + } + protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException { try { File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS); - return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(file))); + return new RankingExpression(name, readExpressionFromFile(file)); } catch (InterruptedException e) { throw new IllegalStateException("Gave up waiting for expression " + name); @@ -245,8 +253,8 @@ public class RankProfilesConfigImporter { private static final Pattern valuePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.value"); private static final Pattern typePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.type"); - private Map<String, TensorType> types = new HashMap<>(); - private Map<String, String> values = new HashMap<>(); + private final Map<String, TensorType> types = new HashMap<>(); + private final Map<String, String> values = new HashMap<>(); void addIfSmallConstantInfo(String key, String value) { tryValue(key, value); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java index 20993583d41..250ae9a4883 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java @@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; -import com.yahoo.text.Utf8; import java.io.IOException; import java.util.Optional; @@ -44,7 +43,7 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC @Override protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException { try { - return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(constantsPath.append(fileReference.value()).toFile()))); + return new RankingExpression(name, readExpressionFromFile(constantsPath.append(fileReference.value()).toFile())); } catch (IOException e) { throw new IllegalArgumentException("Missing expression file '" + fileReference.value() + "' for expression '" + name + "'.", e); } diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 Binary files differnew file mode 100644 index 00000000000..30f23b963db --- /dev/null +++ b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 diff --git a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg index b82aca648a8..aab15fd400f 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg @@ -84,8 +84,8 @@ rankprofile[3].fef.property[1].name "rankingExpression(firstphase).expressionNam rankprofile[3].fef.property[1].value "overflow.firstphase" rankprofile[3].fef.property[2].name "vespa.rank.secondphase" rankprofile[3].fef.property[2].value "rankingExpression(secondphase)" -rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript" -rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")" +rankprofile[3].fef.property[3].name "rankingExpression(secondphase).expressionName" +rankprofile[3].fef.property[3].value "overflow.secondphase" rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize" rankprofile[3].fef.property[4].value "101" rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize" diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg index f7106d02ad3..8cb02567538 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg @@ -1,2 +1,4 @@ expression[0].name "overflow.firstphase" expression[0].fileref "overflow.firstphase.expr" +expression[1].name "overflow.secondphase" +expression[1].fileref "overflow.firstphase.expr.lz4" |