From 5a3519d9d26df9b18e680f2bb7dbe9e3f25bcb0b Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 11 Sep 2018 11:20:39 +0200 Subject: Import small constants --- .../evaluation/RankProfilesConfigImporter.java | 50 +++++++++++++++++++++- .../evaluation/SmallConstantImportingTest.java | 25 +++++++++++ .../config/smallconstant/rank-profiles.cfg | 9 ++++ .../config/smallconstant/ranking-constants.cfg | 0 .../config/smallconstant/smallconstant.sd | 18 ++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 model-evaluation/src/test/java/ai/vespa/models/evaluation/SmallConstantImportingTest.java create mode 100644 model-evaluation/src/test/resources/config/smallconstant/rank-profiles.cfg create mode 100644 model-evaluation/src/test/resources/config/smallconstant/ranking-constants.cfg create mode 100644 model-evaluation/src/test/resources/config/smallconstant/smallconstant.sd (limited to 'model-evaluation') diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 87ac53488db..d2fca309a19 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Converts RankProfilesConfig instances to RankingExpressions for evaluation. @@ -60,10 +62,11 @@ public class RankProfilesConfigImporter { throws ParseException { List functions = new ArrayList<>(); Map referencedFunctions = new HashMap<>(); + SmallConstantsInfo smallConstantsInfo = new SmallConstantsInfo(); ExpressionFunction firstPhase = null; ExpressionFunction secondPhase = null; - List constants = readConstants(constantsConfig); + List constants = readLargeConstants(constantsConfig); for (RankProfilesConfig.Rankprofile.Fef.Property property : profile.fef().property()) { Optional reference = FunctionReference.fromSerial(property.name()); @@ -86,12 +89,17 @@ public class RankProfilesConfigImporter { secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } + else { + smallConstantsInfo.addIfSmallConstantInfo(property.name(), property.value()); + } } if (functionByName("firstphase", functions) == null && firstPhase != null) // may be already included, depending on body functions.add(firstPhase); if (functionByName("secondphase", functions) == null && secondPhase != null) // may be already included, depending on body functions.add(secondPhase); + constants.addAll(smallConstantsInfo.asConstants()); + try { return new Model(profile.name(), functions, referencedFunctions, constants); } @@ -107,7 +115,7 @@ public class RankProfilesConfigImporter { return null; } - private List readConstants(RankingConstantsConfig constantsConfig) { + private List readLargeConstants(RankingConstantsConfig constantsConfig) { List constants = new ArrayList<>(); for (RankingConstantsConfig.Constant constantConfig : constantsConfig.constant()) { @@ -138,4 +146,42 @@ public class RankProfilesConfigImporter { } } + /** Collected information about small constants */ + private static class SmallConstantsInfo { + + private static final Pattern valuePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.value"); + private static final Pattern typePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.type"); + + private Map types = new HashMap<>(); + private Map values = new HashMap<>(); + + void addIfSmallConstantInfo(String key, String value) { + tryValue(key, value); + tryType(key, value); + } + + private void tryValue(String key, String value) { + Matcher matcher = valuePattern.matcher(key); + if (matcher.matches()) + values.put(matcher.group(1), value); + } + + private void tryType(String key, String value) { + Matcher matcher = typePattern.matcher(key); + if (matcher.matches()) + types.put(matcher.group(1), TensorType.fromSpec(value)); + } + + List asConstants() { + List constants = new ArrayList<>(); + for (Map.Entry entry : values.entrySet()) { + TensorType type = types.get(entry.getKey()); + if (type == null) throw new IllegalStateException("Missing type of '" + entry.getKey() + "'"); // Won't happen + constants.add(new Constant(entry.getKey(), Tensor.from(type, entry.getValue()))); + } + return constants; + } + + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/SmallConstantImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/SmallConstantImportingTest.java new file mode 100644 index 00000000000..5fd2e1de4ed --- /dev/null +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/SmallConstantImportingTest.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SmallConstantImportingTest { + + @Test + public void testImportingSmallConstant() { + ModelTester tester = new ModelTester("src/test/resources/config/smallconstant/"); + + assertEquals(1, tester.models().size()); + + Model model = tester.models().get("my_profile"); + tester.assertFunction("firstphase", "reduce(constant(my_tensor), sum)", model); + assertEquals(3.0, model.evaluatorOf().evaluate().asDouble(), 0.00000000001); + + } + +} diff --git a/model-evaluation/src/test/resources/config/smallconstant/rank-profiles.cfg b/model-evaluation/src/test/resources/config/smallconstant/rank-profiles.cfg new file mode 100644 index 00000000000..840f1a06296 --- /dev/null +++ b/model-evaluation/src/test/resources/config/smallconstant/rank-profiles.cfg @@ -0,0 +1,9 @@ +rankprofile[0].name "my_profile" +rankprofile[0].fef.property[0].name "constant(my_tensor).type" +rankprofile[0].fef.property[0].value "tensor(x{},y{})" +rankprofile[0].fef.property[1].name "constant(my_tensor).value" +rankprofile[0].fef.property[1].value "{{x:1,y:2}:1.0,{x:2,y:1}:2.0}" +rankprofile[0].fef.property[2].name "vespa.rank.firstphase" +rankprofile[0].fef.property[2].value "rankingExpression(firstphase)" +rankprofile[0].fef.property[3].name "rankingExpression(firstphase).rankingScript" +rankprofile[0].fef.property[3].value "reduce(constant(my_tensor), sum)" diff --git a/model-evaluation/src/test/resources/config/smallconstant/ranking-constants.cfg b/model-evaluation/src/test/resources/config/smallconstant/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/model-evaluation/src/test/resources/config/smallconstant/smallconstant.sd b/model-evaluation/src/test/resources/config/smallconstant/smallconstant.sd new file mode 100644 index 00000000000..47d2d95b968 --- /dev/null +++ b/model-evaluation/src/test/resources/config/smallconstant/smallconstant.sd @@ -0,0 +1,18 @@ +search smallconstant { + + document smallconstant { + } + + rank-profile my_profile { + first-phase { + expression: sum(my_tensor) + } + constants { + my_tensor + value: { {x:1,y:2}:1, {x:2,y:1}:2 } + type: tensor(x{},y{}) + } + } + } + +} \ No newline at end of file -- cgit v1.2.3