diff options
Diffstat (limited to 'model-evaluation/src/test')
5 files changed, 15 insertions, 102 deletions
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index d94e5b2af1b..60cf0d25ded 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -3,10 +3,8 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; -import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; import java.io.File; @@ -20,9 +18,15 @@ public class ModelsEvaluatorTest { private static final double delta = 0.00000000001; + private ModelsEvaluator createModels() { + String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg"; + RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig(""); + return new ModelsEvaluator(config); + } + @Test public void testTensorEvaluation() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); FunctionEvaluator function = models.evaluatorOf("macros", "fourtimessum"); function.bind("var1", Tensor.from("{{x:0}:3,{x:1}:5}")); function.bind("var2", Tensor.from("{{x:0}:7,{x:1}:11}")); @@ -31,7 +35,7 @@ public class ModelsEvaluatorTest { @Test public void testEvaluationDependingOnMacroTakingArguments() { - ModelsEvaluator models = createModels("src/test/resources/config/rankexpression/"); + ModelsEvaluator models = createModels(); FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); function.bind("match", 3); function.bind("rankBoost", 5); @@ -42,14 +46,6 @@ public class ModelsEvaluatorTest { // TODO: Test that binding nonexisting variable doesn't work // TODO: Test that rebinding doesn't work // TODO: Test with nested macros - - private ModelsEvaluator createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - return new ModelsEvaluator(config, constantsConfig); - } + // TODO: Test TF/ONNX model } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java index 84e01e58280..d45372fc7da 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java @@ -3,10 +3,8 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; -import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; import java.io.File; @@ -23,41 +21,14 @@ import static org.junit.Assert.assertNotNull; public class RankProfilesImporterTest { @Test - public void testImportingModels() { - Map<String, Model> models = createModels("src/test/resources/config/models/"); - - assertEquals(4, models.size()); - - Model xgboost = models.get("xgboost_2_2"); - assertFunction("xgboost_2_2", - "(optimized sum of condition trees of size 192 bytes)", - xgboost); - - Model onnxMnistSoftmax = models.get("mnist_softmax"); - assertFunction("default.add", - "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))", - onnxMnistSoftmax); - assertEquals("tensor(d1[10],d2[784])", - onnxMnistSoftmax.evaluatorOf("default.add").context().get("constant(mnist_softmax_Variable)").type().toString()); - - Model tfMnistSoftmax = models.get("mnist_softmax_saved"); - assertFunction("serving_default.y", - "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))", - tfMnistSoftmax); - - Model tfMnist = models.get("mnist_saved"); - assertFunction("serving_default.y", - "join(reduce(join(map(join(reduce(join(join(join(rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), 0.009999999776482582, f(a,b)(a * b)), rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.050701 * if (a >= 0, a, 1.673263 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))", - tfMnist); - } - - @Test - public void testImportingRankExpressions() { - Map<String, Model> models = createModels("src/test/resources/config/rankexpression/"); - + public void testImporting() { + String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg"; + RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig(""); + Map<String, Model> models = new RankProfilesConfigImporter().importFrom(config); assertEquals(18, models.size()); Model macros = models.get("macros"); + assertNotNull(macros); assertEquals("macros", macros.name()); assertEquals(4, macros.functions().size()); assertFunction("fourtimessum", "4 * (var1 + var2)", macros); @@ -73,9 +44,8 @@ public class RankProfilesImporterTest { } private void assertFunction(String name, String expression, Model model) { - assertNotNull("Model is present in config", model); ExpressionFunction function = model.function(name); - assertNotNull("Function '" + name + "' is in " + model, function); + assertNotNull(function); assertEquals(name, function.getName()); assertEquals(expression, function.getBody().getRoot().toString()); } @@ -87,13 +57,4 @@ public class RankProfilesImporterTest { assertEquals(expression, function.getBody().getRoot().toString()); } - private Map<String, Model> createModels(String path) { - Path configDir = Path.fromString(path); - RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), - RankProfilesConfig.class).getConfig(""); - RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), - RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporter().importFrom(config, constantsConfig); - } - } diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg deleted file mode 100644 index 1cc36f75158..00000000000 --- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg +++ /dev/null @@ -1,14 +0,0 @@ -rankprofile[0].name "mnist_saved" -rankprofile[0].fef.property[0].name "rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add).rankingScript" -rankprofile[0].fef.property[0].value "join(reduce(join(rename(input, (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))" -rankprofile[0].fef.property[1].name "rankingExpression(serving_default.y).rankingScript" -rankprofile[0].fef.property[1].value "join(reduce(join(map(join(reduce(join(join(join(rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), 0.009999999776482582, f(a,b)(a * b)), rankingExpression(imported_ml_macro_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.050701 * if (a >= 0, a, 1.673263 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))" -rankprofile[1].name "xgboost_2_2" -rankprofile[1].fef.property[0].name "rankingExpression(xgboost_2_2).rankingScript" -rankprofile[1].fef.property[0].value "if (f29 < -0.1234567, if (f56 < -0.242398, 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (f60 < -0.482947, if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)" -rankprofile[2].name "mnist_softmax_saved" -rankprofile[2].fef.property[0].name "rankingExpression(serving_default.y).rankingScript" -rankprofile[2].fef.property[0].value "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))" -rankprofile[3].name "mnist_softmax" -rankprofile[3].fef.property[0].name "rankingExpression(default.add).rankingScript" -rankprofile[3].fef.property[0].value "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))" diff --git a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg b/model-evaluation/src/test/resources/config/models/ranking-constants.cfg deleted file mode 100644 index 2b7495ace5e..00000000000 --- a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg +++ /dev/null @@ -1,30 +0,0 @@ -constant[0].name "mnist_saved_dnn_hidden1_weights_read" -constant[0].fileref "" -constant[0].type "tensor(d3[300],d4[784])" -constant[1].name "mnist_saved_dnn_hidden2_weights_read" -constant[1].fileref "" -constant[1].type "tensor(d2[100],d3[300])" -constant[2].name "mnist_softmax_saved_layer_Variable_1_read" -constant[2].fileref "" -constant[2].type "tensor(d1[10])" -constant[3].name "mnist_saved_dnn_hidden1_bias_read" -constant[3].fileref "" -constant[3].type "tensor(d3[300])" -constant[4].name "mnist_saved_dnn_hidden2_bias_read" -constant[4].fileref "" -constant[4].type "tensor(d2[100])" -constant[5].name "mnist_softmax_Variable" -constant[5].fileref "" -constant[5].type "tensor(d1[10],d2[784])" -constant[6].name "mnist_saved_dnn_outputs_weights_read" -constant[6].fileref "" -constant[6].type "tensor(d1[10],d2[100])" -constant[7].name "mnist_softmax_saved_layer_Variable_read" -constant[7].fileref "" -constant[7].type "tensor(d1[10],d2[784])" -constant[8].name "mnist_softmax_Variable_1" -constant[8].fileref "" -constant[8].type "tensor(d1[10])" -constant[9].name "mnist_saved_dnn_outputs_bias_read" -constant[9].fileref "" -constant[9].type "tensor(d1[10])"
\ No newline at end of file diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-constants.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-constants.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/model-evaluation/src/test/resources/config/rankexpression/ranking-constants.cfg +++ /dev/null |