diff options
author | Jon Bratseth <bratseth@gmail.com> | 2023-03-07 16:10:29 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-07 16:10:29 +0100 |
commit | 9b627ff454f85763f7a4c9c95b6ecec7a2ec6484 (patch) | |
tree | 36be2d1f87214a9ac10beed9381d5269f3f4f894 /model-evaluation | |
parent | 56057c40b54fc5b031ab9c2bfea9450e0ee77993 (diff) | |
parent | b9995e437a32e28b31d49c18eb653d1b2f08a5f0 (diff) |
Merge pull request #26343 from vespa-engine/arnej/simpler-test-harness
Arnej/simpler test harness
Diffstat (limited to 'model-evaluation')
-rw-r--r-- | model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java | 17 | ||||
-rw-r--r-- | model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java | 2 | ||||
-rw-r--r-- | model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java | 4 | ||||
-rw-r--r-- | model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java | 2 | ||||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read) | bin | 2409 -> 2409 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read) | bin | 1881616 -> 1881616 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read) | bin | 806 -> 806 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read) | bin | 240013 -> 240013 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read) | bin | 86 -> 86 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read) | bin | 8010 -> 8010 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable) | bin | 62733 -> 62733 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1) | bin | 86 -> 86 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read) | bin | 86 -> 86 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read.tbf (renamed from model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read) | bin | 62733 -> 62733 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/models/ranking-constants.cfg | 22 | ||||
-rw-r--r-- | model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr (renamed from model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr) | 0 | ||||
-rw-r--r-- | model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr.lz4 (renamed from model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4) | bin | 1520 -> 1520 bytes | |||
-rw-r--r-- | model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg | 4 |
18 files changed, 34 insertions, 17 deletions
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index ab2f53db863..3f9b86e67e4 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; @@ -9,6 +10,9 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.io.File; +import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -36,8 +40,19 @@ public class ModelTester { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId(path, "onnx-models.cfg")); + Map<String, File> fileMap = new HashMap<>(); + for (var cfgEntry : onnxModelsConfig.model()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + for (var cfgEntry : constantsConfig.constant()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + for (var cfgEntry : expressionsConfig.expression()) { + fileMap.put(cfgEntry.fileref().value(), new File(path + cfgEntry.fileref().value())); + } + var fileAcquirer = MockFileAcquirer.returnFiles(fileMap); - return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), MockFileAcquirer.returnFile(null)) + return new RankProfilesConfigImporter(fileAcquirer, new OnnxRuntime()) .importFrom(config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 3cd04db8edd..126d3492039 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -142,7 +142,7 @@ public class ModelsEvaluatorTest { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR).append("constants"), MockFileAcquirer.returnFile(null)), + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(CONFIG_DIR), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java index 0dd3bd29a2c..c166128549f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java @@ -32,8 +32,10 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC @Override protected Tensor readTensorFromFile(String name, TensorType type, FileReference fileReference) { try { + var path = constantsPath.append(fileReference.value()); + var file = path.toFile(); return TypedBinaryFormat.decode(Optional.of(type), - GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); + GrowableByteBuffer.wrap(IOUtils.readFileBytes(file))); } catch (IOException e) { log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 14da15f60d0..8b6cad1914f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -349,7 +349,7 @@ public class ModelsEvaluationHandlerTest { RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId("ranking-expressions.cfg")); OnnxModelsConfig onnxModelsConfig = ConfigGetter.getConfig(OnnxModelsConfig.class, fileConfigId("onnx-models.cfg")); - return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR).append("constants"), MockFileAcquirer.returnFile(null)), + return new ModelsEvaluator(new RankProfilesConfigImporterWithMockedConstants(Path.fromString(MODELS_DIR), MockFileAcquirer.returnFile(null)), config, constantsConfig, expressionsConfig, onnxModelsConfig); } diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read.tbf Binary files differindex bac75f7b1e7..bac75f7b1e7 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read.tbf Binary files differindex bd3f05be826..bd3f05be826 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read.tbf Binary files differindex fca7c76df3f..fca7c76df3f 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read.tbf Binary files differindex 396dea8f4bc..396dea8f4bc 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read.tbf Binary files differindex 42f85478c10..42f85478c10 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read.tbf Binary files differindex a3cc7d765f6..a3cc7d765f6 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable.tbf Binary files differindex e768328bff5..e768328bff5 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1.tbf Binary files differindex 4fa0eadb0d3..4fa0eadb0d3 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read.tbf Binary files differindex 4fa0eadb0d3..4fa0eadb0d3 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read.tbf Binary files differindex e768328bff5..e768328bff5 100644 --- a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read.tbf diff --git a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg b/model-evaluation/src/test/resources/config/models/ranking-constants.cfg index 2b7495ace5e..335a7ec526b 100644 --- a/model-evaluation/src/test/resources/config/models/ranking-constants.cfg +++ b/model-evaluation/src/test/resources/config/models/ranking-constants.cfg @@ -1,30 +1,30 @@ constant[0].name "mnist_saved_dnn_hidden1_weights_read" -constant[0].fileref "" +constant[0].fileref "constants/mnist_saved_dnn_hidden1_weights_read.tbf" constant[0].type "tensor(d3[300],d4[784])" constant[1].name "mnist_saved_dnn_hidden2_weights_read" -constant[1].fileref "" +constant[1].fileref "constants/mnist_saved_dnn_hidden2_weights_read.tbf" constant[1].type "tensor(d2[100],d3[300])" constant[2].name "mnist_softmax_saved_layer_Variable_1_read" -constant[2].fileref "" +constant[2].fileref "constants/mnist_softmax_saved_layer_Variable_1_read.tbf" constant[2].type "tensor(d1[10])" constant[3].name "mnist_saved_dnn_hidden1_bias_read" -constant[3].fileref "" +constant[3].fileref "constants/mnist_saved_dnn_hidden1_bias_read.tbf" constant[3].type "tensor(d3[300])" constant[4].name "mnist_saved_dnn_hidden2_bias_read" -constant[4].fileref "" +constant[4].fileref "constants/mnist_saved_dnn_hidden2_bias_read.tbf" constant[4].type "tensor(d2[100])" constant[5].name "mnist_softmax_Variable" -constant[5].fileref "" +constant[5].fileref "constants/mnist_softmax_Variable.tbf" constant[5].type "tensor(d1[10],d2[784])" constant[6].name "mnist_saved_dnn_outputs_weights_read" -constant[6].fileref "" +constant[6].fileref "constants/mnist_saved_dnn_outputs_weights_read.tbf" constant[6].type "tensor(d1[10],d2[100])" constant[7].name "mnist_softmax_saved_layer_Variable_read" -constant[7].fileref "" +constant[7].fileref "constants/mnist_softmax_saved_layer_Variable_read.tbf" constant[7].type "tensor(d1[10],d2[784])" constant[8].name "mnist_softmax_Variable_1" -constant[8].fileref "" +constant[8].fileref "constants/mnist_softmax_Variable_1.tbf" constant[8].type "tensor(d1[10])" constant[9].name "mnist_saved_dnn_outputs_bias_read" -constant[9].fileref "" -constant[9].type "tensor(d1[10])"
\ No newline at end of file +constant[9].fileref "constants/mnist_saved_dnn_outputs_bias_read.tbf" +constant[9].type "tensor(d1[10])" diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr index 70a3a2eb6cc..70a3a2eb6cc 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr +++ b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr.lz4 Binary files differindex 30f23b963db..30f23b963db 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 +++ b/model-evaluation/src/test/resources/config/rankexpression/expressions/overflow.firstphase.expr.lz4 diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg index 8cb02567538..e8fde6fafb2 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg @@ -1,4 +1,4 @@ expression[0].name "overflow.firstphase" -expression[0].fileref "overflow.firstphase.expr" +expression[0].fileref "expressions/overflow.firstphase.expr" expression[1].name "overflow.secondphase" -expression[1].fileref "overflow.firstphase.expr.lz4" +expression[1].fileref "expressions/overflow.firstphase.expr.lz4" |