diff options
15 files changed, 45 insertions, 25 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java index a16c32d47ab..96886941fe1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java @@ -3,9 +3,12 @@ package com.yahoo.config.model; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; +import ai.vespa.models.evaluation.RankProfilesConfigImporter; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.VespaModel; @@ -74,7 +77,8 @@ public class ModelEvaluationTest { assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); - ModelsEvaluator evaluator = new ModelsEvaluator(config, constantsConfig); + ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter() + .importFrom(config, constantsConfig)); assertEquals(4, evaluator.models().size()); @@ -107,4 +111,13 @@ public class ModelEvaluationTest { assertNotNull(tensorflow_mnist_softmax.evaluatorOf("serving_default", "y")); } + // We don't have function file distribution so just return empty tensor constants + private static class ToleratingMissingConstantFilesRankProfilesConfigImporter extends RankProfilesConfigImporter { + + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + return Tensor.from(type, "{}"); + } + + } + } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 48c71b5a04a..b507c5d7f08 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -3,6 +3,7 @@ package ai.vespa.models.evaluation; import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -22,8 +23,13 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap<String, Model> models; + @Inject public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + } + + public ModelsEvaluator(Map<String, Model> models) { + this.models = ImmutableMap.copyOf(models); } /** Returns the models of this as an immutable map */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index cd21a0a6813..98634e184bf 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -35,7 +35,7 @@ import java.util.logging.Logger; * * @author bratseth */ -class RankProfilesConfigImporter { +public class RankProfilesConfigImporter { private static final Logger log = Logger.getLogger("CONSTANTS"); @@ -43,7 +43,7 @@ class RankProfilesConfigImporter { * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. */ - Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { + public Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { try { Map<String, Model> models = new HashMap<>(); for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) { @@ -120,23 +120,9 @@ class RankProfilesConfigImporter { return constants; } - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { try { - // TODO: Only allow these two fallbacks in testing mode - if (fileReference.isEmpty()) { // this may be the case in unit tests - log.warning("Got empty file reference for constant '" + name + "', using an empty tensor"); - return Tensor.from(type, "{}"); - } File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); - if ( ! dir.exists()) { // this may be the case in unit tests - log.warning("Got reference to nonexisting file " + dir + "e for constant '" + name + - "', using an empty tensor"); - return Tensor.from(type, "{}"); - } - - // TODO: Move these 2 lines to FileReference - - dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); File file = dir.listFiles()[0]; // directory contains one file having the original name if (file.getName().endsWith(".tbf")) diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 63e17e37bde..d2729ab7b6e 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -3,15 +3,20 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import static org.junit.Assert.assertEquals; @@ -38,7 +43,8 @@ public class ModelTester { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporterWithMockedConstants().importFrom(config, constantsConfig); + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants")) + .importFrom(config, constantsConfig); } public void assertFunction(String name, String expression, Model model) { @@ -61,15 +67,23 @@ public class ModelTester { private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName()); - Map<String, Tensor> constants = new HashMap<>(); + private final Path constantsPath; + + public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) { + this.constantsPath = constantsPath; + } @Override - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { - if ( ! constants.containsKey(name)) { - log.warning("Missing a mocked tensor constant for '" + name + "': Returning an empty tensor"); + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + try { + return TypedBinaryFormat.decode(Optional.of(type), + GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); + } + catch (IOException e) { + log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + + ". Returning an empty tensor"); return Tensor.from(type, "{}"); } - return constants.get(name); } } diff --git a/model-evaluation/src/test/resources/config/models/constants/README b/model-evaluation/src/test/resources/config/models/constants/README new file mode 100644 index 00000000000..4a274aa95c8 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/README @@ -0,0 +1 @@ +These constants was created by writing TypedBinaryFormat.encode(tensor) on each large constant produced by these models. diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read Binary files differnew file mode 100644 index 00000000000..bac75f7b1e7 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read Binary files differnew file mode 100644 index 00000000000..bd3f05be826 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read Binary files differnew file mode 100644 index 00000000000..fca7c76df3f --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read Binary files differnew file mode 100644 index 00000000000..396dea8f4bc --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read Binary files differnew file mode 100644 index 00000000000..42f85478c10 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read Binary files differnew file mode 100644 index 00000000000..a3cc7d765f6 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable Binary files differnew file mode 100644 index 00000000000..e768328bff5 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 Binary files differnew file mode 100644 index 00000000000..4fa0eadb0d3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read Binary files differnew file mode 100644 index 00000000000..5cc9575b971 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read Binary files differnew file mode 100644 index 00000000000..70a6fd42c91 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read |