diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-10 08:47:00 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-09-10 08:47:00 +0200 |
commit | 48f9a4071b5f6f75114cd1b94eb09bec36f77061 (patch) | |
tree | 4fe427fe2f1aab3f93c476b58e2b743d5543f73b /model-evaluation/src | |
parent | 68708408abd1d47bb8f5e5a1d80855739b717c2f (diff) |
Revert "Merge pull request #6859 from vespa-engine/revert-6855-bratseth/test-with-constants"
This reverts commit 68708408abd1d47bb8f5e5a1d80855739b717c2f, reversing
changes made to 56591420e8cb8c6a9085d7327b977c3c5b1971b1.
Diffstat (limited to 'model-evaluation/src')
17 files changed, 31 insertions, 51 deletions
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 48c71b5a04a..b507c5d7f08 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -3,6 +3,7 @@ package ai.vespa.models.evaluation; import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -22,8 +23,13 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap<String, Model> models; + @Inject public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + } + + public ModelsEvaluator(Map<String, Model> models) { + this.models = ImmutableMap.copyOf(models); } /** Returns the models of this as an immutable map */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index cd21a0a6813..00f60654318 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -1,15 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -21,8 +17,6 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,15 +29,13 @@ import java.util.logging.Logger; * * @author bratseth */ -class RankProfilesConfigImporter { - - private static final Logger log = Logger.getLogger("CONSTANTS"); +public class RankProfilesConfigImporter { /** * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. */ - Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { + public Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { try { Map<String, Model> models = new HashMap<>(); for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) { @@ -120,23 +112,9 @@ class RankProfilesConfigImporter { return constants; } - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { try { - // TODO: Only allow these two fallbacks in testing mode - if (fileReference.isEmpty()) { // this may be the case in unit tests - log.warning("Got empty file reference for constant '" + name + "', using an empty tensor"); - return Tensor.from(type, "{}"); - } File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); - if ( ! dir.exists()) { // this may be the case in unit tests - log.warning("Got reference to nonexisting file " + dir + "e for constant '" + name + - "', using an empty tensor"); - return Tensor.from(type, "{}"); - } - - // TODO: Move these 2 lines to FileReference - - dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); File file = dir.listFiles()[0]; // directory contains one file having the original name if (file.getName().endsWith(".tbf")) diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java index a823f16d727..716965784e3 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java @@ -1,25 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; -import com.yahoo.path.Path; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; -import java.io.File; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * Tests instantiating models from rank-profiles configs. diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 63e17e37bde..0a400f7126f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -3,15 +3,19 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import java.util.HashMap; +import java.io.IOException; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import static org.junit.Assert.assertEquals; @@ -38,7 +42,8 @@ public class ModelTester { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporterWithMockedConstants().importFrom(config, constantsConfig); + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants")) + .importFrom(config, constantsConfig); } public void assertFunction(String name, String expression, Model model) { @@ -61,15 +66,23 @@ public class ModelTester { private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName()); - Map<String, Tensor> constants = new HashMap<>(); + private final Path constantsPath; + + public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) { + this.constantsPath = constantsPath; + } @Override - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { - if ( ! constants.containsKey(name)) { - log.warning("Missing a mocked tensor constant for '" + name + "': Returning an empty tensor"); + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + try { + return TypedBinaryFormat.decode(Optional.of(type), + GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); + } + catch (IOException e) { + log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + + ". Returning an empty tensor"); return Tensor.from(type, "{}"); } - return constants.get(name); } } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index d94e5b2af1b..0708d872ff6 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -9,8 +9,6 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; -import java.io.File; - import static org.junit.Assert.assertEquals; /** diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java index 210ffb823b2..20abd9c0fb0 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java @@ -3,8 +3,6 @@ package ai.vespa.models.evaluation; import org.junit.Test; -import java.util.Map; - import static org.junit.Assert.assertEquals; /** diff --git a/model-evaluation/src/test/resources/config/models/constants/README b/model-evaluation/src/test/resources/config/models/constants/README new file mode 100644 index 00000000000..4a274aa95c8 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/README @@ -0,0 +1 @@ +These constants was created by writing TypedBinaryFormat.encode(tensor) on each large constant produced by these models. diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read Binary files differnew file mode 100644 index 00000000000..bac75f7b1e7 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read Binary files differnew file mode 100644 index 00000000000..bd3f05be826 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read Binary files differnew file mode 100644 index 00000000000..fca7c76df3f --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read Binary files differnew file mode 100644 index 00000000000..396dea8f4bc --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read Binary files differnew file mode 100644 index 00000000000..42f85478c10 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read Binary files differnew file mode 100644 index 00000000000..a3cc7d765f6 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable Binary files differnew file mode 100644 index 00000000000..e768328bff5 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 Binary files differnew file mode 100644 index 00000000000..4fa0eadb0d3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read Binary files differnew file mode 100644 index 00000000000..5cc9575b971 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read Binary files differnew file mode 100644 index 00000000000..70a6fd42c91 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read |