From 7658c3228d2c5dd6bda073c55521b650c34fcc16 Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Mon, 10 Sep 2018 08:38:06 +0200 Subject: Revert "Test with constants" --- .../yahoo/config/model/ModelEvaluationTest.java | 15 +---------- .../vespa/models/evaluation/ModelsEvaluator.java | 8 +----- .../evaluation/RankProfilesConfigImporter.java | 28 ++++++++++++++++++--- .../models/evaluation/MlModelsImportingTest.java | 14 +++++++++++ .../ai/vespa/models/evaluation/ModelTester.java | 27 ++++++-------------- .../models/evaluation/ModelsEvaluatorTest.java | 2 ++ .../evaluation/RankProfileImportingTest.java | 2 ++ .../test/resources/config/models/constants/README | 1 - .../constants/mnist_saved_dnn_hidden1_bias_read | Bin 2409 -> 0 bytes .../constants/mnist_saved_dnn_hidden1_weights_read | Bin 1881616 -> 0 bytes .../constants/mnist_saved_dnn_hidden2_bias_read | Bin 806 -> 0 bytes .../constants/mnist_saved_dnn_hidden2_weights_read | Bin 240013 -> 0 bytes .../constants/mnist_saved_dnn_outputs_bias_read | Bin 86 -> 0 bytes .../constants/mnist_saved_dnn_outputs_weights_read | Bin 8010 -> 0 bytes .../config/models/constants/mnist_softmax_Variable | Bin 62733 -> 0 bytes .../models/constants/mnist_softmax_Variable_1 | Bin 86 -> 0 bytes .../mnist_softmax_saved_layer_Variable_1_read | Bin 86 -> 0 bytes .../mnist_softmax_saved_layer_Variable_read | Bin 62733 -> 0 bytes 18 files changed, 52 insertions(+), 45 deletions(-) delete mode 100644 model-evaluation/src/test/resources/config/models/constants/README delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read delete mode 100644 model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read diff --git a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java index 96886941fe1..a16c32d47ab 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ModelEvaluationTest.java @@ -3,12 +3,9 @@ package com.yahoo.config.model; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; -import ai.vespa.models.evaluation.RankProfilesConfigImporter; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.VespaModel; @@ -77,8 +74,7 @@ public class ModelEvaluationTest { assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); - ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter() - .importFrom(config, constantsConfig)); + ModelsEvaluator evaluator = new ModelsEvaluator(config, constantsConfig); assertEquals(4, evaluator.models().size()); @@ -111,13 +107,4 @@ public class ModelEvaluationTest { assertNotNull(tensorflow_mnist_softmax.evaluatorOf("serving_default", "y")); } - // We don't have function file distribution so just return empty tensor constants - private static class ToleratingMissingConstantFilesRankProfilesConfigImporter extends RankProfilesConfigImporter { - - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { - return Tensor.from(type, "{}"); - } - - } - } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index b507c5d7f08..48c71b5a04a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -3,7 +3,6 @@ package ai.vespa.models.evaluation; import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -23,13 +22,8 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap models; - @Inject public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); - } - - public ModelsEvaluator(Map models) { - this.models = ImmutableMap.copyOf(models); + models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); } /** Returns the models of this as an immutable map */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 00f60654318..cd21a0a6813 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -1,11 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -17,6 +21,8 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,13 +35,15 @@ import java.util.logging.Logger; * * @author bratseth */ -public class RankProfilesConfigImporter { +class RankProfilesConfigImporter { + + private static final Logger log = Logger.getLogger("CONSTANTS"); /** * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. */ - public Map importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { + Map importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { try { Map models = new HashMap<>(); for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) { @@ -112,9 +120,23 @@ public class RankProfilesConfigImporter { return constants; } - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + Tensor readTensorFromFile(String name, TensorType type, String fileReference) { try { + // TODO: Only allow these two fallbacks in testing mode + if (fileReference.isEmpty()) { // this may be the case in unit tests + log.warning("Got empty file reference for constant '" + name + "', using an empty tensor"); + return Tensor.from(type, "{}"); + } File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); + if ( ! dir.exists()) { // this may be the case in unit tests + log.warning("Got reference to nonexisting file " + dir + "e for constant '" + name + + "', using an empty tensor"); + return Tensor.from(type, "{}"); + } + + // TODO: Move these 2 lines to FileReference + + dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); File file = dir.listFiles()[0]; // directory contains one file having the original name if (file.getName().endsWith(".tbf")) diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java index 716965784e3..a823f16d727 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java @@ -1,11 +1,25 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * Tests instantiating models from rank-profiles configs. diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 0a400f7126f..63e17e37bde 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -3,19 +3,15 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import java.io.IOException; +import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.logging.Logger; import static org.junit.Assert.assertEquals; @@ -42,8 +38,7 @@ public class ModelTester { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants")) - .importFrom(config, constantsConfig); + return new RankProfilesConfigImporterWithMockedConstants().importFrom(config, constantsConfig); } public void assertFunction(String name, String expression, Model model) { @@ -66,23 +61,15 @@ public class ModelTester { private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName()); - private final Path constantsPath; - - public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) { - this.constantsPath = constantsPath; - } + Map constants = new HashMap<>(); @Override - protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { - try { - return TypedBinaryFormat.decode(Optional.of(type), - GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); - } - catch (IOException e) { - log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + - ". Returning an empty tensor"); + Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + if ( ! constants.containsKey(name)) { + log.warning("Missing a mocked tensor constant for '" + name + "': Returning an empty tensor"); return Tensor.from(type, "{}"); } + return constants.get(name); } } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 0708d872ff6..d94e5b2af1b 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import org.junit.Test; +import java.io.File; + import static org.junit.Assert.assertEquals; /** diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java index 20abd9c0fb0..210ffb823b2 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java @@ -3,6 +3,8 @@ package ai.vespa.models.evaluation; import org.junit.Test; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** diff --git a/model-evaluation/src/test/resources/config/models/constants/README b/model-evaluation/src/test/resources/config/models/constants/README deleted file mode 100644 index 4a274aa95c8..00000000000 --- a/model-evaluation/src/test/resources/config/models/constants/README +++ /dev/null @@ -1 +0,0 @@ -These constants was created by writing TypedBinaryFormat.encode(tensor) on each large constant produced by these models. diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read deleted file mode 100644 index bac75f7b1e7..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read deleted file mode 100644 index bd3f05be826..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read deleted file mode 100644 index fca7c76df3f..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read deleted file mode 100644 index 396dea8f4bc..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read deleted file mode 100644 index 42f85478c10..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read deleted file mode 100644 index a3cc7d765f6..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable deleted file mode 100644 index e768328bff5..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 deleted file mode 100644 index 4fa0eadb0d3..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read deleted file mode 100644 index 5cc9575b971..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read and /dev/null differ diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read deleted file mode 100644 index 70a6fd42c91..00000000000 Binary files a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read and /dev/null differ -- cgit v1.2.3