diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-07 15:31:02 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-09-07 15:31:02 +0200 |
commit | 8f38e82a52561c4d6088f381f3c115c2b2976d60 (patch) | |
tree | f520e15c0027730e4d60cc0205796bcd721b27a1 /model-evaluation | |
parent | 83fec633592486ef694856ece90eb76469a2a5f6 (diff) |
Test with constants
Diffstat (limited to 'model-evaluation')
14 files changed, 31 insertions, 24 deletions
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 48c71b5a04a..b507c5d7f08 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -3,6 +3,7 @@ package ai.vespa.models.evaluation; import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -22,8 +23,13 @@ public class ModelsEvaluator extends AbstractComponent { private final ImmutableMap<String, Model> models; + @Inject public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { - models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + this(new RankProfilesConfigImporter().importFrom(config, constantsConfig)); + } + + public ModelsEvaluator(Map<String, Model> models) { + this.models = ImmutableMap.copyOf(models); } /** Returns the models of this as an immutable map */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index cd21a0a6813..98634e184bf 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -35,7 +35,7 @@ import java.util.logging.Logger; * * @author bratseth */ -class RankProfilesConfigImporter { +public class RankProfilesConfigImporter { private static final Logger log = Logger.getLogger("CONSTANTS"); @@ -43,7 +43,7 @@ class RankProfilesConfigImporter { * Returns a map of the models contained in this config, indexed on name. * The map is modifiable and owned by the caller. */ - Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { + public Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) { try { Map<String, Model> models = new HashMap<>(); for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) { @@ -120,23 +120,9 @@ class RankProfilesConfigImporter { return constants; } - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { try { - // TODO: Only allow these two fallbacks in testing mode - if (fileReference.isEmpty()) { // this may be the case in unit tests - log.warning("Got empty file reference for constant '" + name + "', using an empty tensor"); - return Tensor.from(type, "{}"); - } File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); - if ( ! dir.exists()) { // this may be the case in unit tests - log.warning("Got reference to nonexisting file " + dir + "e for constant '" + name + - "', using an empty tensor"); - return Tensor.from(type, "{}"); - } - - // TODO: Move these 2 lines to FileReference - - dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference); File file = dir.listFiles()[0]; // directory contains one file having the original name if (file.getName().endsWith(".tbf")) diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 63e17e37bde..d2729ab7b6e 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -3,15 +3,20 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.FileSource; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import static org.junit.Assert.assertEquals; @@ -38,7 +43,8 @@ public class ModelTester { RankProfilesConfig.class).getConfig(""); RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), RankingConstantsConfig.class).getConfig(""); - return new RankProfilesConfigImporterWithMockedConstants().importFrom(config, constantsConfig); + return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants")) + .importFrom(config, constantsConfig); } public void assertFunction(String name, String expression, Model model) { @@ -61,15 +67,23 @@ public class ModelTester { private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName()); - Map<String, Tensor> constants = new HashMap<>(); + private final Path constantsPath; + + public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) { + this.constantsPath = constantsPath; + } @Override - Tensor readTensorFromFile(String name, TensorType type, String fileReference) { - if ( ! constants.containsKey(name)) { - log.warning("Missing a mocked tensor constant for '" + name + "': Returning an empty tensor"); + protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) { + try { + return TypedBinaryFormat.decode(Optional.of(type), + GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile()))); + } + catch (IOException e) { + log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() + + ". Returning an empty tensor"); return Tensor.from(type, "{}"); } - return constants.get(name); } } diff --git a/model-evaluation/src/test/resources/config/models/constants/README b/model-evaluation/src/test/resources/config/models/constants/README new file mode 100644 index 00000000000..4a274aa95c8 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/README @@ -0,0 +1 @@ +These constants was created by writing TypedBinaryFormat.encode(tensor) on each large constant produced by these models. diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read Binary files differnew file mode 100644 index 00000000000..bac75f7b1e7 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read Binary files differnew file mode 100644 index 00000000000..bd3f05be826 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read Binary files differnew file mode 100644 index 00000000000..fca7c76df3f --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read Binary files differnew file mode 100644 index 00000000000..396dea8f4bc --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read Binary files differnew file mode 100644 index 00000000000..42f85478c10 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read Binary files differnew file mode 100644 index 00000000000..a3cc7d765f6 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable Binary files differnew file mode 100644 index 00000000000..e768328bff5 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 Binary files differnew file mode 100644 index 00000000000..4fa0eadb0d3 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read Binary files differnew file mode 100644 index 00000000000..5cc9575b971 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read Binary files differnew file mode 100644 index 00000000000..70a6fd42c91 --- /dev/null +++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read |