Test with constants

14 files changed, 31 insertions, 24 deletions

diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java
index 48c71b5a04a..b507c5d7f08 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java
@@ -3,6 +3,7 @@ package ai.vespa.models.evaluation;
 
 import com.google.common.annotations.Beta;
 import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
@@ -22,8 +23,13 @@ public class ModelsEvaluator extends AbstractComponent {
 
     private final ImmutableMap<String, Model> models;
 
+    @Inject
     public ModelsEvaluator(RankProfilesConfig config, RankingConstantsConfig constantsConfig) {
-        models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config, constantsConfig));
+        this(new RankProfilesConfigImporter().importFrom(config, constantsConfig));
+    }
+
+    public ModelsEvaluator(Map<String, Model> models) {
+        this.models = ImmutableMap.copyOf(models);
     }
 
     /** Returns the models of this as an immutable map */
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
index cd21a0a6813..98634e184bf 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -35,7 +35,7 @@ import java.util.logging.Logger;
  *
  * @author bratseth
  */
-class RankProfilesConfigImporter {
+public class RankProfilesConfigImporter {
 
     private static final Logger log = Logger.getLogger("CONSTANTS");
 
@@ -43,7 +43,7 @@ class RankProfilesConfigImporter {
      * Returns a map of the models contained in this config, indexed on name.
      * The map is modifiable and owned by the caller.
      */
-    Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) {
+    public Map<String, Model> importFrom(RankProfilesConfig config, RankingConstantsConfig constantsConfig) {
         try {
             Map<String, Model> models = new HashMap<>();
             for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) {
@@ -120,23 +120,9 @@ class RankProfilesConfigImporter {
         return constants;
     }
 
-    Tensor readTensorFromFile(String name, TensorType type, String fileReference) {
+    protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) {
         try {
-            // TODO: Only allow these two fallbacks in testing mode
-            if (fileReference.isEmpty()) { // this may be the case in unit tests
-                log.warning("Got empty file reference for constant '" + name + "', using an empty tensor");
-                return Tensor.from(type, "{}");
-            }
             File dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference);
-            if ( ! dir.exists()) { // this may be the case in unit tests
-                log.warning("Got reference to nonexisting file " + dir + "e for constant '" + name +
-                            "', using an empty tensor");
-                return Tensor.from(type, "{}");
-            }
-
-            // TODO: Move these 2 lines to FileReference
-
-            dir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), fileReference);
             File file = dir.listFiles()[0]; // directory contains one file having the original name
 
             if (file.getName().endsWith(".tbf"))
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java
index 63e17e37bde..d2729ab7b6e 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java
@@ -3,15 +3,20 @@ package ai.vespa.models.evaluation;
 
 import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.config.subscription.FileSource;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.io.IOUtils;
 import com.yahoo.path.Path;
 import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.logging.Logger;
 
 import static org.junit.Assert.assertEquals;
@@ -38,7 +43,8 @@ public class ModelTester {
                                                        RankProfilesConfig.class).getConfig("");
         RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()),
                                                                     RankingConstantsConfig.class).getConfig("");
-        return new RankProfilesConfigImporterWithMockedConstants().importFrom(config, constantsConfig);
+        return new RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"))
+                       .importFrom(config, constantsConfig);
     }
 
     public void assertFunction(String name, String expression, Model model) {
@@ -61,15 +67,23 @@ public class ModelTester {
 
         private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName());
 
-        Map<String, Tensor> constants = new HashMap<>();
+        private final Path constantsPath;
+
+        public RankProfilesConfigImporterWithMockedConstants(Path constantsPath) {
+            this.constantsPath = constantsPath;
+        }
 
         @Override
-        Tensor readTensorFromFile(String name, TensorType type, String fileReference) {
-            if ( ! constants.containsKey(name)) {
-                log.warning("Missing a mocked tensor constant for '" + name + "': Returning an empty tensor");
+        protected Tensor readTensorFromFile(String name, TensorType type, String fileReference) {
+            try {
+                return TypedBinaryFormat.decode(Optional.of(type),
+                                                GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantsPath.append(name).toFile())));
+            }
+            catch (IOException e) {
+                log.warning("Missing a mocked tensor constant for '" + name + "': " + e.getMessage() +
+                            ". Returning an empty tensor");
                 return Tensor.from(type, "{}");
             }
-            return constants.get(name);
         }
 
     }
diff --git a/model-evaluation/src/test/resources/config/models/constants/README b/model-evaluation/src/test/resources/config/models/constants/README
new file mode 100644
index 00000000000..4a274aa95c8
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/README
@@ -0,0 +1 @@
+These constants was created by writing TypedBinaryFormat.encode(tensor) on each large constant produced by these models.
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read
new file mode 100644
index 00000000000..bac75f7b1e7
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_bias_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read
new file mode 100644
index 00000000000..bd3f05be826
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden1_weights_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read
new file mode 100644
index 00000000000..fca7c76df3f
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_bias_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read
new file mode 100644
index 00000000000..396dea8f4bc
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_hidden2_weights_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read
new file mode 100644
index 00000000000..42f85478c10
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_bias_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read
new file mode 100644
index 00000000000..a3cc7d765f6
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_saved_dnn_outputs_weights_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable
new file mode 100644
index 00000000000..e768328bff5
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1 b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1
new file mode 100644
index 00000000000..4fa0eadb0d3
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_Variable_1
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read
new file mode 100644
index 00000000000..5cc9575b971
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_1_read
diff --git a/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read
new file mode 100644
index 00000000000..70a6fd42c91
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/models/constants/mnist_softmax_saved_layer_Variable_read