Vespa global model import

5 files changed, 120 insertions, 1 deletions

diff --git a/config-model/src/test/integration/vespa/models/constant1asLarge.json b/config-model/src/test/integration/vespa/models/constant1asLarge.json
new file mode 100644
index 00000000000..d2944d255af
--- /dev/null
+++ b/config-model/src/test/integration/vespa/models/constant1asLarge.json
@@ -0,0 +1,7 @@
+{
+  "cells": [
+    { "address": { "x": "0" }, "value": 0.5 },
+    { "address": { "x": "1" }, "value": 1.5 },
+    { "address": { "x": "2" }, "value": 2.5 }
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/integration/vespa/models/example.model b/config-model/src/test/integration/vespa/models/example.model
new file mode 100644
index 00000000000..9579be4e44c
--- /dev/null
+++ b/config-model/src/test/integration/vespa/models/example.model
@@ -0,0 +1,25 @@
+model example {
+
+    # All inputs that are not scalar (aka 0-dimensional tensor) must be declared
+    input1: tensor(name{}, x[3])
+    input2: tensor(x[3])
+
+    constants {
+        constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5}
+        constant2: 3.0
+    }
+
+    constant constant1asLarge {
+        type: tensor(x[3])
+        file: constant1asLarge.json
+    }
+
+    function foo1() {
+        expression: max(sum(input1 * input2, name) * constant1, x) * constant2
+    }
+
+    function foo2() {
+        expression: max(sum(input1 * input2, name) * constant1asLarge, x) * constant2
+    }
+
+}
\ No newline at end of file
diff --git a/config-model/src/test/integration/vespa/services.xml b/config-model/src/test/integration/vespa/services.xml
new file mode 100644
index 00000000000..aa1c0223bdf
--- /dev/null
+++ b/config-model/src/test/integration/vespa/services.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services>
+    <container version="1.0">
+
+    </container>
+</services>
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java
new file mode 100644
index 00000000000..a75699d2a1d
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java
@@ -0,0 +1,77 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.derived.RawRankProfile;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.ml.ImportedModelTester;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests adding Vespa ranking expression based models in the models/ dir
+ *
+ * @author bratseth
+ */
+public class VespaMlModelTestCase {
+
+    private final Path applicationDir = Path.fromString("src/test/integration/vespa/");
+
+    private final String expectedRankConfig =
+            "constant(constant1).type : tensor(x[3])\n" +
+            "constant(constant1).value : tensor(x[3]):{{x:0}:0.5,{x:1}:1.5,{x:2}:2.5}\n" +
+            "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" +
+            "rankingExpression(foo1).input2.type : tensor(x[3])\n" +
+            "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" +
+            "rankingExpression(foo2).rankingScript : max(reduce(input1 * input2, sum, name) * constant1asLarge,x) * 3.0\n" +
+            "rankingExpression(foo2).input2.type : tensor(x[3])\n" +
+            "rankingExpression(foo2).input1.type : tensor(name{},x[3])\n";
+
+    /** The model name */
+    private final String name = "example";
+
+    @After
+    public void removeGeneratedModelFiles() {
+        IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+    }
+
+    @Test
+    public void testGlobalVespaModel() throws IOException {
+        ImportedModelTester tester = new ImportedModelTester(name, applicationDir);
+        VespaModel model = tester.createVespaModel();
+        tester.assertLargeConstant("constant1asLarge", model, Optional.of(3L));
+        assertEquals(expectedRankConfig, rankConfigOf("example", model));
+
+        // At this point the expression is stored - copy application to another location which do not have a models dir
+        Path storedAppDir = applicationDir.append("copy");
+        try {
+            storedAppDir.toFile().mkdirs();
+            IOUtils.copy(applicationDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString());
+            IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
+                                  storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+            ImportedModelTester storedTester = new ImportedModelTester(name, storedAppDir);
+            VespaModel storedModel = storedTester.createVespaModel();
+            storedTester.assertLargeConstant("constant1asLarge", model, Optional.of(3L));
+            assertEquals(expectedRankConfig, rankConfigOf("example", storedModel));
+        }
+        finally {
+            IOUtils.recursiveDeleteDir(storedAppDir.toFile());
+        }
+    }
+
+    private String rankConfigOf(String rankProfileName, VespaModel model) {
+        StringBuilder b = new StringBuilder();
+        RawRankProfile profile = model.rankProfileList().getRankProfile(rankProfileName);
+        for (var property : profile.configProperties())
+            b.append(property.getFirst()).append(" : ").append(property.getSecond()).append("\n");
+        return b.toString();
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
index 563572b4af6..41811738ea4 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.ml;
 
+import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.config.model.ApplicationPackageTester;
 import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
@@ -8,10 +9,12 @@ import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.io.IOUtils;
 import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankingConstant;
 import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
 import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
 import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
+import com.yahoo.searchdefinition.derived.RawRankProfile;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import com.yahoo.vespa.model.VespaModel;
@@ -34,7 +37,8 @@ public class ImportedModelTester {
 
     private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
                                                                               new OnnxImporter(),
-                                                                              new XGBoostImporter());
+                                                                              new XGBoostImporter(),
+                                                                              new VespaImporter());
 
     private final String modelName;
     private final Path applicationDir;