Merge with master

21 files changed, 300 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
index 6a5abee7cb3..ed254bf3e08 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
@@ -2,7 +2,6 @@
 package com.yahoo.config.model.provision;
 
 import com.google.common.collect.ImmutableMap;
-import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.model.builder.xml.XmlHelper;
 import com.yahoo.net.HostName;
 import com.yahoo.text.XML;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
index dbfad0a1370..5b88361c15b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
@@ -141,7 +141,10 @@ public class ConstantTensorJsonValidator {
      * additionally, those for indexed bounded dimensions needs to fall within the dimension size.
      */
     private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException {
-        assertNextTokenIs(JsonToken.VALUE_STRING);
+        final JsonToken token = parser.nextToken();
+        if (token != JsonToken.VALUE_STRING) {
+            throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString()));
+        }
 
         if (dimension instanceof TensorType.IndexedBoundDimension) {
             validateBoundedCoordinate((TensorType.IndexedBoundDimension) dimension);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
new file mode 100644
index 00000000000..7cc37938658
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
@@ -0,0 +1,70 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.RankingConstant;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.io.FileNotFoundException;
+import java.io.Reader;
+
+/**
+ * RankingConstantsValidator validates all constant tensors (ranking constants) bundled with an application package
+ *
+ * @author Vegard Sjonfjell
+ */
+
+public class RankingConstantsValidator extends Validator {
+    private static class ExceptionMessageCollector {
+        public String combinedMessage;
+        public boolean exceptionsOccurred = false;
+
+        public ExceptionMessageCollector(String messagePrelude) {
+            this.combinedMessage = messagePrelude;
+        }
+
+        public ExceptionMessageCollector add(Throwable throwable, String rcName, String rcFilename) {
+            exceptionsOccurred = true;
+            combinedMessage += String.format("\nRanking constant \"%s\" (%s): %s", rcName, rcFilename, throwable.getMessage());
+            return this;
+        }
+    }
+
+    public static class TensorValidationFailed extends RuntimeException {
+        public TensorValidationFailed(String message) {
+            super(message);
+        }
+    }
+
+    @Override
+    public void validate(VespaModel model, DeployState deployState) {
+        final ApplicationPackage applicationPackage = deployState.getApplicationPackage();
+        final ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Failed to validate constant tensor file(s):");
+
+        for (SearchDefinition sd : deployState.getSearchDefinitions()) {
+            for (RankingConstant rc : sd.getSearch().getRankingConstants()) {
+                try {
+                    validateRankingConstant(rc, applicationPackage);
+                } catch (InvalidConstantTensor | FileNotFoundException ex) {
+                    exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName());
+                }
+            }
+        }
+
+        if (exceptionMessageCollector.exceptionsOccurred) {
+            throw new TensorValidationFailed(exceptionMessageCollector.combinedMessage);
+        }
+    }
+
+    public static void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException {
+        final ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName()));
+        final Reader tensorReader = tensorApplicationFile.createReader();
+
+        ConstantTensorJsonValidator tensorValidator = new ConstantTensorJsonValidator(tensorReader, rankingConstant.getTensorType());
+        tensorValidator.validate();
+    }
+}
\ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 819d2638614..68873ebf500 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -51,6 +51,7 @@ public class Validation {
         new RankSetupValidator(force).validate(model, deployState);
         new NoPrefixForIndexes().validate(model, deployState);
         new DeploymentFileValidator().validate(model, deployState);
+        new RankingConstantsValidator().validate(model, deployState);
 
         Optional<Model> currentActiveModel = deployState.getPreviousModel();
         if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel))
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd
new file mode 100644
index 00000000000..4b0c4e297a6
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd
@@ -0,0 +1,29 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search simple {
+  document simple {}
+
+  constant constant_tensor_1 {
+    file: tensors/constant_tensor_1.json
+    type: tensor(x[], y[])
+  }
+
+  constant constant_tensor_2 {
+    file: tensors/constant_tensor_2.json
+    type: tensor(x[])
+  }
+
+  constant constant_tensor_3 {
+    file: tensors/constant_tensor_3.json
+    type: tensor(cpp{}, d{})
+  }
+
+  constant constant_tensor_4 {
+    file: tensors/constant_tensor_4.json
+    type: tensor(x{}, y{})
+  }
+
+  constant constant_tensor_5 {
+    file: tensors/constant_tensor_5.json
+    type: tensor(x[], y[], z[])
+  }
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml
new file mode 100644
index 00000000000..66af18ed7bc
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services>
+    <admin version="2.0">
+        <adminserver hostalias="node1" />
+        <logserver hostalias="node1" />
+    </admin>
+    <content version="1.0">
+        <redundancy>1</redundancy>
+        <documents>
+            <document type='simple' mode="index"/>
+        </documents>
+        <nodes>
+            <node hostalias='node1' distribution-key='0'/>
+        </nodes>
+    </content>
+</services>
\ No newline at end of file
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json
new file mode 100644
index 00000000000..5b4fd45c0a3
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "3", "y": "2" },
+      "value": 2.0
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json
new file mode 100644
index 00000000000..d54daa0bd89
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": 5 },
+      "value": 42.0
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json
new file mode 100644
index 00000000000..e44d0ca0569
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json
@@ -0,0 +1,12 @@
+{
+  "cells": [
+    {
+      "address": { "cpp": "bjarne stroustrup", "d": "andrei alexandrescu" },
+      "value": 99.5
+    },
+    {
+      "address": { "cpp": "stephan lavavej", "cd": "walter bright" },
+      "value": 47.0
+    }
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json
new file mode 100644
index 00000000000..46439be5e3f
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "3", "z": "-99" },
+      "value": 23
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json
new file mode 100644
index 00000000000..01d9204084f
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "32", "y": "9", "z": "45" },
+      "value": 3
+    }
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd
new file mode 100644
index 00000000000..4b0c4e297a6
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd
@@ -0,0 +1,29 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search simple {
+  document simple {}
+
+  constant constant_tensor_1 {
+    file: tensors/constant_tensor_1.json
+    type: tensor(x[], y[])
+  }
+
+  constant constant_tensor_2 {
+    file: tensors/constant_tensor_2.json
+    type: tensor(x[])
+  }
+
+  constant constant_tensor_3 {
+    file: tensors/constant_tensor_3.json
+    type: tensor(cpp{}, d{})
+  }
+
+  constant constant_tensor_4 {
+    file: tensors/constant_tensor_4.json
+    type: tensor(x{}, y{})
+  }
+
+  constant constant_tensor_5 {
+    file: tensors/constant_tensor_5.json
+    type: tensor(x[], y[], z[])
+  }
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml b/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml
new file mode 100644
index 00000000000..be6672f415f
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services>
+    <admin version="2.0">
+        <adminserver hostalias="node1" />
+        <logserver hostalias="node1" />
+    </admin>
+    <content version="1.0">
+        <redundancy>1</redundancy>
+        <documents>
+            <document type='simple' mode="index"/>
+        </documents>
+        <nodes>
+            <node hostalias='node1' distribution-key='0'/>
+        </nodes>
+    </content>
+</services>
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json
new file mode 100644
index 00000000000..5b4fd45c0a3
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "3", "y": "2" },
+      "value": 2.0
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json
new file mode 100644
index 00000000000..af3a02745f3
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "5" },
+      "value": 42.0
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json
new file mode 100644
index 00000000000..d06fcbee42a
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json
@@ -0,0 +1,12 @@
+{
+  "cells": [
+    {
+      "address": { "cpp": "bjarne stroustrup", "d": "andrei alexandrescu" },
+      "value": 99.5
+    },
+    {
+      "address": { "cpp": "stephan lavavej", "d": "walter bright" },
+      "value": 47.0
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json
new file mode 100644
index 00000000000..cb2273f7b11
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "3", "y": "-99" },
+      "value": 23
+    }
+  ]
+}
diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json
new file mode 100644
index 00000000000..01d9204084f
--- /dev/null
+++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json
@@ -0,0 +1,8 @@
+{
+  "cells": [
+    {
+      "address": { "x": "32", "y": "9", "z": "45" },
+      "value": 3
+    }
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
index a8554c54867..3f768088f4a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
@@ -8,7 +8,6 @@ import com.yahoo.cloud.config.SentinelConfig;
 import com.yahoo.config.model.ApplicationConfigProducerRoot;
 import com.yahoo.config.model.deploy.DeployProperties;
 import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.config.model.test.TestDriver;
 import com.yahoo.config.model.test.TestRoot;
 import com.yahoo.config.provision.ApplicationId;
@@ -24,10 +23,7 @@ import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.StatisticsComponent;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
 import org.junit.Test;
-import org.xml.sax.SAXException;
 
-import java.io.IOException;
-import java.util.List;
 import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.is;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
index 912af4f63c1..cdf4d88148e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
@@ -133,7 +133,24 @@ public class ConstantTensorJsonValidatorTest {
                         "        }",
                         "   ]",
                         "}"));
+    }
+
+    @Test
+    public void ensure_that_tensor_coordinates_are_strings() {
+        expectedException.expect(InvalidConstantTensor.class);
+        expectedException.expectMessage("Tensor coordinate is not a string (VALUE_NUMBER_INT)");
 
+        validateTensorJson(
+                TensorType.fromSpec("tensor(x[])"),
+                inputJsonToReader(
+                        "{",
+                        "   'cells': [",
+                        "        {",
+                        "            'address': { 'x': 47 },",
+                        "            'value': 33.0",
+                        "        }",
+                        "   ]",
+                        "}"));
     }
 
     @Test
@@ -152,7 +169,6 @@ public class ConstantTensorJsonValidatorTest {
                         "        }",
                         "   ]",
                         "}"));
-
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java
new file mode 100644
index 00000000000..c2aee71063d
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java
@@ -0,0 +1,28 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationFailed;
+
+public class RankingConstantsValidatorTest {
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    @Test
+    public void ensure_that_valid_ranking_constants_do_not_fail() {
+        new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_ok/").create();
+    }
+
+    @Test
+    public void ensure_that_failing_ranking_constants_fails() {
+        expectedException.expect(TensorValidationFailed.class);
+        expectedException.expectMessage("Ranking constant \"constant_tensor_2\" (tensors/constant_tensor_2.json): Tensor coordinate is not a string (VALUE_NUMBER_INT)");
+        expectedException.expectMessage("Ranking constant \"constant_tensor_3\" (tensors/constant_tensor_3.json): Tensor dimension \"cd\" does not exist");
+        expectedException.expectMessage("Ranking constant \"constant_tensor_4\" (tensors/constant_tensor_4.json): Tensor dimension \"z\" does not exist");
+
+        new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create();
+    }
+}
\ No newline at end of file