Validate constant file names and bypass cointent validation for compressed files

6 files changed, 62 insertions, 49 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
index 67a180d334d..7bad00c925d 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
@@ -20,6 +20,7 @@ import java.util.logging.Logger;
  * @since 5.1
  */
 public class FilesApplicationFile extends ApplicationFile {
+
     private static final Logger log = Logger.getLogger("FilesApplicationFile");
     private final File file;
     private final ObjectMapper mapper = new ObjectMapper();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
index 66be376271a..05a80c2f5d6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
@@ -3,6 +3,8 @@ package com.yahoo.searchdefinition;
 
 import com.yahoo.tensor.TensorType;
 
+import java.util.Objects;
+
 /**
  * Represents a global ranking constant (declared in a .sd file)
  *
@@ -20,7 +22,11 @@ public class RankingConstant {
         this.name = name;
     }
 
-    public void setFileName(String fileName) { this.fileName = fileName; }
+    public void setFileName(String fileName) { 
+        Objects.requireNonNull(fileName, "Filename cannot be null");
+        this.fileName = fileName; 
+    }
+
     public void setFileReference(String fileRef) { this.fileRef = fileRef; }
     public void setType(TensorType tensorType) { this.tensorType = tensorType; }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
index 2c30e271953..9a148aa65e6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java
@@ -29,45 +29,30 @@ public class ConstantTensorJsonValidator {
     private static final String FIELD_VALUE = "value";
 
     private static final JsonFactory jsonFactory = new JsonFactory();
-    private JsonParser parser;
 
+    private JsonParser parser;
     private Map<String, TensorType.Dimension> tensorDimensions;
 
-    public static class InvalidConstantTensor extends RuntimeException {
-        public InvalidConstantTensor(JsonParser parser, String message) {
-            super(message + " " + parser.getCurrentLocation().toString());
+    public void validate(String fileName, TensorType type, Reader tensorData) {
+        if (fileName.endsWith(".json")) {
+            validateTensor(type, tensorData);
         }
-
-        public InvalidConstantTensor(JsonParser parser, Exception base) {
-            super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base);
+        else if (fileName.endsWith(".json.lz4")) {
+            // don't validate; the cost probably outweights the advantage
         }
-    }
-
-    @FunctionalInterface
-    private interface SubroutineThrowingIOException {
-        void invoke() throws IOException;
-    }
-
-    private void wrapIOException(SubroutineThrowingIOException lambda) {
-        try {
-            lambda.invoke();
-        } catch (IOException e) {
-            throw new InvalidConstantTensor(parser, e);
+        else {
+            throw new IllegalArgumentException("Ranking constant file names must end with either '.json' or '.json.lz4'");
         }
     }
-
-    public ConstantTensorJsonValidator(Reader tensorFile, TensorType tensorType) {
+    
+    private void validateTensor(TensorType type, Reader tensorData) {
         wrapIOException(() -> {
-            this.parser = jsonFactory.createParser(tensorFile);
-            this.tensorDimensions = tensorType
+            this.parser = jsonFactory.createParser(tensorData);
+            this.tensorDimensions = type
                     .dimensions()
                     .stream()
                     .collect(Collectors.toMap(TensorType.Dimension::name, Function.identity()));
-        });
-    }
 
-    public void validate() {
-        wrapIOException(() -> {
             assertNextTokenIs(JsonToken.START_OBJECT);
             assertNextTokenIs(JsonToken.FIELD_NAME);
             assertFieldNameIs(FIELD_CELLS);
@@ -86,10 +71,10 @@ public class ConstantTensorJsonValidator {
         wrapIOException(() -> {
             assertCurrentTokenIs(JsonToken.START_OBJECT);
 
-            final List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE));
+            List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE));
             for (int i = 0; i < 2; i++) {
                 assertNextTokenIs(JsonToken.FIELD_NAME);
-                final String fieldName = parser.getCurrentName();
+                String fieldName = parser.getCurrentName();
 
                 if (fieldNameCandidates.contains(fieldName)) {
                     fieldNameCandidates.remove(fieldName);
@@ -111,13 +96,13 @@ public class ConstantTensorJsonValidator {
     private void validateTensorAddress() throws IOException {
         assertNextTokenIs(JsonToken.START_OBJECT);
 
-        final Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet());
+        Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet());
 
         // Iterate within the address key, value pairs
         while ((parser.nextToken() != JsonToken.END_OBJECT)) {
             assertCurrentTokenIs(JsonToken.FIELD_NAME);
 
-            final String dimensionName = parser.getCurrentName();
+            String dimensionName = parser.getCurrentName();
             TensorType.Dimension dimension = tensorDimensions.get(dimensionName);
             if (dimension == null) {
                 throw new InvalidConstantTensor(parser, String.format("Tensor dimension \"%s\" does not exist", parser.getCurrentName()));
@@ -142,7 +127,7 @@ public class ConstantTensorJsonValidator {
      * additionally, those for indexed bounded dimensions needs to fall within the dimension size.
      */
     private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException {
-        final JsonToken token = parser.nextToken();
+        JsonToken token = parser.nextToken();
         if (token != JsonToken.VALUE_STRING) {
             throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString()));
         }
@@ -157,7 +142,7 @@ public class ConstantTensorJsonValidator {
     private void validateBoundedCoordinate(TensorType.IndexedBoundDimension dimension) {
         wrapIOException(() -> {
             try {
-                final int value = Integer.parseInt(parser.getValueAsString());
+                int value = Integer.parseInt(parser.getValueAsString());
                 if (value >= dimension.size().get()) {
                     throw new InvalidConstantTensor(parser, String.format("Coordinate \"%s\" not within limits of bounded dimension %s", value, dimension.name()));
 
@@ -183,7 +168,7 @@ public class ConstantTensorJsonValidator {
     }
 
     private void validateTensorValue() throws IOException {
-        final JsonToken token = parser.nextToken();
+        JsonToken token = parser.nextToken();
 
         if (token != JsonToken.VALUE_NUMBER_FLOAT && token != JsonToken.VALUE_NUMBER_INT) {
             throw new InvalidConstantTensor(parser, String.format("Tensor value is not a number (%s)", token.toString()));
@@ -205,11 +190,34 @@ public class ConstantTensorJsonValidator {
     }
 
     private void assertFieldNameIs(String wantedFieldName) throws IOException {
-        final String actualFieldName = parser.getCurrentName();
+        String actualFieldName = parser.getCurrentName();
 
         if (!actualFieldName.equals(wantedFieldName)) {
             throw new InvalidConstantTensor(parser, String.format("Expected field name \"%s\", got \"%s\"", wantedFieldName, actualFieldName));
         }
     }
 
+    static class InvalidConstantTensor extends RuntimeException {
+        InvalidConstantTensor(JsonParser parser, String message) {
+            super(message + " " + parser.getCurrentLocation().toString());
+        }
+
+        InvalidConstantTensor(JsonParser parser, Exception base) {
+            super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base);
+        }
+    }
+
+    @FunctionalInterface
+    private interface SubroutineThrowingIOException {
+        void invoke() throws IOException;
+    }
+
+    private void wrapIOException(SubroutineThrowingIOException lambda) {
+        try {
+            lambda.invoke();
+        } catch (IOException e) {
+            throw new InvalidConstantTensor(parser, e);
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
index 6b686526532..7017a5a233c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
@@ -63,10 +63,9 @@ public class RankingConstantsValidator extends Validator {
 
     private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException {
         ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName()));
-        Reader tensorReader = tensorApplicationFile.createReader();
-
-        ConstantTensorJsonValidator tensorValidator = new ConstantTensorJsonValidator(tensorReader, rankingConstant.getTensorType());
-        tensorValidator.validate();
+        new ConstantTensorJsonValidator().validate(rankingConstant.getFileName(), 
+                                                   rankingConstant.getTensorType(),
+                                                   tensorApplicationFile.createReader());
     }
 
 }
\ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index 6d8abfb7781..916e342eb8e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -18,7 +18,7 @@ import java.util.*;
 /**
  * Superclass for search clusters.
  *
- * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a>
+ * @author Peter Boros
  */
 public abstract class AbstractSearchCluster extends AbstractConfigProducer
     implements 
@@ -89,21 +89,18 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
         }
     }
 
-    public AbstractSearchCluster(AbstractConfigProducer parent,
-            String clusterName, int index) {
+    public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
         super(parent, "cluster." + clusterName);
         this.clusterName = clusterName;
         this.index = index;
     }
+
     public void addDocumentNames(SearchDefinition searchDefinition) {
         String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
         documentNames.add(dName);
     }
 
-    /**
-     * Returns a List with document names used in this search cluster
-     * @return  contained document names
-     */
+    /** Returns a List with document names used in this search cluster */
     public List<String> getDocumentNames() { return documentNames; }
 
     public List<SearchDefinitionSpec> getLocalSDS() {
@@ -138,4 +135,5 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     public abstract void getConfig(IlscriptsConfig.Builder builder);
     public abstract void getConfig(RankProfilesConfig.Builder builder);
     public abstract void getConfig(AttributesConfig.Builder builder);
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
index cdf4d88148e..718f4592709 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java
@@ -12,13 +12,13 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson;
 import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor;
 
 public class ConstantTensorJsonValidatorTest {
+
     private static Reader inputJsonToReader(String... lines) {
         return new StringReader(inputJson(lines));
     }
 
     private static void validateTensorJson(TensorType tensorType, Reader jsonTensorReader) {
-        ConstantTensorJsonValidator validator = new ConstantTensorJsonValidator(jsonTensorReader, tensorType);
-        validator.validate();
+        new ConstantTensorJsonValidator().validate("dummy.json", tensorType, jsonTensorReader);
     }
 
     @Rule
@@ -275,4 +275,5 @@ public class ConstantTensorJsonValidatorTest {
                         "   }",
                         "}"));
     }
+
 }
\ No newline at end of file