diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-01-17 13:28:51 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-01-17 13:28:51 +0100 |
commit | 890fc32d464ee27fadf60ca5e91c1f3f12a4f3c7 (patch) | |
tree | 4790942b7953e0e52fddd8521abeac955c2bc366 /config-model | |
parent | 6e3c957056614b3aae28043f96fff4d564075208 (diff) |
Validate constant file names and bypass cointent validation for compressed files
Diffstat (limited to 'config-model')
5 files changed, 61 insertions, 49 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 66be376271a..05a80c2f5d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -3,6 +3,8 @@ package com.yahoo.searchdefinition; import com.yahoo.tensor.TensorType; +import java.util.Objects; + /** * Represents a global ranking constant (declared in a .sd file) * @@ -20,7 +22,11 @@ public class RankingConstant { this.name = name; } - public void setFileName(String fileName) { this.fileName = fileName; } + public void setFileName(String fileName) { + Objects.requireNonNull(fileName, "Filename cannot be null"); + this.fileName = fileName; + } + public void setFileReference(String fileRef) { this.fileRef = fileRef; } public void setType(TensorType tensorType) { this.tensorType = tensorType; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 2c30e271953..9a148aa65e6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -29,45 +29,30 @@ public class ConstantTensorJsonValidator { private static final String FIELD_VALUE = "value"; private static final JsonFactory jsonFactory = new JsonFactory(); - private JsonParser parser; + private JsonParser parser; private Map<String, TensorType.Dimension> tensorDimensions; - public static class InvalidConstantTensor extends RuntimeException { - public InvalidConstantTensor(JsonParser parser, String message) { - super(message + " " + parser.getCurrentLocation().toString()); + public void validate(String fileName, TensorType type, Reader tensorData) { + if (fileName.endsWith(".json")) { + validateTensor(type, tensorData); } - - public InvalidConstantTensor(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + else if (fileName.endsWith(".json.lz4")) { + // don't validate; the cost probably outweights the advantage } - } - - @FunctionalInterface - private interface SubroutineThrowingIOException { - void invoke() throws IOException; - } - - private void wrapIOException(SubroutineThrowingIOException lambda) { - try { - lambda.invoke(); - } catch (IOException e) { - throw new InvalidConstantTensor(parser, e); + else { + throw new IllegalArgumentException("Ranking constant file names must end with either '.json' or '.json.lz4'"); } } - - public ConstantTensorJsonValidator(Reader tensorFile, TensorType tensorType) { + + private void validateTensor(TensorType type, Reader tensorData) { wrapIOException(() -> { - this.parser = jsonFactory.createParser(tensorFile); - this.tensorDimensions = tensorType + this.parser = jsonFactory.createParser(tensorData); + this.tensorDimensions = type .dimensions() .stream() .collect(Collectors.toMap(TensorType.Dimension::name, Function.identity())); - }); - } - public void validate() { - wrapIOException(() -> { assertNextTokenIs(JsonToken.START_OBJECT); assertNextTokenIs(JsonToken.FIELD_NAME); assertFieldNameIs(FIELD_CELLS); @@ -86,10 +71,10 @@ public class ConstantTensorJsonValidator { wrapIOException(() -> { assertCurrentTokenIs(JsonToken.START_OBJECT); - final List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE)); + List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE)); for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - final String fieldName = parser.getCurrentName(); + String fieldName = parser.getCurrentName(); if (fieldNameCandidates.contains(fieldName)) { fieldNameCandidates.remove(fieldName); @@ -111,13 +96,13 @@ public class ConstantTensorJsonValidator { private void validateTensorAddress() throws IOException { assertNextTokenIs(JsonToken.START_OBJECT); - final Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet()); + Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet()); // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - final String dimensionName = parser.getCurrentName(); + String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { throw new InvalidConstantTensor(parser, String.format("Tensor dimension \"%s\" does not exist", parser.getCurrentName())); @@ -142,7 +127,7 @@ public class ConstantTensorJsonValidator { * additionally, those for indexed bounded dimensions needs to fall within the dimension size. */ private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException { - final JsonToken token = parser.nextToken(); + JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_STRING) { throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString())); } @@ -157,7 +142,7 @@ public class ConstantTensorJsonValidator { private void validateBoundedCoordinate(TensorType.IndexedBoundDimension dimension) { wrapIOException(() -> { try { - final int value = Integer.parseInt(parser.getValueAsString()); + int value = Integer.parseInt(parser.getValueAsString()); if (value >= dimension.size().get()) { throw new InvalidConstantTensor(parser, String.format("Coordinate \"%s\" not within limits of bounded dimension %s", value, dimension.name())); @@ -183,7 +168,7 @@ public class ConstantTensorJsonValidator { } private void validateTensorValue() throws IOException { - final JsonToken token = parser.nextToken(); + JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_NUMBER_FLOAT && token != JsonToken.VALUE_NUMBER_INT) { throw new InvalidConstantTensor(parser, String.format("Tensor value is not a number (%s)", token.toString())); @@ -205,11 +190,34 @@ public class ConstantTensorJsonValidator { } private void assertFieldNameIs(String wantedFieldName) throws IOException { - final String actualFieldName = parser.getCurrentName(); + String actualFieldName = parser.getCurrentName(); if (!actualFieldName.equals(wantedFieldName)) { throw new InvalidConstantTensor(parser, String.format("Expected field name \"%s\", got \"%s\"", wantedFieldName, actualFieldName)); } } + static class InvalidConstantTensor extends RuntimeException { + InvalidConstantTensor(JsonParser parser, String message) { + super(message + " " + parser.getCurrentLocation().toString()); + } + + InvalidConstantTensor(JsonParser parser, Exception base) { + super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + } + } + + @FunctionalInterface + private interface SubroutineThrowingIOException { + void invoke() throws IOException; + } + + private void wrapIOException(SubroutineThrowingIOException lambda) { + try { + lambda.invoke(); + } catch (IOException e) { + throw new InvalidConstantTensor(parser, e); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 6b686526532..7017a5a233c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -63,10 +63,9 @@ public class RankingConstantsValidator extends Validator { private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException { ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName())); - Reader tensorReader = tensorApplicationFile.createReader(); - - ConstantTensorJsonValidator tensorValidator = new ConstantTensorJsonValidator(tensorReader, rankingConstant.getTensorType()); - tensorValidator.validate(); + new ConstantTensorJsonValidator().validate(rankingConstant.getFileName(), + rankingConstant.getTensorType(), + tensorApplicationFile.createReader()); } }
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 6d8abfb7781..916e342eb8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -18,7 +18,7 @@ import java.util.*; /** * Superclass for search clusters. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public abstract class AbstractSearchCluster extends AbstractConfigProducer implements @@ -89,21 +89,18 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } } - public AbstractSearchCluster(AbstractConfigProducer parent, - String clusterName, int index) { + public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { super(parent, "cluster." + clusterName); this.clusterName = clusterName; this.index = index; } + public void addDocumentNames(SearchDefinition searchDefinition) { String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); documentNames.add(dName); } - /** - * Returns a List with document names used in this search cluster - * @return contained document names - */ + /** Returns a List with document names used in this search cluster */ public List<String> getDocumentNames() { return documentNames; } public List<SearchDefinitionSpec> getLocalSDS() { @@ -138,4 +135,5 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public abstract void getConfig(IlscriptsConfig.Builder builder); public abstract void getConfig(RankProfilesConfig.Builder builder); public abstract void getConfig(AttributesConfig.Builder builder); + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index cdf4d88148e..718f4592709 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -12,13 +12,13 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson; import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; public class ConstantTensorJsonValidatorTest { + private static Reader inputJsonToReader(String... lines) { return new StringReader(inputJson(lines)); } private static void validateTensorJson(TensorType tensorType, Reader jsonTensorReader) { - ConstantTensorJsonValidator validator = new ConstantTensorJsonValidator(jsonTensorReader, tensorType); - validator.validate(); + new ConstantTensorJsonValidator().validate("dummy.json", tensorType, jsonTensorReader); } @Rule @@ -275,4 +275,5 @@ public class ConstantTensorJsonValidatorTest { " }", "}")); } + }
\ No newline at end of file |