diff options
7 files changed, 77 insertions, 56 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 67a180d334d..7bad00c925d 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; * @since 5.1 */ public class FilesApplicationFile extends ApplicationFile { + private static final Logger log = Logger.getLogger("FilesApplicationFile"); private final File file; private final ObjectMapper mapper = new ObjectMapper(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 585fb6902ab..09f01a605b4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -11,10 +11,10 @@ import java.util.List; * An application file represents a file within an application package. This class can be used to traverse the entire * application package file structure, as well as read and write files to it, and create directories. * - * @author lulf - * @since 5.1 + * @author Ulf Lillengen */ public abstract class ApplicationFile implements Comparable<ApplicationFile> { + private static final String metaDir = ".meta"; public static final String ContentStatusNew = "new"; public static final String ContentStatusChanged = "changed"; @@ -154,6 +154,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract MetaData getMetaData(); public static class MetaData { + public String status = "unknown"; public String md5 = ""; @@ -173,7 +174,8 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } } - public static interface PathFilter { + public interface PathFilter { boolean accept(Path path); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 5b3ce7136d5..05a80c2f5d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -3,12 +3,15 @@ package com.yahoo.searchdefinition; import com.yahoo.tensor.TensorType; +import java.util.Objects; + /** * Represents a global ranking constant (declared in a .sd file) * * @author arnej */ public class RankingConstant { + /** The search definition-unique name of this constant */ private final String name; private TensorType tensorType = null; @@ -19,7 +22,11 @@ public class RankingConstant { this.name = name; } - public void setFileName(String fileName) { this.fileName = fileName; } + public void setFileName(String fileName) { + Objects.requireNonNull(fileName, "Filename cannot be null"); + this.fileName = fileName; + } + public void setFileReference(String fileRef) { this.fileRef = fileRef; } public void setType(TensorType tensorType) { this.tensorType = tensorType; } @@ -45,4 +52,5 @@ public class RankingConstant { .append("'"); return b.toString(); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 5b88361c15b..9a148aa65e6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -23,50 +23,36 @@ import java.util.stream.Collectors; * @author Vegard Sjonfjell */ public class ConstantTensorJsonValidator { + private static final String FIELD_CELLS = "cells"; private static final String FIELD_ADDRESS = "address"; private static final String FIELD_VALUE = "value"; private static final JsonFactory jsonFactory = new JsonFactory(); - private JsonParser parser; + private JsonParser parser; private Map<String, TensorType.Dimension> tensorDimensions; - public static class InvalidConstantTensor extends RuntimeException { - public InvalidConstantTensor(JsonParser parser, String message) { - super(message + " " + parser.getCurrentLocation().toString()); + public void validate(String fileName, TensorType type, Reader tensorData) { + if (fileName.endsWith(".json")) { + validateTensor(type, tensorData); } - - public InvalidConstantTensor(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + else if (fileName.endsWith(".json.lz4")) { + // don't validate; the cost probably outweights the advantage } - } - - @FunctionalInterface - private static interface SubroutineThrowingIOException { - void invoke() throws IOException; - } - - private void wrapIOException(SubroutineThrowingIOException lambda) { - try { - lambda.invoke(); - } catch (IOException e) { - throw new InvalidConstantTensor(parser, e); + else { + throw new IllegalArgumentException("Ranking constant file names must end with either '.json' or '.json.lz4'"); } } - - public ConstantTensorJsonValidator(Reader tensorFile, TensorType tensorType) { + + private void validateTensor(TensorType type, Reader tensorData) { wrapIOException(() -> { - this.parser = jsonFactory.createParser(tensorFile); - this.tensorDimensions = tensorType + this.parser = jsonFactory.createParser(tensorData); + this.tensorDimensions = type .dimensions() .stream() .collect(Collectors.toMap(TensorType.Dimension::name, Function.identity())); - }); - } - public void validate() { - wrapIOException(() -> { assertNextTokenIs(JsonToken.START_OBJECT); assertNextTokenIs(JsonToken.FIELD_NAME); assertFieldNameIs(FIELD_CELLS); @@ -85,10 +71,10 @@ public class ConstantTensorJsonValidator { wrapIOException(() -> { assertCurrentTokenIs(JsonToken.START_OBJECT); - final List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE)); + List<String> fieldNameCandidates = new ArrayList<>(Arrays.asList(FIELD_ADDRESS, FIELD_VALUE)); for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - final String fieldName = parser.getCurrentName(); + String fieldName = parser.getCurrentName(); if (fieldNameCandidates.contains(fieldName)) { fieldNameCandidates.remove(fieldName); @@ -110,13 +96,13 @@ public class ConstantTensorJsonValidator { private void validateTensorAddress() throws IOException { assertNextTokenIs(JsonToken.START_OBJECT); - final Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet()); + Set<String> cellDimensions = new HashSet<>(tensorDimensions.keySet()); // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - final String dimensionName = parser.getCurrentName(); + String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { throw new InvalidConstantTensor(parser, String.format("Tensor dimension \"%s\" does not exist", parser.getCurrentName())); @@ -141,7 +127,7 @@ public class ConstantTensorJsonValidator { * additionally, those for indexed bounded dimensions needs to fall within the dimension size. */ private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException { - final JsonToken token = parser.nextToken(); + JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_STRING) { throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString())); } @@ -156,7 +142,7 @@ public class ConstantTensorJsonValidator { private void validateBoundedCoordinate(TensorType.IndexedBoundDimension dimension) { wrapIOException(() -> { try { - final int value = Integer.parseInt(parser.getValueAsString()); + int value = Integer.parseInt(parser.getValueAsString()); if (value >= dimension.size().get()) { throw new InvalidConstantTensor(parser, String.format("Coordinate \"%s\" not within limits of bounded dimension %s", value, dimension.name())); @@ -182,7 +168,7 @@ public class ConstantTensorJsonValidator { } private void validateTensorValue() throws IOException { - final JsonToken token = parser.nextToken(); + JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_NUMBER_FLOAT && token != JsonToken.VALUE_NUMBER_INT) { throw new InvalidConstantTensor(parser, String.format("Tensor value is not a number (%s)", token.toString())); @@ -204,10 +190,34 @@ public class ConstantTensorJsonValidator { } private void assertFieldNameIs(String wantedFieldName) throws IOException { - final String actualFieldName = parser.getCurrentName(); + String actualFieldName = parser.getCurrentName(); if (!actualFieldName.equals(wantedFieldName)) { throw new InvalidConstantTensor(parser, String.format("Expected field name \"%s\", got \"%s\"", wantedFieldName, actualFieldName)); } } + + static class InvalidConstantTensor extends RuntimeException { + InvalidConstantTensor(JsonParser parser, String message) { + super(message + " " + parser.getCurrentLocation().toString()); + } + + InvalidConstantTensor(JsonParser parser, Exception base) { + super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + } + } + + @FunctionalInterface + private interface SubroutineThrowingIOException { + void invoke() throws IOException; + } + + private void wrapIOException(SubroutineThrowingIOException lambda) { + try { + lambda.invoke(); + } catch (IOException e) { + throw new InvalidConstantTensor(parser, e); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 7cc37938658..7017a5a233c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -19,6 +19,7 @@ import java.io.Reader; */ public class RankingConstantsValidator extends Validator { + private static class ExceptionMessageCollector { public String combinedMessage; public boolean exceptionsOccurred = false; @@ -42,8 +43,8 @@ public class RankingConstantsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { - final ApplicationPackage applicationPackage = deployState.getApplicationPackage(); - final ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Failed to validate constant tensor file(s):"); + ApplicationPackage applicationPackage = deployState.getApplicationPackage(); + ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Failed to validate constant tensor file(s):"); for (SearchDefinition sd : deployState.getSearchDefinitions()) { for (RankingConstant rc : sd.getSearch().getRankingConstants()) { @@ -60,11 +61,11 @@ public class RankingConstantsValidator extends Validator { } } - public static void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException { - final ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName())); - final Reader tensorReader = tensorApplicationFile.createReader(); - - ConstantTensorJsonValidator tensorValidator = new ConstantTensorJsonValidator(tensorReader, rankingConstant.getTensorType()); - tensorValidator.validate(); + private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException { + ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName())); + new ConstantTensorJsonValidator().validate(rankingConstant.getFileName(), + rankingConstant.getTensorType(), + tensorApplicationFile.createReader()); } + }
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 6d8abfb7781..916e342eb8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -18,7 +18,7 @@ import java.util.*; /** * Superclass for search clusters. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public abstract class AbstractSearchCluster extends AbstractConfigProducer implements @@ -89,21 +89,18 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } } - public AbstractSearchCluster(AbstractConfigProducer parent, - String clusterName, int index) { + public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { super(parent, "cluster." + clusterName); this.clusterName = clusterName; this.index = index; } + public void addDocumentNames(SearchDefinition searchDefinition) { String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); documentNames.add(dName); } - /** - * Returns a List with document names used in this search cluster - * @return contained document names - */ + /** Returns a List with document names used in this search cluster */ public List<String> getDocumentNames() { return documentNames; } public List<SearchDefinitionSpec> getLocalSDS() { @@ -138,4 +135,5 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public abstract void getConfig(IlscriptsConfig.Builder builder); public abstract void getConfig(RankProfilesConfig.Builder builder); public abstract void getConfig(AttributesConfig.Builder builder); + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index cdf4d88148e..718f4592709 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -12,13 +12,13 @@ import static com.yahoo.test.json.JsonTestHelper.inputJson; import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; public class ConstantTensorJsonValidatorTest { + private static Reader inputJsonToReader(String... lines) { return new StringReader(inputJson(lines)); } private static void validateTensorJson(TensorType tensorType, Reader jsonTensorReader) { - ConstantTensorJsonValidator validator = new ConstantTensorJsonValidator(jsonTensorReader, tensorType); - validator.validate(); + new ConstantTensorJsonValidator().validate("dummy.json", tensorType, jsonTensorReader); } @Rule @@ -275,4 +275,5 @@ public class ConstantTensorJsonValidatorTest { " }", "}")); } + }
\ No newline at end of file |