diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-02-26 16:41:25 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-02-26 16:41:25 +0100 |
commit | 174745d431c59bfd7d8077b817dc38090010fd35 (patch) | |
tree | 135c22a0f22cbdbdcf0b48bf8b9ea2c4c80d7eca /config-model/src/main/java/com/yahoo/searchdefinition | |
parent | 446dc287bf35768b73e1390df07a9462ae9db800 (diff) |
Don't validate when reloading models
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition')
63 files changed, 483 insertions, 397 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java index 1019b794cdd..fd92b724a60 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.document.SDDocumentType; * @author Einar M R Rosenvinge */ public class FieldOperationApplierForSearch extends FieldOperationApplier { + @Override public void process(SDDocumentType sdoc) { //Do nothing @@ -18,4 +19,5 @@ public class FieldOperationApplierForSearch extends FieldOperationApplier { apply(field); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index e7cd21ac834..8ca82008e8b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -196,7 +196,7 @@ public class SearchBuilder { * @throws IllegalStateException Thrown if this method has already been called. */ public void build() { - build(new BaseDeployLogger()); + build(true, new BaseDeployLogger()); } /** @@ -206,7 +206,7 @@ public class SearchBuilder { * @throws IllegalStateException Thrown if this method has already been called. * @param deployLogger The logger to use during build */ - public void build(DeployLogger deployLogger) { + public void build(boolean validate, DeployLogger deployLogger) { if (isBuilt) throw new IllegalStateException("Model already built"); List<Search> built = new ArrayList<>(); @@ -227,20 +227,20 @@ public class SearchBuilder { DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); - DocumentGraphValidator validator = new DocumentGraphValidator(); - validator.validateDocumentGraph(sdocs); + if (validate) + new DocumentGraphValidator().validateDocumentGraph(sdocs); DocumentModelBuilder builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // These two needed for a couple of old unit tests, ideally these are just read from app - process(search, deployLogger, new QueryProfiles(queryProfileRegistry)); + process(search, deployLogger, new QueryProfiles(queryProfileRegistry), validate); built.add(search); } builder.addToModel(searchList); - if ( ! builder.valid() ) { + + if ( validate && ! builder.valid() ) throw new IllegalArgumentException("Impossible to build a correct model."); - } searchList = built; isBuilt = true; } @@ -249,8 +249,8 @@ public class SearchBuilder { * Processes and returns the given {@link Search} object. This method has been factored out of the {@link * #build()} method so that subclasses can choose not to build anything. */ - protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { - Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles); + protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { + Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles, validate); } /** @@ -344,7 +344,7 @@ public class SearchBuilder { rankProfileRegistry, queryprofileRegistry); builder.importFile(fileName); - builder.build(deployLogger); + builder.build(true, deployLogger); return builder; } @@ -360,7 +360,7 @@ public class SearchBuilder { for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(new BaseDeployLogger()); + builder.build(true, new BaseDeployLogger()); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java index 1201de86d8d..6c12c6c94d1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java @@ -30,7 +30,7 @@ public class UnprocessingSearchBuilder extends SearchBuilder { } @Override - public void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { + public void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { // empty } @@ -40,4 +40,5 @@ public class UnprocessingSearchBuilder extends SearchBuilder { builder.build(); return builder.getSearch(); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index 518920d465e..cf123d0f7c1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { } @Override - public void process() { + public void process(boolean validate) { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index b22c813f062..e0d32ea8ccd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,19 +26,19 @@ public class AddExtraFieldsToDocument extends Processor { } @Override - public void process() { + public void process(boolean validate) { SDDocumentType document = search.getDocument(); if (document != null) { for (Field field : search.extraFieldList()) { - addSdField(search, document, (SDField)field); + addSdField(search, document, (SDField)field, validate); } for (SummaryField field : search.getSummary("default").getSummaryFields()) { - addSummaryField(search, document, field); + addSummaryField(search, document, field, validate); } } } - private void addSdField(Search search, SDDocumentType document, SDField field) { + private void addSdField(Search search, SDDocumentType document, SDField field, boolean validate) { if (field.getIndexToCount() == 0 && field.getAttributes().isEmpty()) { return; } @@ -48,15 +48,15 @@ public class AddExtraFieldsToDocument extends Processor { if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) { type = DataType.getArray(type); } - addField(search, document, new SDField(document, atr.getName(), type)); + addField(search, document, new SDField(document, atr.getName(), type), validate); } else if (!atr.getName().equals(field.getName())) { - addField(search, document, new SDField(document, atr.getName(), atr.getDataType())); + addField(search, document, new SDField(document, atr.getName(), atr.getDataType()), validate); } } - addField(search, document, field); + addField(search, document, field, validate); } - private void addSummaryField(Search search, SDDocumentType document, SummaryField field) { + private void addSummaryField(Search search, SDDocumentType document, SummaryField field, boolean validate) { Field docField = document.getField(field.getName()); if (docField == null) { ImmutableSDField existingField = search.getField(field.getName()); @@ -68,13 +68,15 @@ public class AddExtraFieldsToDocument extends Processor { document.addField(existingField.asField()); } } else if (!docField.getDataType().equals(field.getDataType())) { - throw newProcessException(search, field, "Summary field has conflicting type."); + if (validate) + throw newProcessException(search, field, "Summary field has conflicting type."); } } - private void addField(Search search, SDDocumentType document, Field field) { + private void addField(Search search, SDDocumentType document, Field field, boolean validate) { if (document.getField(field.getName()) != null) { - throw newProcessException(search, field, "Field shadows another."); + if (validate) + throw newProcessException(search, field, "Field shadows another."); } document.addField(field); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index e7d0d5c3dbb..9ec596792fa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -20,7 +20,7 @@ public class AttributeProperties extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { String fieldName = field.getName(); @@ -41,7 +41,7 @@ public class AttributeProperties extends Processor { } } } - if (!created) { + if (validate && !created) { throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" + field.getName() + "' is not created by the indexing statement"); } @@ -57,7 +57,7 @@ public class AttributeProperties extends Processor { * @return true if the attribute has been created by this field, else false */ static boolean attributeCreated(SDField field, String attributeName) { - if (!field.doesAttributing()) { + if ( ! field.doesAttributing()) { return false; } for (Attribute attribute : field.getAttributes().values()) { @@ -67,4 +67,5 @@ public class AttributeProperties extends Processor { } return false; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index ecef4b1d7eb..34903abb288 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -14,8 +14,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * Fields that derive to attribute(s) and no indices should use the WORD indexing form, * in a feeble attempt to match the most peoples expectations as closely as possible. * - * @author vegardh - * + * @author Vegard Havdal */ public class AttributesImplicitWord extends Processor { @@ -24,7 +23,7 @@ public class AttributesImplicitWord extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index b320d2e9084..a3c4c97cf31 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -22,7 +22,8 @@ public class Bolding extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; for (SDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields()) { if (summary.getTransform().isBolded() && diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index 498d5234183..37d60c1d32e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -10,8 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document - * @author vegardh - * + * @author Vegard Havdal */ public class BuiltInFieldSets extends Processor { @@ -24,7 +23,7 @@ public class BuiltInFieldSets extends Processor { } @Override - public void process() { + public void process(boolean validate) { addDocumentFieldSet(); addSearchFieldSet(); // "Hook" the field sets on search onto the document types, since we will include them diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index bd9255267de..d7b688be203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -38,12 +38,12 @@ public class CreatePositionZCurve extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { DataType fieldType = field.getDataType(); if ( ! isSupportedPositionType(fieldType)) continue; - if (field.doesIndexing()) { + if (validate && field.doesIndexing()) { fail(search, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " + "replace 'index' statement with 'attribute'."); } @@ -56,7 +56,7 @@ public class CreatePositionZCurve extends Processor { field.getAttributes().remove(fieldName); String zName = PositionDataType.getZCurveFieldName(fieldName); - SDField zCurveField = createZCurveField(field, zName); + SDField zCurveField = createZCurveField(field, zName, validate); search.addExtraField(zCurveField); search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName()); @@ -65,11 +65,11 @@ public class CreatePositionZCurve extends Processor { ensureCompatibleSummary(field, zName, PositionDataType.getPositionSummaryFieldName(fieldName), DataType.getArray(DataType.STRING), // will become "xmlstring" - SummaryTransform.POSITIONS, summaryTo); + SummaryTransform.POSITIONS, summaryTo, validate); ensureCompatibleSummary(field, zName, PositionDataType.getDistanceSummaryFieldName(fieldName), DataType.INT, - SummaryTransform.DISTANCE, summaryTo); + SummaryTransform.DISTANCE, summaryTo, validate); // clear indexing script field.setIndexingScript(null); SDField posX = field.getStructField(PositionDataType.FIELD_X); @@ -83,12 +83,12 @@ public class CreatePositionZCurve extends Processor { if (doesSummary) ensureCompatibleSummary(field, zName, field.getName(), field.getDataType(), - SummaryTransform.GEOPOS, summaryTo); + SummaryTransform.GEOPOS, summaryTo, validate); } } - private SDField createZCurveField(SDField inputField, String fieldName) { - if (search.getConcreteField(fieldName) != null || search.getAttribute(fieldName) != null) { + private SDField createZCurveField(SDField inputField, String fieldName, boolean validate) { + if (validate && search.getConcreteField(fieldName) != null || search.getAttribute(fieldName) != null) { throw newProcessException(search, null, "Incompatible position attribute '" + fieldName + "' already created."); } @@ -108,7 +108,7 @@ public class CreatePositionZCurve extends Processor { } private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType, - SummaryTransform summaryTransform, Collection<String> summaryTo) { + SummaryTransform summaryTransform, Collection<String> summaryTo, boolean validate) { SummaryField summary = search.getSummaryField(summaryName); if (summary == null) { summary = new SummaryField(summaryName, summaryType, summaryTransform); @@ -116,7 +116,8 @@ public class CreatePositionZCurve extends Processor { summary.addDestinations(summaryTo); field.addSummaryField(summary); } else if (!summary.getDataType().equals(summaryType)) { - fail(search, field, "Incompatible summary field '" + summaryName + "' type "+summary.getDataType()+" already created."); + if (validate) + fail(search, field, "Incompatible summary field '" + summaryName + "' type "+summary.getDataType()+" already created."); } else if (summary.getTransform() == SummaryTransform.NONE) { summary.setTransform(summaryTransform); summary.addDestination("default"); @@ -198,4 +199,5 @@ public class CreatePositionZCurve extends Processor { new ZCurveExpression(), new AttributeExpression(replace)); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java index cd44eb9fc87..7cc9b4e9b52 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -15,7 +15,9 @@ public class DeprecateAttributePrefetch extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { for (Attribute a : field.getAttributes().values()) { if (Boolean.TRUE.equals(a.getPrefetchValue())) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 42fe58d2842..861ebad7085 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -13,8 +13,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Non-primitive key types for map and weighted set forbidden (though OK in document model) - * @author vegardh * + * @author Vegard Havdal */ public class DisallowComplexMapAndWsetKeyTypes extends Processor { @@ -23,17 +23,19 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do // not start using the static stuff on SDDocumentTypes any more. for (SDField field : search.allConcreteFields()) { if (field.getDataType() instanceof WeightedSetDataType) { DataType nestedType = ((WeightedSetDataType)field.getDataType()).getNestedType(); - if (!(nestedType instanceof PrimitiveDataType)) { + if ( ! (nestedType instanceof PrimitiveDataType)) { fail(search, field, "Weighted set must have a primitive key type."); } } else if (field.getDataType() instanceof MapDataType) { - if (!(((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) { + if ( ! (((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) { fail(search, field, "Map key type must be a primitive type"); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 608e36c187a..a936045af3a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -12,12 +12,15 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * @author baldersheim */ public class DiversitySettingsValidator extends Processor { + public DiversitySettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) { if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); @@ -27,20 +30,21 @@ public class DiversitySettingsValidator extends Processor { private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { String attributeName = settings.getAttribute(); new AttributeValidator(search.getName(), rankProfile.getName(), - search.getAttribute(attributeName), attributeName).validate(); + search.getAttribute(attributeName), attributeName).validate(); } private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { + public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { super(searchName, rankProfileName, attribute, attributeName); } protected void validateThatAttributeIsSingleAndNotPredicate() { - if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || + if ( ! attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || attribute.getType().equals(Attribute.Type.PREDICATE)) { failValidation("must be single value numeric, or enumerated attribute, but it is '" - + attribute.getDataType().getName() + "'"); + + attribute.getDataType().getName() + "'"); } } @@ -54,5 +58,7 @@ public class DiversitySettingsValidator extends Processor { public String getValidationType() { return "diversity"; } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 2bfc7a418de..59bc3dc66f4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -26,7 +26,7 @@ public class ExactMatch extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { @@ -56,8 +56,7 @@ public class ExactMatch extends Processor { } field.addQueryCommand("exact " + exactTerminator); - // The following part illustrates how nice it would have been with canonical - // representation of indices + // The following part illustrates how nice it would have been with canonical representation of indices if (field.doesIndexing()) { exactMatchSettingsForField(field); } @@ -92,5 +91,6 @@ public class ExactMatch extends Processor { return exp; } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java index 5bfdd3b28b4..9cfac625da5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java @@ -15,12 +15,15 @@ import java.util.stream.Collectors; * @author bjorncs */ public class FastAccessValidator extends Processor { + public FastAccessValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + String invalidAttributes = search.allFields() .flatMap(field -> field.getAttributes().values().stream()) .filter(FastAccessValidator::isIncompatibleAttribute) @@ -50,4 +53,5 @@ public class FastAccessValidator extends Processor { return false; } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java index 8a38f5fc57c..15c11589245 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -24,7 +24,9 @@ public class FieldSetValidity extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { checkFieldNames(search, fieldSet); checkMatching(search, fieldSet); @@ -51,7 +53,7 @@ public class FieldSetValidity extends Processor { if (fsMatching==null) { fsMatching = fieldMatching; } else { - if (fsMatching!=null && !fsMatching.equals(fieldMatching)) { + if ( ! fsMatching.equals(fieldMatching)) { warn(search, field.asField(), "The matching settings for the fields in " + fieldSet + " are inconsistent " + "(explicitly or because of field type). This may lead to recall and ranking issues."); @@ -69,7 +71,7 @@ public class FieldSetValidity extends Processor { if (fsNorm==null) { fsNorm = fieldNorm; } else { - if (fsNorm!=null && !fsNorm.equals(fieldNorm)) { + if ( ! fsNorm.equals(fieldNorm)) { warn(search, field.asField(), "The normalization settings for the fields in " + fieldSet + " are inconsistent " + "(explicitly or because of field type). This may lead to recall and ranking issues."); @@ -87,7 +89,7 @@ public class FieldSetValidity extends Processor { if (fsStemming==null) { fsStemming = fieldStemming; } else { - if (fsStemming!=null && !fsStemming.equals(fieldStemming)) { + if ( ! fsStemming.equals(fieldStemming)) { warn(search, field.asField(), "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+ "' are inconsistent (explicitly or because of field type). " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index 1f1a311b0e0..39d35cce694 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -16,8 +16,8 @@ import java.util.logging.Level; /** * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles - * @author vegardh * + * @author Vegard Havdal */ public class FilterFieldNames extends Processor { @@ -26,12 +26,13 @@ public class FilterFieldNames extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField f : search.allConcreteFields()) { if (f.getRanking().isFilter()) { filterField(f.getName()); } } + for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) { Set<String> filterFields = new LinkedHashSet<>(); findFilterFields(search, profile, filterFields); @@ -52,12 +53,11 @@ public class FilterFieldNames extends Processor { private void findFilterFields(Search search, RankProfile profile, Set<String> filterFields) { for (Iterator<RankProfile.RankSetting> itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) { RankProfile.RankSetting setting = itr.next(); - if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) && - ((Boolean)setting.getValue()).booleanValue()) + if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) && ((Boolean)setting.getValue())) { String fieldName = setting.getFieldName(); if (search.getConcreteField(fieldName) != null) { - if (!profile.filterFields().contains(fieldName)) { + if ( ! profile.filterFields().contains(fieldName)) { filterFields.add(fieldName); } } else { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 2b35b1c42f3..9f4a63b8a9a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -26,15 +26,15 @@ public class ImplicitSummaries extends Processor { } @Override - public void process() { - DocumentSummary defaultSummary=search.getSummary("default"); - if (defaultSummary==null) { - defaultSummary=new DocumentSummary("default"); + public void process(boolean validate) { + DocumentSummary defaultSummary = search.getSummary("default"); + if (defaultSummary == null) { + defaultSummary = new DocumentSummary("default"); search.addSummary(defaultSummary); } for (SDField field : search.allConcreteFields()) { - collectSummaries(field,search); + collectSummaries(field, search, validate); } for (DocumentSummary documentSummary : search.getSummaries().values()) { @@ -46,14 +46,14 @@ public class ImplicitSummaries extends Processor { sdField.addSummaryFieldSources(summaryField); } - private void collectSummaries(SDField field,Search search) { + private void collectSummaries(SDField field ,Search search, boolean validate) { SummaryField addedSummaryField=null; // Implicit String fieldName = field.getName(); - SummaryField fieldSummaryField=field.getSummaryField(fieldName); + SummaryField fieldSummaryField = field.getSummaryField(fieldName); if (fieldSummaryField == null && field.doesSummarying()) { - fieldSummaryField=new SummaryField(fieldName, field.getDataType()); + fieldSummaryField = new SummaryField(fieldName, field.getDataType()); fieldSummaryField.setImplicit(true); addSummaryFieldSources(fieldSummaryField, field); fieldSummaryField.addDestination("default"); @@ -84,9 +84,7 @@ public class ImplicitSummaries extends Processor { // Position attributes if (field.doesSummarying()) { for (Attribute attribute : field.getAttributes().values()) { - if (!attribute.isPosition()) { - continue; - } + if ( ! attribute.isPosition()) continue; DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(search); attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); @@ -96,12 +94,12 @@ public class ImplicitSummaries extends Processor { // Explicits for (SummaryField summaryField : field.getSummaryFields()) { // Make sure we fetch from attribute here too - Attribute attribute=field.getAttributes().get(fieldName); - if (attribute!=null && summaryField.getTransform()==SummaryTransform.NONE) { + Attribute attribute = field.getAttributes().get(fieldName); + if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } - if (isValid(summaryField,search)) { + if (isValid(summaryField, search, validate)) { addToDestinations(summaryField, search); } } @@ -119,14 +117,14 @@ public class ImplicitSummaries extends Processor { private void addPrefetchAttribute(Attribute attribute,SDField field,Search search) { - if (attribute.getPrefetchValue()==null) { // Prefetch by default - unless any summary makes this dynamic + if (attribute.getPrefetchValue() == null) { // Prefetch by default - unless any summary makes this dynamic // Check if there is an implicit dynamic definition - SummaryField fieldSummaryField=field.getSummaryField(attribute.getName()); - if (fieldSummaryField!=null && fieldSummaryField.getTransform().isDynamic()) return; + SummaryField fieldSummaryField = field.getSummaryField(attribute.getName()); + if (fieldSummaryField != null && fieldSummaryField.getTransform().isDynamic()) return; // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later) - SummaryField explicitSummaryField=search.getExplicitSummaryField(attribute.getName()); - if (explicitSummaryField!=null && explicitSummaryField.getTransform().isDynamic()) return; + SummaryField explicitSummaryField = search.getExplicitSummaryField(attribute.getName()); + if (explicitSummaryField != null && explicitSummaryField.getTransform().isDynamic()) return; } DocumentSummary summary = getOrCreateAttributePrefetchSummary(search); @@ -138,17 +136,17 @@ public class ImplicitSummaries extends Processor { } // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable. - private boolean isValid(SummaryField summaryField,Search search) { + private boolean isValid(SummaryField summaryField, Search search, boolean validate) { if (summaryField.getTransform() == SummaryTransform.DISTANCE || summaryField.getTransform() == SummaryTransform.POSITIONS) { int sourceCount = summaryField.getSourceCount(); - if (sourceCount != 1) { + if (validate && sourceCount != 1) { throw newProcessException(search.getName(), summaryField.getName(), "Expected 1 source field, got " + sourceCount + "."); } String sourceName = summaryField.getSingleSource(); - if (search.getAttribute(sourceName) == null) { + if (validate && search.getAttribute(sourceName) == null) { throw newProcessException(search.getName(), summaryField.getName(), "Summary source attribute '" + sourceName + "' not found."); } @@ -157,33 +155,34 @@ public class ImplicitSummaries extends Processor { String fieldName = summaryField.getSourceField(); SDField sourceField = search.getConcreteField(fieldName); - if (sourceField == null) { + if (validate && sourceField == null) { throw newProcessException(search, summaryField, "Source field '" + fieldName + "' does not exist."); } - if (!sourceField.doesSummarying() && - !summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) && - !summaryField.getTransform().equals(SummaryTransform.GEOPOS)) + if (! sourceField.doesSummarying() && + ! summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) && + ! summaryField.getTransform().equals(SummaryTransform.GEOPOS)) { // Summary transform attribute may indicate that the ilscript was rewritten to remove summary // by another search that uses this same field in inheritance. deployLogger.log(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + - " is not creating a summary value in its indexing statement"); + " is not creating a summary value in its indexing statement"); return false; } if (summaryField.getTransform().isDynamic() && summaryField.getName().equals(sourceField.getName()) && sourceField.doesAttributing()) { - Attribute attribute=sourceField.getAttributes().get(sourceField.getName()); - if (attribute!=null) { - String destinations="document summary 'default'"; - if (summaryField.getDestinations().size()>0) { + Attribute attribute = sourceField.getAttributes().get(sourceField.getName()); + if (attribute != null) { + String destinations = "document summary 'default'"; + if (summaryField.getDestinations().size() >0) { destinations = "document summaries " + summaryField.getDestinations(); } - deployLogger.log(Level.WARNING, "Will fetch the disk summary value of " + sourceField + " in " + destinations + - " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " + - "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " + - "to this attribute, remove any bolding and dynamic snippeting from this field"); + deployLogger.log(Level.WARNING, + "Will fetch the disk summary value of " + sourceField + " in " + destinations + + " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " + + "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " + + "to this attribute, remove any bolding and dynamic snippeting from this field"); // Note: The dynamic setting has already overridden the attribute map setting, // so we do not need to actually do attribute.setSummary(false) here // Also, we can not do this, since it makes it impossible to fetch this attribute @@ -195,26 +194,25 @@ public class ImplicitSummaries extends Processor { } private void addToDestinations(SummaryField summaryField,Search search) { - if (summaryField.getDestinations().size()==0) { - addToDestination("default",summaryField,search); + if (summaryField.getDestinations().size() == 0) { + addToDestination("default", summaryField, search); } else { for (String destinationName : summaryField.getDestinations()) - addToDestination(destinationName,summaryField,search); + addToDestination(destinationName, summaryField, search); } } - private void addToDestination(String destinationName,SummaryField summaryField,Search search) { - DocumentSummary destination=search.getSummary(destinationName); - if (destination==null) { - destination=new DocumentSummary(destinationName); + private void addToDestination(String destinationName, SummaryField summaryField,Search search) { + DocumentSummary destination = search.getSummary(destinationName); + if (destination == null) { + destination = new DocumentSummary(destinationName); search.addSummary(destination); destination.add(summaryField); } else { - SummaryField existingField= - destination.getSummaryField(summaryField.getName()); - SummaryField merged=summaryField.mergeWith(existingField); + SummaryField existingField= destination.getSummaryField(summaryField.getName()); + SummaryField merged = summaryField.mergeWith(existingField); destination.add(merged); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 41b0e083230..7464f574255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -21,15 +21,15 @@ public class ImplicitSummaryFields extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (DocumentSummary docsum : search.getSummaries().values()) { - addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES)); - addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES)); + addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); + addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); } } - private void addField(DocumentSummary docsum, SummaryField field) { - if (docsum.getSummaryField(field.getName()) != null) { + private void addField(DocumentSummary docsum, SummaryField field, boolean validate) { + if (validate && docsum.getSummaryField(field.getName()) != null) { throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" + field.getName() + "'."); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java index 0c1e46a474c..5e8065b887d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java @@ -20,13 +20,14 @@ import java.util.Map; */ public class ImportedFieldsInSummayValidator extends Processor { - public ImportedFieldsInSummayValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + if (search.importedFields().isPresent()) { validateDocumentSummaries(search.getSummaries()); } @@ -48,15 +49,15 @@ public class ImportedFieldsInSummayValidator extends Processor { } private void validateImportedSummaryField(DocumentSummary summary, SummaryField field, ImportedField importedField) { - if (field.getDataType().equals(DataType.PREDICATE) && - importedField.targetField().getDataType().equals(DataType.PREDICATE)) { + if (field.getDataType().equals(DataType.PREDICATE) + && importedField.targetField().getDataType().equals(DataType.PREDICATE)) { fail(summary, field, "Is of type predicate. Not supported in document summaries"); } } private void fail(DocumentSummary summary, SummaryField importedField, String msg) { throw new IllegalArgumentException("For search '" + search.getName() + "', document summary '" + summary.getName() + - "', imported summary field '" + importedField.getName() + "': " + msg); + "', imported summary field '" + importedField.getName() + "': " + msg); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index 9b6a86ad13e..9997351d246 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -32,14 +32,14 @@ public class ImportedFieldsResolver extends Processor { } @Override - public void process() { - search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field)); + public void process(boolean validate) { + search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); search.setImportedFields(new ImportedFields(importedFields)); } - private void resolveImportedField(TemporaryImportedField importedField) { + private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { DocumentReference reference = validateDocumentReference(importedField); - SDField targetField = validateTargetField(importedField, reference); + SDField targetField = validateTargetField(importedField, reference, validate); importedFields.put(importedField.fieldName(), new ImportedField(importedField.fieldName(), reference, targetField)); } @@ -52,19 +52,27 @@ public class ImportedFieldsResolver extends Processor { return reference; } - private SDField validateTargetField(TemporaryImportedField importedField, DocumentReference reference) { + private SDField validateTargetField(TemporaryImportedField importedField, + DocumentReference reference, + boolean validate) { String targetFieldName = importedField.targetFieldName(); Search targetSearch = reference.targetSearch(); - if (isImportedField(targetSearch, targetFieldName)) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Is an imported field. Not supported"); + if (validate && isImportedField(targetSearch, targetFieldName)) { + fail(importedField, targetFieldAsString(targetFieldName, reference) + + ": Is an imported field. Not supported"); } SDField targetField = targetSearch.getConcreteField(targetFieldName); if (targetField == null) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Not found"); + fail(importedField, targetFieldAsString(targetFieldName, reference) + + ": Not found"); } else if (!targetField.doesAttributing()) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Is not an attribute field. Only attribute fields supported"); + if (validate) + fail(importedField, targetFieldAsString(targetFieldName, reference) + + ": Is not an attribute field. Only attribute fields supported"); } else if (targetField.doesIndexing()) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Is an index field. Not supported"); + if (validate) + fail(importedField, targetFieldAsString(targetFieldName, reference) + + ": Is an index field. Not supported"); } return targetField; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java index 8814cd8a61e..018183b91d8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * Because of the way the parser works (allowing any token as identifier), * it is not practical to limit the syntax of field names there, do it here. * Important to disallow dash, has semantic in IL. - * @author vegardh * + * @author Vehard Havdal */ public class IndexFieldNames extends Processor { @@ -23,9 +23,11 @@ public class IndexFieldNames extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { - if (!field.getName().matches(FIELD_NAME_REGEXP) && !legalDottedPositionField(field)) { + if ( ! field.getName().matches(FIELD_NAME_REGEXP) && ! legalDottedPositionField(field)) { fail(search, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP); } } @@ -34,10 +36,12 @@ public class IndexFieldNames extends Processor { /** * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. For 6.0, rename * to _position and _distance and delete this method. + * * @param field an {@link com.yahoo.searchdefinition.document.SDField} * @return true if allowed */ private boolean legalDottedPositionField(SDField field) { return field.getName().endsWith(".position") || field.getName().endsWith(".distance"); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java index 6427396ab88..0740029de90 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -14,8 +14,8 @@ import java.util.Iterator; * Fail if: * 1) There are index: settings without explicit index names (name same as field name) * 2) All the index-to indexes differ from the field name. - * @author vegardh * + * @author Vegard Havdal */ public class IndexSettingsNonFieldNames extends Processor { @@ -24,7 +24,9 @@ public class IndexSettingsNonFieldNames extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { boolean fieldNameUsed = false; for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) { @@ -33,11 +35,12 @@ public class IndexSettingsNonFieldNames extends Processor { fieldNameUsed = true; } } - if (!fieldNameUsed) { + if ( ! fieldNameUsed) { for (Index index : field.getIndices().values()) { if (index.getName().equals(field.getName())) { throw new IllegalArgumentException("Error in " + field + " in " + search + - ": When all index names differ from field name, index parameter settings must specify index name explicitly."); + ": When all index names differ from field name, index " + + "parameter settings must specify index name explicitly."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java deleted file mode 100644 index 8ac7655812f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Search; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * @author baldersheim - */ -public class IndexTo2FieldSet extends Processor { - - public IndexTo2FieldSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process() { - for (SDField field : search.allConcreteFields()) { - processField(field); - } - } - private void processField(SDField field) { - if (field.usesStructOrMap()) { - for (SDField subField : field.getStructFields()) { - processField(subField); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index 7a1076e758b..f6e95a81e77 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -15,10 +15,10 @@ import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * <p>This processor modifies all indexing scripts so that they input the value of the owning field by default. It also - * ensures that all fields used as input exist.</p> + * This processor modifies all indexing scripts so that they input the value of the owning field by default. It also + * ensures that all fields used as input exist. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class IndexingInputs extends Processor { @@ -27,16 +27,17 @@ public class IndexingInputs extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); - if (script == null) { - continue; - } + if (script == null) continue; + String fieldName = field.getName(); script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script); script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script); - new VerifyInputExpression(search, field).visit(script); + if (validate) + new VerifyInputExpression(search, field).visit(script); + field.setIndexingScript(script); } } @@ -95,16 +96,14 @@ public class IndexingInputs extends Processor { @Override protected void doVisit(Expression exp) { - if (!(exp instanceof InputExpression)) { - return; - } + if ( ! (exp instanceof InputExpression)) return; + SDDocumentType docType = search.getDocument(); String inputField = ((InputExpression)exp).getFieldName(); - if (docType.getField(inputField) != null) { - return; - } + if (docType.getField(inputField) != null) return; + fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " + - "in document type '" + docType.getName() + "'."); + "in document type '" + docType.getName() + "'."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 99cbf633221..420d0fc619b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -16,11 +16,11 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.*; /** - * <p>This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents + * This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents * any output expression from writing to any field except for the owning field. Finally, for <tt>SummaryExpression</tt>, - * this processor expands to write all appropriate summary fields.</p> + * this processor expands to write all appropriate summary fields. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class IndexingOutputs extends Processor { @@ -29,21 +29,19 @@ public class IndexingOutputs extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); - if (script == null) { - continue; - } + if (script == null) continue; + Set<String> summaryFields = new TreeSet<>(); findSummaryTo(search, field, summaryFields, summaryFields); - MyConverter converter = new MyConverter(search, field, summaryFields); + MyConverter converter = new MyConverter(search, field, summaryFields, validate); field.setIndexingScript((ScriptExpression)converter.convert(script)); } } - public void findSummaryTo(Search search, SDField field, Set<String> dynamicSummary, - Set<String> staticSummary) { + public void findSummaryTo(Search search, SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { Map<String, SummaryField> summaryFields = search.getSummaryFields(field); if (summaryFields.isEmpty()) { fillSummaryToFromField(field, dynamicSummary, staticSummary); @@ -59,7 +57,8 @@ public class IndexingOutputs extends Processor { } } - private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, Set<String> dynamicSummary, Set<String> staticSummary) { + private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, + Set<String> dynamicSummary, Set<String> staticSummary) { SummaryTransform summaryTransform = summaryField.getTransform(); String summaryName = summaryField.getName(); if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { @@ -98,23 +97,25 @@ public class IndexingOutputs extends Processor { final Search search; final Field field; final Set<String> summaryFields; + final boolean validate; - MyConverter(Search search, Field field, Set<String> summaryFields) { + MyConverter(Search search, Field field, Set<String> summaryFields, boolean validate) { this.search = search; this.field = field; this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; + this.validate = validate; } @Override protected boolean shouldConvert(Expression exp) { - if (!(exp instanceof OutputExpression)) { + if ( ! (exp instanceof OutputExpression)) { return false; } String fieldName = ((OutputExpression)exp).getFieldName(); if (fieldName == null) { return true; // inject appropriate field name } - if (!fieldName.equals(field.getName())) { + if ( validate && ! fieldName.equals(field.getName())) { fail(search, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" + field.getName() + "'."); } @@ -137,5 +138,7 @@ public class IndexingOutputs extends Processor { } return new StatementExpression(ret); } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index f9ea7ed1542..7ee22be1490 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -25,7 +25,9 @@ public class IndexingValidation extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + VerificationContext context = new VerificationContext(new MyAdapter(search)); for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); @@ -122,8 +124,8 @@ public class IndexingValidation extends Processor { } else { throw new UnsupportedOperationException(); } - if (!fieldType.isAssignableFrom(valueType) && - !fieldType.isAssignableFrom(createCompatType(valueType))) { + if ( ! fieldType.isAssignableFrom(valueType) && + ! fieldType.isAssignableFrom(createCompatType(valueType))) { throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + " '" + fieldName + "' which is " + fieldType.getName() + "."); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index cc634abef01..f3907ae68cb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -22,10 +22,12 @@ public class IndexingValues extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (Field field : search.getDocument().fieldSet()) { SDField sdField = (SDField)field; - if (!sdField.isExtraField()) { + if ( ! sdField.isExtraField()) { new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java index a850f74de79..c119dc2660b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -30,17 +30,18 @@ public class IntegerIndex2Attribute extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { - // Avoid changing for example RISE fields - if (field.getIndex(field.getName()) != null && !(field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue; + if (field.getIndex(field.getName()) != null + && ! (field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue; ScriptExpression script = field.getIndexingScript(); Set<String> attributeNames = new HashSet<>(); new MyVisitor(attributeNames).visit(script); field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); - warn(search, field, "Changed to attribute because numerical indexes (field has type "+field.getDataType().getName()+") is not currently supported." + - " Index-only settings may fail. Ignore this warning for streaming search."); + warn(search, field, "Changed to attribute because numerical indexes (field has type " + + field.getDataType().getName() + ") is not currently supported." + + " Index-only settings may fail. Ignore this warning for streaming search."); } } } @@ -83,4 +84,5 @@ public class IntegerIndex2Attribute extends Processor { return new AttributeExpression(indexName); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java index a003864c83a..7d13233bc12 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -31,7 +31,7 @@ public class LiteralBoost extends Processor { /** Adds extra search fields and indices to express literal boosts */ @Override - public void process() { + public void process(boolean validate) { checkRankModifierRankType(search); addLiteralBoostsToFields(search); reduceFieldLiteralBoosts(search); @@ -71,8 +71,8 @@ public class LiteralBoost extends Processor { private void reduceFieldLiteralBoost(SDField field,Search search) { SDField literalField = addField(search, field, "literal", - "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }", - "literal-boost"); + "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }", + "literal-boost"); literalField.setWeight(field.getWeight() + field.getLiteralBoost()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java index 9b5a6389a91..f853b99bbb2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -15,8 +15,8 @@ import java.util.Map; /** * Takes the aliases set on field by parser and sets them on correct Index or Attribute - * @author vegardh * + * @author vegardh */ public class MakeAliases extends Processor { @@ -25,21 +25,21 @@ public class MakeAliases extends Processor { } @Override - public void process() { + public void process(boolean validate) { List<String> usedAliases = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { String alias = e.getKey(); String name = e.getValue(); - String errMsg = "For search '"+search.getName()+"': alias '"+alias+"' "; - if (search.existsIndex(alias)) { - throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an index."); + String errMsg = "For search '" + search.getName() + "': alias '" + alias + "' "; + if (validate && search.existsIndex(alias)) { + throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an index."); } - if (search.getAttribute(alias)!=null) { - throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an attribute."); + if (validate && search.getAttribute(alias) != null) { + throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an attribute."); } - if (usedAliases.contains(alias)) { - throw new IllegalArgumentException(errMsg+"specified more than once."); + if (validate && usedAliases.contains(alias)) { + throw new IllegalArgumentException(errMsg + "specified more than once."); } usedAliases.add(alias); @@ -47,7 +47,7 @@ public class MakeAliases extends Processor { Attribute attribute = field.getAttributes().get(name); if (index != null) { index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search - } else if (attribute != null && !field.doesIndexing()) { + } else if (attribute != null && ! field.doesIndexing()) { attribute.getAliases().add(alias); } else { index = new Index(name); @@ -57,4 +57,5 @@ public class MakeAliases extends Processor { } } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index 0ebe384d361..2c43a65da99 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -25,7 +25,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * * <p>This must be done after other summary processors.</p> * - * @author bratseth + * @author bratseth */ public class MakeDefaultSummaryTheSuperSet extends Processor { @@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { } @Override - public void process() { + public void process(boolean validate) { DocumentSummary defaultSummary=search.getSummary("default"); for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index d1d3155a858..fe584fa41c0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -22,23 +22,28 @@ import java.util.Map; */ public class MatchConsistency extends Processor { - public MatchConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public MatchConsistency(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + Map<String, Matching.Type> types = new HashMap<>(); for (SDField field : search.allConcreteFields()) { new MyVisitor(search, field, types).visit(field.getIndexingScript()); } } - void checkMatching(Search search, SDField field, Map<String, Type> types, String indexTo) { + private void checkMatching(Search search, SDField field, Map<String, Type> types, String indexTo) { Type prevType = types.get(indexTo); if (prevType == null) { types.put(indexTo, field.getMatching().getType()); - } else if (!field.getMatching().getType().equals(prevType)) { + } else if ( ! field.getMatching().getType().equals(prevType)) { warn(search, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() + ") is inconsistent with that given for the same index in a previous field (had " + prevType + ")."); @@ -51,7 +56,7 @@ public class MatchConsistency extends Processor { final SDField field; final Map<String, Type> types; - public MyVisitor(Search search, SDField field, Map<String, Type> types) { + MyVisitor(Search search, SDField field, Map<String, Type> types) { this.search = search; this.field = field; this.types = types; @@ -64,4 +69,5 @@ public class MatchConsistency extends Processor { } } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index ae8cd092e77..043eb1f82eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -20,7 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) { RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); if (settings != null) { @@ -31,8 +33,9 @@ public class MatchPhaseSettingsValidator extends Processor { private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { String attributeName = settings.getAttribute(); - new AttributeValidator(search.getName(), rankProfile.getName(), - search.getAttribute(attributeName), attributeName).validate(); + new AttributeValidator(search.getName(), + rankProfile.getName(), + search.getAttribute(attributeName), attributeName).validate(); } public static class AttributeValidator { @@ -66,13 +69,13 @@ public class MatchPhaseSettingsValidator extends Processor { attribute.getType().equals(Attribute.Type.STRING) || attribute.getType().equals(Attribute.Type.PREDICATE)) { - failValidation("must be single value numeric, but it is '" - + attribute.getDataType().getName() + "'"); + failValidation("must be single value numeric, but it is '" + + attribute.getDataType().getName() + "'"); } } protected void validateThatAttributeIsFastSearch() { - if (!attribute.isFastSearch()) { + if ( ! attribute.isFastSearch()) { failValidation("must be fast-search, but it is not"); } } @@ -88,5 +91,7 @@ public class MatchPhaseSettingsValidator extends Processor { "', rank-profile '" + rankProfileName + "': " + getValidationType() + " attribute '" + attributeName + "' "; } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index 82b1d668397..45e018f8fc3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -31,16 +31,15 @@ public class MultifieldIndexHarmonizer extends Processor { } @Override - public void process() { + public void process(boolean validate) { populateIndexToFields(search); resolveAllConflicts(search); } private void populateIndexToFields(Search search) { for (SDField field : search.allConcreteFields() ) { - if (!field.doesIndexing()) { - continue; - } + if ( ! field.doesIndexing()) continue; + for (Iterator j = field.getFieldNameAsIterator(); j.hasNext();) { String indexName = (String)j.next(); addIndexField(indexName, field); @@ -49,10 +48,10 @@ public class MultifieldIndexHarmonizer extends Processor { } private void addIndexField(String indexName,SDField field) { - List<SDField> fields=indexToFields.get(indexName); - if (fields==null) { - fields=new java.util.ArrayList<>(); - indexToFields.put(indexName,fields); + List<SDField> fields = indexToFields.get(indexName); + if (fields == null) { + fields = new java.util.ArrayList<>(); + indexToFields.put(indexName, fields); } fields.add(field); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index 2b5e885b8fc..cdfba54ee5a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -26,23 +26,23 @@ public class NGramMatch extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType().equals(Matching.Type.GRAM)) - implementGramMatch(search, field); - else if (field.getMatching().getGramSize()>=0) + implementGramMatch(search, field, validate); + else if (validate && field.getMatching().getGramSize() >= 0) throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); } } - private void implementGramMatch(Search search, SDField field) { - if (field.doesAttributing()) + private void implementGramMatch(Search search, SDField field, boolean validate) { + if (validate && field.doesAttributing()) throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' not 'attribute' in indexing"); int n = field.getMatching().getGramSize(); - if (n<0) + if (n < 0) n=DEFAULT_GRAM_SIZE; // not set - use default gram size - if (n==0) + if (validate && n == 0) throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); field.getNormalizing().inferCodepoint(); field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing @@ -72,5 +72,7 @@ public class NGramMatch extends Processor { } return exp; } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index 2c82a9bafcb..c89c709ffbf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -13,21 +13,26 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect. */ public class OptimizeIlscript extends Processor { - public OptimizeIlscript(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + + public OptimizeIlscript(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); - if (script == null) { - continue; - } + if (script == null) continue; + field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); - if (!field.getIndexingScript().toString().equals(script.toString())) { - warn(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); + if ( ! field.getIndexingScript().toString().equals(script.toString())) { + warn(search, field, "Rewrote ilscript from:\n" + script.toString() + + "\nto\n" + field.getIndexingScript().toString()); } } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 4b9b090cdc5..3583c4a0162 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -33,26 +33,26 @@ public class PredicateProcessor extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { - if (field.doesIndexing()) { + if (validate && field.doesIndexing()) { fail(search, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); } if (field.doesAttributing()) { Attribute attribute = field.getAttributes().get(field.getName()); for (Index index : field.getIndices().values()) { BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton(); - if (booleanDefinition == null || !booleanDefinition.hasArity()) { + if (validate && (booleanDefinition == null || ! booleanDefinition.hasArity())) { fail(search, field, "Missing arity value in predicate field."); } - if (booleanDefinition.getArity() < 2) { + if (validate && (booleanDefinition.getArity() < 2)) { fail(search, field, "Invalid arity value in predicate field, must be greater than 1."); } double threshold = booleanDefinition.getDensePostingListThreshold(); - if (threshold <= 0 || threshold > 1) { + if (validate && (threshold <= 0 || threshold > 1)) { fail(search, field, "Invalid dense-posting-list-threshold value in predicate field. " + - "Value must be in range (0..1]."); + "Value must be in range (0..1]."); } attribute.setArity(booleanDefinition.getArity()); @@ -70,11 +70,11 @@ public class PredicateProcessor extends Processor { summaryField.setTransform(SummaryTransform.NONE); } } - } else if (field.getDataType().getPrimitiveType() == DataType.PREDICATE) { + } else if (validate && field.getDataType().getPrimitiveType() == DataType.PREDICATE) { fail(search, field, "Collections of predicates are not allowed."); - } else if (field.getDataType() == DataType.RAW && field.doesIndexing()) { + } else if (validate && field.getDataType() == DataType.RAW && field.doesIndexing()) { fail(search, field, "Indexing of RAW fields is not supported."); - } else { + } else if (validate) { // if field is not a predicate, disallow predicate-related index parameters for (Index index : field.getIndices().values()) { if (index.getBooleanIndexDefiniton() != null) { @@ -94,9 +94,8 @@ public class PredicateProcessor extends Processor { private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) { Expression script = field.getIndexingScript(); - if (script == null) { - return; - } + if (script == null) return; + script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); ExpressionConverter converter = new PredicateOutputTransformer(search); @@ -133,6 +132,7 @@ public class PredicateProcessor extends Processor { protected Expression newTransform(DataType fieldType) { return new OptimizePredicateExpression(); } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index ceb2f06f1d5..f76894f985c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -32,7 +32,6 @@ public class Processing { SetLanguage::new, UriHack::new, LiteralBoost::new, - IndexTo2FieldSet::new, TagType::new, IndexingInputs::new, OptimizeIlscript::new, @@ -76,7 +75,7 @@ public class Processing { ImportedFieldsInSummayValidator::new, FastAccessValidator::new, ReservedMacroNames::new, - //RankingExpressionTypeValidator::new, + RankingExpressionTypeValidator::new, // These should be last. IndexingValidation::new, @@ -95,11 +94,12 @@ public class Processing { public static void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { + QueryProfiles queryProfiles, + boolean validate) { search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(Processor::process); + .forEach(processor -> processor.process(validate)); } @FunctionalInterface diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index a9923675236..b0fbc6c1998 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -30,13 +30,16 @@ public abstract class Processor { /** * Base constructor + * * @param search the search to process * @param deployLogger Logger du use when logging deploy output. * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion. * @param queryProfiles The query profiles contained in the application this search is part of. */ - public Processor(Search search, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public Processor(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { this.search = search; this.deployLogger = deployLogger; this.rankProfileRegistry = rankProfileRegistry; @@ -46,8 +49,12 @@ public abstract class Processor { /** * Processes the input search definition by <b>modifying</b> the input search and its documents, and returns the * input search definition. + * + * @param validate true to throw exceptions on validation errors, false to make the best possible effort + * at completing processing without throwing an exception. + * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. */ - public abstract void process(); + public abstract void process(boolean validate); /** * Convenience method for adding a no-strings-attached implementation field for a regular field diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index baacceea667..f7f314f8444 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -33,7 +33,9 @@ public class RankingExpressionTypeValidator extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) { try { validate(profile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index f29d53d1c86..76afcd5d520 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor { } @Override - public void process() { + public void process(boolean validate) { clearSummaryAttributeAspectForConcreteFields(); clearSummaryAttributeAspectForExplicitSummaryFields(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java index 4152d07f8db..43a58fc9634 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -11,11 +11,12 @@ import java.util.HashSet; import java.util.Set; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class ReservedDocumentNames extends Processor { private static final Set<String> RESERVED_NAMES = new HashSet<>(); + static { for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) { RESERVED_NAMES.add(dataType.getName()); @@ -27,7 +28,9 @@ public class ReservedDocumentNames extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + String docName = search.getDocument().getName(); if (RESERVED_NAMES.contains(docName)) { throw new IllegalArgumentException("For search '" + search.getName() + "': Document name '" + docName + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java index f7036c2d7bd..b8eb1e1d8cf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java @@ -20,6 +20,7 @@ import java.util.logging.Level; * @author lesters */ public class ReservedMacroNames extends Processor { + private static Set<String> reservedNames = getReservedNames(); public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { @@ -27,14 +28,16 @@ public class ReservedMacroNames extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (RankProfile rp : rankProfileRegistry.allRankProfiles()) { for (String macroName : rp.getMacros().keySet()) { if (reservedNames.contains(macroName)) { deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " + - "in rank profile \"" + rp.getName() + "\" " + - "has a reserved name. This might mean that the macro shadows " + - "the built-in function with the same name." + "in rank profile \"" + rp.getName() + "\" " + + "has a reserved name. This might mean that the macro shadows " + + "the built-in function with the same name." ); } } @@ -49,4 +52,5 @@ public class ReservedMacroNames extends Processor { } return names.build(); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java index b3b9ece3909..403de1253b4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as * during feeding - * @author vegardh * + * @author Vegard Havdal */ public class SearchMustHaveDocument extends Processor { @@ -19,10 +19,12 @@ public class SearchMustHaveDocument extends Processor { } @Override - public void process() { - if (search.getDocument()==null) { - throw new IllegalArgumentException("For search '" + search.getName() + "': A search specification must have an equally named document inside of it."); - } + public void process(boolean validate) { + if ( ! validate) return; + + if (search.getDocument() == null) + throw new IllegalArgumentException("For search '" + search.getName() + + "': A search specification must have an equally named document inside of it."); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java index 38c1d1a94d3..a0c884d25f9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -24,7 +24,7 @@ public class SetLanguage extends Processor { } @Override - public void process() { + public void process(boolean validate) { List<String> textFieldsWithoutLanguage = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { @@ -37,10 +37,10 @@ public class SetLanguage extends Processor { fieldString.append(fieldName).append(" "); } warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " + - "and should be defined as the first field in the searchdefinition." + - "Preceding text fields that will not have their language set: " + - fieldString.toString() + - " (This warning is omitted for any subsequent fields that also do set_language.)"); + "and should be defined as the first field in the searchdefinition." + + "Preceding text fields that will not have their language set: " + + fieldString.toString() + + " (This warning is omitted for any subsequent fields that also do set_language.)"); return; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java index ffc6e1e32a2..a19ea8d7068 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (field.getRanking().isFilter()) { field.setRankType(RankType.EMPTY); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java index 3498515ab56..6426c724a07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Validate conflicting settings for sorting - * @author vegardh * + * @author Vegard Havdal */ public class SortingSettings extends Processor { @@ -21,15 +21,17 @@ public class SortingSettings extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { for (Attribute attribute : field.getAttributes().values()) { Sorting sorting = attribute.getSorting(); - if (sorting.getFunction()!=Sorting.Function.UCA) { - if (sorting.getStrength()!=null && sorting.getStrength()!=Sorting.Strength.PRIMARY) { + if (sorting.getFunction() != Sorting.Function.UCA) { + if (sorting.getStrength()!=null && sorting.getStrength() != Sorting.Strength.PRIMARY) { warn(search, field, "Sort strength only works for sort function 'uca'."); } - if (sorting.getLocale()!=null && !"".equals(sorting.getLocale())) { + if (sorting.getLocale() != null && ! "".equals(sorting.getLocale())) { warn(search, field, "Sort locale only works for sort function 'uca'."); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java index b01fbb47bf1..d56b0272f06 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -16,11 +16,14 @@ public class StringSettingsOnNonStringFields extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { - if (!doCheck(field)) continue; + if ( ! doCheck(field)) continue; if (field.getMatching().isTypeUserSet()) { - warn(search, field, "Matching type "+field.getMatching().getType()+" is only allowed for string fields."); + warn(search, field, "Matching type " + field.getMatching().getType() + + " is only allowed for string fields."); } if (field.getRanking().isLiteral()) { warn(search, field, "Rank type literal only applies to string fields"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index 3bb33cfb949..a952d3732b3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -26,39 +26,43 @@ public class SummaryConsistency extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (DocumentSummary summary : search.getSummaries().values()) { if (summary.getName().equals("default")) continue; + for (SummaryField summaryField : summary.getSummaryFields() ) { - assertConsistency(summaryField, search); + assertConsistency(summaryField, search, validate); makeAttributeTransformIfAppropriate(summaryField, search); } } } - /** If the source is an attribute, make this use the attribute transform */ - private void makeAttributeTransformIfAppropriate(SummaryField summaryField,Search search) { - if (summaryField.getTransform() != SummaryTransform.NONE) return; - Attribute attribute = search.getAttribute(summaryField.getSingleSource()); - if (attribute == null) return; - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - - private void assertConsistency(SummaryField summaryField, Search search) { - SummaryField existingDefault = search.getSummary("default").getSummaryField(summaryField.getName()); // Compare to default + private void assertConsistency(SummaryField summaryField, Search search, boolean validate) { + // Compare to default: + SummaryField existingDefault = search.getSummary("default").getSummaryField(summaryField.getName()); if (existingDefault != null) { - assertConsistentTypes(existingDefault, summaryField); + if (validate) + assertConsistentTypes(existingDefault, summaryField); makeConsistentWithDefaultOrThrow(existingDefault, summaryField); } else { // If no default, compare to whichever definition of the field SummaryField existing = search.getExplicitSummaryField(summaryField.getName()); if (existing == null) return; - assertConsistentTypes(existing, summaryField); + if (validate) + assertConsistentTypes(existing, summaryField); makeConsistentOrThrow(existing, summaryField, search); } } + /** If the source is an attribute, make this use the attribute transform */ + private void makeAttributeTransformIfAppropriate(SummaryField summaryField,Search search) { + if (summaryField.getTransform() != SummaryTransform.NONE) return; + Attribute attribute = search.getAttribute(summaryField.getSingleSource()); + if (attribute == null) return; + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) @@ -90,6 +94,7 @@ public class SummaryConsistency extends Processor { assertEqualTransform(field1,field2); } } + private void makeConsistentWithDefaultOrThrow(SummaryField defaultField, SummaryField newField) { if (newField.getTransform().equals(SummaryTransform.NONE)) { newField.setTransform(defaultField.getTransform()); @@ -99,12 +104,11 @@ public class SummaryConsistency extends Processor { } } - private void assertEqualTransform(SummaryField field1, SummaryField field2) { if ( ! field2.getTransform().equals(field1.getTransform())) { - throw new IllegalArgumentException("Conflicting summary transforms. " + field2 +" is already defined as " + - field1 + ". A field with the same name " + - "can not have different transforms in different summary classes"); + throw new IllegalArgumentException("Conflicting summary transforms. " + field2 + " is already defined as " + + field1 + ". A field with the same name " + + "can not have different transforms in different summary classes"); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java index 0f9864e854f..647a433f201 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -16,8 +16,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * An SD field explicitly says summary:dynamic , but the field is wset, array or struct. * If there is an explicitly defined summary class, saying dynamic in one of its summary * fields is always legal. - * @author vegardh * + * @author Vegard Havdal */ public class SummaryDynamicStructsArrays extends Processor { @@ -26,16 +26,18 @@ public class SummaryDynamicStructsArrays extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { DataType type = field.getDataType(); - if (type instanceof ArrayDataType || type instanceof WeightedSetDataType - || type instanceof StructDataType) { + if (type instanceof ArrayDataType || type instanceof WeightedSetDataType || type instanceof StructDataType) { for (SummaryField sField : field.getSummaryFields()) { if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) { throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " + - "for fields of type struct, array or weighted set. Use an explicit summary class with explicit summary fields sourcing from" + - " the array/struct/weighted set."); + "for fields of type struct, array or weighted set. Use an " + + "explicit summary class with explicit summary fields sourcing" + + " from the array/struct/weighted set."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 746dd284d23..7bcbd9a267c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -16,11 +16,15 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * */ public class SummaryFieldsMustHaveValidSource extends Processor { + SummaryFieldsMustHaveValidSource(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } + @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (DocumentSummary summary : search.getSummaries().values()) { for (SummaryField summaryField : summary.getSummaryFields()) { if (summaryField.getSources().isEmpty()) { @@ -52,8 +56,9 @@ public class SummaryFieldsMustHaveValidSource extends Processor { private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) { if ( ! isValid(source, summaryField, summary) ) { - throw new IllegalArgumentException("For search '" + search.getName() + "', summary class '" + summary.getName() + "'," + - " summary field '" + summaryField.getName() + "': there is no valid source '" + source + "'."); + throw new IllegalArgumentException("For search '" + search.getName() + "', summary class '" + + summary.getName() + "'," + " summary field '" + summaryField.getName() + + "': there is no valid source '" + source + "'."); } } @@ -64,6 +69,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { private static boolean isInThisSummaryClass(DocumentSummary summary, String name) { return summary.getSummaryField(name) != null; } + private boolean isDocumentField(String name) { return search.getField(name) != null; } @@ -71,4 +77,5 @@ public class SummaryFieldsMustHaveValidSource extends Processor { private boolean isSummaryField(String name) { return search.getSummaryField(name) != null; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java index 661904f8028..23569cf39ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -17,8 +17,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * Verifies that equally named summary fields in different summary classes don't use different fields for source. * The summarymap config doesn't model this. * - * @author vegardh - * + * @author Vegard Havdal */ public class SummaryNamesFieldCollisions extends Processor { @@ -27,7 +26,9 @@ public class SummaryNamesFieldCollisions extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>(); for (DocumentSummary summary : search.getSummaries().values()) { if ("default".equals(summary.getName())) continue; @@ -38,12 +39,14 @@ public class SummaryNamesFieldCollisions extends Processor { if (prevClassAndSource!=null) { String prevClass = prevClassAndSource.getFirst(); String prevSource = prevClassAndSource.getSecond(); - if (!prevClass.equals(summary.getName())) { - if (!prevSource.equals(source.getName())) { - throw new IllegalArgumentException("For search '"+search.getName()+"', summary class '"+summary.getName()+"'," + - " summary field '"+summaryField.getName()+"':" + - " Can not use source '"+source.getName()+"' for this summary field, an equally named field in summary class '" + - prevClass + "' uses a different source: '"+prevSource+"'."); + if ( ! prevClass.equals(summary.getName())) { + if ( ! prevSource.equals(source.getName())) { + throw new IllegalArgumentException("For search '"+ search.getName() + + "', summary class '" + summary.getName()+"'," + + " summary field '" + summaryField.getName() + "':" + + " Can not use source '" + source.getName() + + "' for this summary field, an equally named field in summary class '" + + prevClass + "' uses a different source: '"+prevSource+"'."); } } } else { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 8efeb0ed6d6..177fc7f2326 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -17,28 +17,29 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; */ public class TagType extends Processor { - public TagType(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public TagType(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { - if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) { + if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) implementTagType(field); - } } } private void implementTagType(SDField field) { - field.setDataType(DataType.getWeightedSet(DataType.STRING,true,true)); + field.setDataType(DataType.getWeightedSet(DataType.STRING, true, true)); // Don't set matching and ranking if this field is not attribute nor index if (!field.doesIndexing() && !field.doesAttributing()) return; Matching m = field.getMatching(); - if (!m.isTypeUserSet()) { + if ( ! m.isTypeUserSet()) m.setType(Matching.Type.WORD); - } - if (field.getRankType()==null || field.getRankType()== RankType.DEFAULT) + if (field.getRankType() == null || field.getRankType() == RankType.DEFAULT) field.setRankType((RankType.TAGS)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 7ec397b16b4..427089ebd24 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -22,7 +22,9 @@ public class TensorFieldProcessor extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { warnUseOfTensorFieldAsAttribute(field); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index 02655906f65..b45f7d61a37 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -34,36 +34,35 @@ public class TextMatch extends Processor { } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { - if (field.getMatching().getType() != Matching.Type.TEXT) { - continue; - } + if (field.getMatching().getType() != Matching.Type.TEXT) continue; + ScriptExpression script = field.getIndexingScript(); - if (script == null) { - continue; - } + if (script == null) continue; + DataType fieldType = field.getDataType(); if (fieldType instanceof CollectionDataType) { fieldType = ((CollectionDataType)fieldType).getNestedType(); } - if (fieldType != DataType.STRING) { - continue; - } + if (fieldType != DataType.STRING) continue; + Set<String> dynamicSummary = new TreeSet<>(); Set<String> staticSummary = new TreeSet<>(); - new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(search, field, dynamicSummary, staticSummary); + new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(search, + field, + dynamicSummary, + staticSummary); MyVisitor visitor = new MyVisitor(dynamicSummary); visitor.visit(script); - if (!visitor.requiresTokenize) { - continue; - } + if ( ! visitor.requiresTokenize) continue; + ExpressionConverter converter = new MyStringTokenizer(search, findAnnotatorConfig(search, field)); field.setIndexingScript((ScriptExpression)converter.convert(script)); } } - private static AnnotatorConfig findAnnotatorConfig(Search search, SDField field) { + private AnnotatorConfig findAnnotatorConfig(Search search, SDField field) { AnnotatorConfig ret = new AnnotatorConfig(); Stemming activeStemming = field.getStemming(); if (activeStemming == null) { @@ -97,6 +96,7 @@ public class TextMatch extends Processor { requiresTokenize = true; } } + } private static class MyStringTokenizer extends TypedTransformProvider { @@ -121,5 +121,7 @@ public class TextMatch extends Processor { } return exp; } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java index 8ad985f07b9..b329cc3bea3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; import com.yahoo.vespa.indexinglanguage.expressions.*; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public abstract class TypedTransformProvider extends ValueTransformProvider { @@ -27,23 +27,23 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { String fieldName = ((OutputExpression)exp).getFieldName(); if (exp instanceof AttributeExpression) { Attribute attribute = search.getAttribute(fieldName); - if (attribute == null) { + if (attribute == null) throw new IllegalArgumentException("Attribute '" + fieldName + "' not found."); - } fieldType = attribute.getDataType(); - } else if (exp instanceof IndexExpression) { + } + else if (exp instanceof IndexExpression) { Field field = search.getConcreteField(fieldName); - if (field == null) { + if (field == null) throw new IllegalArgumentException("Index field '" + fieldName + "' not found."); - } fieldType = field.getDataType(); - } else if (exp instanceof SummaryExpression) { + } + else if (exp instanceof SummaryExpression) { Field field = search.getSummaryField(fieldName); - if (field == null) { + if (field == null) throw new IllegalArgumentException("Summary field '" + fieldName + "' not found."); - } fieldType = field.getDataType(); - } else { + } + else { throw new UnsupportedOperationException(); } } @@ -58,4 +58,5 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { protected abstract boolean requiresTransform(Expression exp, DataType fieldType); protected abstract Expression newTransform(DataType fieldType); + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index e3e3f003612..ac376982cfa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -16,14 +16,19 @@ import java.util.List; * @author baldersheim */ public class UriHack extends Processor { + private static final List<String> URL_SUFFIX = Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); - public UriHack(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + + public UriHack(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing()) { DataType fieldType = field.getDataType(); @@ -43,10 +48,12 @@ public class UriHack extends Processor { DataType generatedType = DataType.STRING; if (uriField.getDataType() instanceof ArrayDataType) { generatedType = new ArrayDataType(DataType.STRING); - } else if (uriField.getDataType() instanceof WeightedSetDataType) { + } + else if (uriField.getDataType() instanceof WeightedSetDataType) { WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); } + for (String suffix : URL_SUFFIX) { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index b1cbb2cbf68..ed813b42fff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -18,7 +18,9 @@ public class UrlFieldValidator extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + for (SDField field : search.allConcreteFields()) { if ( ! field.getDataType().equals(DataType.URI)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 8e0f7a7f340..45e4b994ac9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -28,7 +28,9 @@ public class ValidateFieldTypes extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + String searchName = search.getName(); Map<String, DataType> seenFields = new HashMap<>(); search.allFields().forEach(field -> { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index bc7d7195eb5..a0b204a25f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -21,22 +21,22 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { } @Override - public void process() { + public void process(boolean validate) { + if ( ! validate) return; + Matching defaultMatching = new Matching(); Ranking defaultRanking = new Ranking(); for (SDField field : search.allConcreteFields()) { - if (field.doesIndexing()) { - continue; - } - if (field.doesAttributing()) { - continue; - } - if (!(field.getRanking().equals(defaultRanking))) { - fail(search, field, "Fields which are not creating an index or attribute can not contain rank settings."); - } - if (!(field.getMatching().equals(defaultMatching))) { - fail(search, field, "Fields which are not creating an index or attribute can not contain match settings."); - } + if (field.doesIndexing()) continue; + if (field.doesAttributing()) continue; + + if ( ! field.getRanking().equals(defaultRanking)) + fail(search, field, + "Fields which are not creating an index or attribute can not contain rank settings."); + if ( ! field.getMatching().equals(defaultMatching)) + fail(search, field, + "Fields which are not creating an index or attribute can not contain match settings."); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index c80624a5019..892bcdad12c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -25,14 +25,14 @@ public class WordMatch extends Processor { super(search, deployLogger, rankProfileRegistry, queryProfiles); } - public void process() { + public void process(boolean validate) { for (SDField field : search.allConcreteFields()) { - if (!field.getMatching().getType().equals(Matching.Type.WORD)) { - continue; - } + if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; + field.setStemming(Stemming.NONE); field.getNormalizing().inferLowercase(); field.addQueryCommand("word"); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 7cb9fee3449..e402217cb9f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -34,7 +34,7 @@ public class RankProfileTypeSettingsProcessor extends Processor { } @Override - public void process() { + public void process(boolean validate) { processAttributeFields(); processImportedFields(); processQueryProfileTypes(); |