diff options
Diffstat (limited to 'config-model')
15 files changed, 252 insertions, 172 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index a3580a404a3..a8f6dd09497 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -6,9 +6,9 @@ import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; -import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; @@ -51,14 +51,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - if (unsupportedFieldType(field, search.getDocument())) { + if (unsupportedFieldType(field)) { return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); - } else if (isArrayOfSimpleStruct(field, search.getDocument())) { + } else if (isArrayOfSimpleStruct(field)) { deriveArrayOfSimpleStruct(field); - } else if (isMapOfSimpleStruct(field, search.getDocument())) { + } else if (isMapOfSimpleStruct(field)) { deriveMapOfSimpleStruct(field); } else if (isMapOfPrimitiveType(field)) { deriveMapOfPrimitiveType(field); @@ -67,9 +67,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } - private static boolean unsupportedFieldType(ImmutableSDField field, SDDocumentType docType) { + private static boolean unsupportedFieldType(ImmutableSDField field) { return (field.usesStructOrMap() && - !isSupportedComplexField(field, docType) && + !isSupportedComplexField(field) && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 81e44850e71..bdd027f4687 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -52,6 +52,7 @@ public final class Attribute implements Cloneable, Serializable { private boolean fastSearch = false; private boolean fastAccess = false; private boolean huge = false; + private boolean mutable = false; private int arity = BooleanIndexDefinition.DEFAULT_ARITY; private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; @@ -181,6 +182,7 @@ public final class Attribute implements Cloneable, Serializable { public boolean isFastAccess() { return fastAccess; } public boolean isHuge() { return huge; } public boolean isPosition() { return isPosition; } + public boolean isMutable() { return mutable; } public int arity() { return arity; } public long lowerBound() { return lowerBound; } @@ -205,6 +207,7 @@ public final class Attribute implements Cloneable, Serializable { public void setHuge(boolean huge) { this.huge = huge; } public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } public void setPosition(boolean position) { this.isPosition = position; } + public void setMutable(boolean mutable) { this.mutable = mutable; } public void setArity(int arity) { this.arity = arity; } public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; } public void setUpperBound(long upperBound) { this.upperBound = upperBound; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index 72eb1c96e0f..c3ac8c77173 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -3,15 +3,9 @@ package com.yahoo.searchdefinition.document; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; -import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.TemporaryStructuredDataType; - -import java.util.Collection; -import java.util.Collections; -import java.util.Optional; /** * Utils used to check whether a complex field supports being represented as struct field attributes. @@ -21,64 +15,42 @@ import java.util.Optional; * - map of primitive type to simple struct * - map of primitive type to primitive type * + * A simple struct can contain fields of any type, but only fields of primitive type can be defined as + * struct field attributes in the complex field using the simple struct. + * * @author geirst */ public class ComplexAttributeFieldUtils { - public static boolean isSupportedComplexField(ImmutableSDField field, SDDocumentType docType) { - return (isArrayOfSimpleStruct(field, docType) || - isMapOfSimpleStruct(field, docType) || + public static boolean isSupportedComplexField(ImmutableSDField field) { + return (isArrayOfSimpleStruct(field) || + isMapOfSimpleStruct(field) || isMapOfPrimitiveType(field)); } - public static boolean isSupportedComplexField(DataType fieldType) { - return (isArrayOfSimpleStruct(fieldType) || - isMapOfSimpleStruct(fieldType) || - isMapOfPrimitiveType(fieldType)); - } - - public static boolean isArrayOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { - return isArrayOfSimpleStruct(field.getDataType(), Optional.of(docType)); - } - - public static boolean isArrayOfSimpleStruct(DataType fieldType) { - return isArrayOfSimpleStruct(fieldType, Optional.empty()); - } - - private static boolean isArrayOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) { - if (fieldType instanceof ArrayDataType) { - ArrayDataType arrayType = (ArrayDataType)fieldType; - return isSimpleStruct(arrayType.getNestedType(), docType); + public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)field.getDataType(); + return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field); } else { return false; } } - public static boolean isMapOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { - return isMapOfSimpleStruct(field.getDataType(), Optional.of(docType)); - } - - public static boolean isMapOfSimpleStruct(DataType fieldType) { - return isMapOfSimpleStruct(fieldType, Optional.empty()); - } - - private static boolean isMapOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) { - if (fieldType instanceof MapDataType) { - MapDataType mapType = (MapDataType)fieldType; + public static boolean isMapOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); return isPrimitiveType(mapType.getKeyType()) && - isSimpleStruct(mapType.getValueType(), docType); + isStructWithPrimitiveStructFieldAttributes(mapType.getValueType(), + field.getStructField("value")); } else { return false; } } public static boolean isMapOfPrimitiveType(ImmutableSDField field) { - return isMapOfPrimitiveType(field.getDataType()); - } - - public static boolean isMapOfPrimitiveType(DataType fieldType) { - if (fieldType instanceof MapDataType) { - MapDataType mapType = (MapDataType)fieldType; + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); return isPrimitiveType(mapType.getKeyType()) && isPrimitiveType(mapType.getValueType()); } else { @@ -86,17 +58,15 @@ public class ComplexAttributeFieldUtils { } } - private static boolean isSimpleStruct(DataType type, Optional<SDDocumentType> docType) { + private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { if (type instanceof StructDataType && !(type.equals(PositionDataType.INSTANCE))) { - StructDataType structType = (StructDataType) type; - Collection<Field> structFields = getStructFields(structType, docType); - if (structFields.isEmpty()) { - return false; - } - for (Field innerField : structFields) { - if (!isPrimitiveType(innerField.getDataType())) { - return false; + for (ImmutableSDField structField : field.getStructFields()) { + Attribute attribute = structField.getAttributes().get(structField.getName()); + if (attribute != null) { + if (!isPrimitiveType(attribute)) { + return false; + } } } return true; @@ -105,20 +75,12 @@ public class ComplexAttributeFieldUtils { } } - private static Collection<Field> getStructFields(StructDataType structType, Optional<SDDocumentType> docType) { - // The struct data type might be unresolved at this point. If so we use the document type to resolve it. - if (docType.isPresent() && (structType instanceof TemporaryStructuredDataType)) { - SDDocumentType realStructType = docType.get().getOwnedType(structType.getName()); - if (structType != null) { - return realStructType.getDocumentType().getFields(); - } - return Collections.emptyList(); - } else { - return structType.getFields(); - } + private static boolean isPrimitiveType(Attribute attribute) { + return attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) && + isPrimitiveType(attribute.getDataType()); } - private static boolean isPrimitiveType(DataType dataType) { + public static boolean isPrimitiveType(DataType dataType) { return dataType.equals(DataType.BYTE) || dataType.equals(DataType.INT) || dataType.equals(DataType.LONG) || @@ -127,10 +89,10 @@ public class ComplexAttributeFieldUtils { dataType.equals(DataType.STRING); } - public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field, SDDocumentType docType) { - if (isArrayOfSimpleStruct(field, docType)) { + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { + if (isArrayOfSimpleStruct(field)) { return hasOnlyStructFieldAttributes(field); - } else if (isMapOfSimpleStruct(field, docType)) { + } else if (isMapOfSimpleStruct(field)) { return hasSingleAttribute(field.getStructField("key")) && hasOnlyStructFieldAttributes(field.getStructField("value")); } else if (isMapOfPrimitiveType(field)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index 46ac3cc1691..4df3660a967 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -16,6 +16,7 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain private Boolean huge; private Boolean fastSearch; private Boolean fastAccess; + private Boolean mutable; private Boolean prefetch; private Boolean enableBitVectors; private Boolean enableOnlyBitVector; @@ -68,6 +69,9 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain public void setFastAccess(Boolean fastAccess) { this.fastAccess = fastAccess; } + public void setMutable(Boolean mutable) { + this.mutable = mutable; + } public Boolean getPrefetch() { return prefetch; @@ -143,6 +147,9 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain if (fastAccess != null) { attribute.setFastAccess(fastAccess); } + if (mutable != null) { + attribute.setMutable(mutable); + } if (prefetch != null) { attribute.setPrefetch(prefetch); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index 34903abb288..a95f4264dc6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -38,7 +38,7 @@ public class AttributesImplicitWord extends Processor { return false; } return (field.getIndexToCount() == 0 - && field.getAttributes().size() > 0 + && !field.getAttributes().isEmpty() && field.getIndices().isEmpty() && !field.getMatching().isTypeUserSet()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 2d2fc10d9c5..b51524b7e62 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -83,7 +83,7 @@ public class ImplicitSummaries extends Processor { } } - if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field, search.getDocument())) { + if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java new file mode 100644 index 00000000000..e35da451bff --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java @@ -0,0 +1,28 @@ +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +public class MutableAttributes extends Processor { + + public MutableAttributes(Search search, DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) + { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate) { + for (SDField field : search.allConcreteFields()) { + if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if (field.getAttributes().get(field.getName()).isMutable()) { + throw new IllegalArgumentException("Field " + field + " in '" + search.getDocument().getName() + + "' can not be marked mutable as it inside the document."); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index cedbebe3b4e..19025d37f8c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -37,6 +37,7 @@ public class Processing { OptimizeIlscript::new, ValidateFieldWithIndexSettingsCreatesIndex::new, AttributesImplicitWord::new, + MutableAttributes::new, CreatePositionZCurve::new, WordMatch::new, DeprecateAttributePrefetch::new, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 005c1dd8b2e..bc89d0dfcf5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -61,7 +61,7 @@ public class ComplexAttributeFieldsValidator extends Validator { } private static boolean isSupportedComplexField(ImmutableSDField field) { - return (ComplexAttributeFieldUtils.isSupportedComplexField(field.getDataType()) || + return (ComplexAttributeFieldUtils.isSupportedComplexField(field) || field.getDataType().equals(PositionDataType.INSTANCE) || field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java index 520494f1697..4dfeb808e31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -10,6 +10,7 @@ import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; @@ -20,10 +21,6 @@ import java.util.List; import java.util.StringTokenizer; import java.util.stream.Collectors; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; - /** * Validates the changes between the current and next set of struct field attributes in a document database. @@ -66,7 +63,7 @@ public class StructFieldAttributeChangeValidator { private List<VespaConfigChangeAction> validateAddAttributeAspect(Context current, Context next, ValidationOverrides overrides, Instant now) { return next.structFieldAttributes.stream() - .filter(nextAttr -> current.hasFieldFor(nextAttr) && + .filter(nextAttr -> current.hasFieldForStructFieldAttribute(nextAttr) && !current.hasStructFieldAttribute(nextAttr)) .map(nextAttr -> VespaRefeedAction.of("field-type-change", overrides, @@ -94,23 +91,23 @@ public class StructFieldAttributeChangeValidator { .anyMatch(attr -> attr.getName().equals(structFieldAttribute.getName())); } - public boolean hasFieldFor(Attribute structFieldAttribute) { + public boolean hasFieldForStructFieldAttribute(Attribute structFieldAttribute) { StringTokenizer fieldNames = new StringTokenizer(structFieldAttribute.getName(), "."); if (!fieldNames.nextToken().equals(field.getName())) { return false; } - if (isArrayOfSimpleStruct(dataType())) { + if (isArrayOfStructType(dataType())) { StructDataType nestedType = (StructDataType)((ArrayDataType)dataType()).getNestedType(); - if (hasLastFieldInStructType(fieldNames, nestedType)) { + if (structTypeContainsLastFieldNameComponent(nestedType, fieldNames)) { return true; } - } else if (isMapOfSimpleStruct(dataType())) { + } else if (isMapOfStructType(dataType())) { MapDataType mapType = (MapDataType)dataType(); StructDataType valueType = (StructDataType)mapType.getValueType(); String subFieldName = fieldNames.nextToken(); if (subFieldName.equals("key") && !fieldNames.hasMoreTokens()) { return true; - } else if (subFieldName.equals("value") && hasLastFieldInStructType(fieldNames, valueType)) { + } else if (subFieldName.equals("value") && structTypeContainsLastFieldNameComponent(valueType, fieldNames)) { return true; } } else if (isMapOfPrimitiveType(dataType())) { @@ -123,10 +120,42 @@ public class StructFieldAttributeChangeValidator { return false; } - private static boolean hasLastFieldInStructType(StringTokenizer fieldNames, StructDataType structType) { - return structType.getField(fieldNames.nextToken()) != null && !fieldNames.hasMoreTokens(); + private static boolean isArrayOfStructType(DataType type) { + if (type instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)type; + return isStructType(arrayType.getNestedType()); + } else { + return false; + } + } + + private static boolean isMapOfStructType(DataType type) { + if (type instanceof MapDataType) { + MapDataType mapType = (MapDataType)type; + return ComplexAttributeFieldUtils.isPrimitiveType(mapType.getKeyType()) && + isStructType(mapType.getValueType()); + } else { + return false; + } } + public static boolean isMapOfPrimitiveType(DataType type) { + if (type instanceof MapDataType) { + MapDataType mapType = (MapDataType)type; + return ComplexAttributeFieldUtils.isPrimitiveType(mapType.getKeyType()) && + ComplexAttributeFieldUtils.isPrimitiveType(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isStructType(DataType type) { + return (type instanceof StructDataType); + } + + private static boolean structTypeContainsLastFieldNameComponent(StructDataType structType, StringTokenizer fieldNames) { + return structType.getField(fieldNames.nextToken()) != null && !fieldNames.hasMoreTokens(); + } } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 2c4c9a47fec..63d3926afad 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -281,6 +281,7 @@ TOKEN : | < ENABLEBITVECTORS: "enable-bit-vectors" > | < ENABLEONLYBITVECTOR: "enable-only-bit-vector" > | < FASTACCESS: "fast-access" > +| < MUTABLE: "mutable" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > | < PREFETCH: "prefetch" > @@ -1190,16 +1191,18 @@ Object attributeSetting(FieldOperationContainer field, AttributeOperation attrib } { ( - <HUGE> { attribute.setHuge(true); } - | <FASTSEARCH> { attribute.setFastSearch(true); } - | <FASTACCESS> { attribute.setFastAccess(true); } - | <ENABLEBITVECTORS> { attribute.setEnableBitVectors(true); } + <HUGE> { attribute.setHuge(true); } + | <FASTSEARCH> { attribute.setFastSearch(true); } + | <FASTACCESS> { attribute.setFastAccess(true); } + | <MUTABLE> { attribute.setMutable(true); } + | <ENABLEBITVECTORS> { attribute.setEnableBitVectors(true); } | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); } - | <NOPREFETCH> { + + | <NOPREFETCH> { deployLogger.log(Level.WARNING, field + ": 'attribute : no-prefetch' is deprecated and has no effect."); attribute.setPrefetch(false); } - | <PREFETCH> { + | <PREFETCH> { deployLogger.log(Level.WARNING, field + ": 'attribute : prefetch' is deprecated and has no effect."); attribute.setPrefetch(true); } @@ -2515,6 +2518,7 @@ String identifier() : { } | <MAXFILTERCOVERAGE> | <MAXHITS> | <MTOKEN> + | <MUTABLE> | <NEVER> | <NONE> | <NOPREFETCH> diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 4ee33abfc08..1b58db0322b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -4,10 +4,11 @@ package com.yahoo.searchdefinition; import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.tensor.TensorType; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.IOException; import java.util.Optional; @@ -22,6 +23,9 @@ import static org.junit.Assert.*; */ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + @Test public void testAttributeSettings() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/attributesettings.sd"); @@ -92,6 +96,59 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { assertTrue(attr.isFastAccess()); } + private Attribute getAttributeF(String sd) throws ParseException { + SearchBuilder builder = new SearchBuilder(); + builder.importString(sd); + builder.build(); + Search search = builder.getSearch(); + SDField field = (SDField) search.getDocument().getField("f"); + return field.getAttributes().get(field.getName()); + } + @Test + public void requireThatMutableIsDefaultOff() throws ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field f type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + "}\n"); + assertFalse(attr.isMutable()); + } + + @Test + public void requireThatMutableCanNotbeSetInDocument() throws ParseException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("Field field 'f' in 'test' can not be marked mutable as it inside the document."); + Attribute attr = getAttributeF( + "search test {\n" + + " document test {\n" + + " field f type int {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " }\n" + + "}\n"); + } + + @Test + public void requireThatMutableExtraFieldCanBeSet() throws IOException, ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " field f type long {\n" + + " indexing: 0 | to_long | attribute\n" + + " attribute: mutable\n" + + " }\n" + + "}\n"); + assertTrue(attr.isMutable()); + } + @Test public void attribute_convert_to_array_copies_internal_state() { StructDataType refType = new StructDataType("my_struct"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 7a423f6f82a..f39896f5779 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -24,50 +24,6 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase static final String tempDir = "temp/"; static final String searchDefRoot = "src/test/derived/"; - private static final boolean WRITE_FILES = false; - - static { - if ("true".equals(System.getProperty("sd.updatetests"))) { - /* - * Use this when you know that your code is correct, and don't want to manually check and fix the .cfg files - * in the exporting tests. - */ - setUpUpdateTest(); - } - // Or uncomment the lines you want below AND set WRITE_FILES to true - //System.setProperty("sd.updatetests", "true"); - //System.setProperty("sd.updatetestfile", "attributes"); - //System.setProperty("sd.updatetestfile", "documentmanager"); - //System.setProperty("sd.updatetestfile", "fdispatchrc"); - //System.setProperty("sd.updatetestfile", "ilscripts"); - //System.setProperty("sd.updatetestfile", "index-info"); - //System.setProperty("sd.updatetestfile", "indexschema"); - //System.setProperty("sd.updatetestfile", "juniperrc"); - //System.setProperty("sd.updatetestfile", "partitions"); - //System.setProperty("sd.updatetestfile", "qr-logging"); - //System.setProperty("sd.updatetestfile", "qr-searchers"); - //System.setProperty("sd.updatetestfile", "rank-profiles"); - //System.setProperty("sd.updatetestfile", "summary"); - //System.setProperty("sd.updatetestfile", "summarymap"); - //System.setProperty("sd.updatetestfile", "translogserver"); - //System.setProperty("sd.updatetestfile", "vsmsummary"); - //System.setProperty("sd.updatetestfile", "vsmfields"); - } - - private static void setUpUpdateTest() { - try { - System.out.println("Property sd.updatetests is true, updating test files from generated ones..."); - System.out.println("Enter export test file name to be updated (eg. index-info), or blank to update every file. 'q' to quit: "); - String fileName = new BufferedReader(new InputStreamReader(System.in)).readLine(); - if ("q".equals(fileName)) { - throw new IllegalArgumentException("Aborted by user"); - } - System.setProperty("sd.updatetestfile", fileName); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } - } - protected DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException { File toDir = new File(tempDir + dirName); toDir.mkdirs(); @@ -158,22 +114,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase } } - protected void assertEqualConfig(String name, String config) throws IOException { - final String expectedConfigDirName = searchDefRoot + name + "/"; - assertEqualFiles(expectedConfigDirName + config + ".cfg", - tempDir + name + "/" + config + ".cfg"); - } - - @SuppressWarnings({ "ConstantConditions" }) public static void assertEqualFiles(String correctFileName, String checkFileName) throws IOException { - if (WRITE_FILES || "true".equals(System.getProperty("sd.updatetests"))) { - String updateFile = System.getProperty("sd.updatetestfile"); - if (WRITE_FILES || "".equals(updateFile) || correctFileName.endsWith(updateFile + ".cfg") || correctFileName.endsWith(updateFile + ".MODEL.cfg")) { - System.out.println("Copying " + checkFileName + " to " + correctFileName); - IOUtils.copy(checkFileName, correctFileName); - return; - } - } // Set updateOnAssert to true if you want update the files with correct answer. assertConfigFiles(correctFileName, checkFileName, false); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 91a89c204c9..3f45b28b994 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -25,20 +25,16 @@ public class ComplexAttributeFieldUtilsTestCase { return field; } - public SDDocumentType docType() { - return search.getDocument(); - } - public boolean isSupportedComplexField() { - return ComplexAttributeFieldUtils.isSupportedComplexField(field(), docType()); + return ComplexAttributeFieldUtils.isSupportedComplexField(field()); } public boolean isArrayOfSimpleStruct() { - return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field(), docType()); + return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field()); } public boolean isMapOfSimpleStruct() { - return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field(), docType()); + return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field()); } public boolean isMapOfPrimitiveType() { @@ -46,7 +42,7 @@ public class ComplexAttributeFieldUtilsTestCase { } public boolean isComplexFieldWithOnlyStructFieldAttributes() { - return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field(), docType()); + return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field()); } } @@ -57,7 +53,7 @@ public class ComplexAttributeFieldUtilsTestCase { " document test {", " struct elem {", " field name type string {}", - " field weight type string {}", + " field weight type int {}", " }", sdFieldContent, " }", @@ -72,7 +68,7 @@ public class ComplexAttributeFieldUtilsTestCase { " document test {", " struct elem {", " field name type string {}", - " field weight type array<string> {}", + " field weights type array<int> {}", " }", sdFieldContent, " }", @@ -194,7 +190,7 @@ public class ComplexAttributeFieldUtilsTestCase { ComplexFixture f = new ComplexFixture("elem_array", joinLines("field elem_array type array<elem> {", " struct-field name { indexing: attribute }", - " struct-field weight { indexing: attribute }", + " struct-field weights { indexing: attribute }", "}")); assertFalse(f.isSupportedComplexField()); assertFalse(f.isArrayOfSimpleStruct()); @@ -207,7 +203,7 @@ public class ComplexAttributeFieldUtilsTestCase { joinLines("field elem_map type map<int, elem> {", " indexing: summary", " struct-field key { indexing: attribute }", - " struct-field value.weight { indexing: attribute }", + " struct-field value.weights { indexing: attribute }", "}")); assertFalse(f.isSupportedComplexField()); assertFalse(f.isArrayOfSimpleStruct()); @@ -217,4 +213,32 @@ public class ComplexAttributeFieldUtilsTestCase { } } + @Test + public void only_struct_field_attributes_are_considered_when_tagging_a_complex_field() throws ParseException { + { + ComplexFixture f = new ComplexFixture("elem_array", + joinLines("field elem_array type array<elem> {", + " struct-field name { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + ComplexFixture f = new ComplexFixture("elem_map", + joinLines("field elem_map type map<int, elem> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 6483933385d..3ba3745f46e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -31,18 +31,42 @@ public class ComplexAttributeFieldsValidatorTestCase { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("For cluster 'mycluster', search 'test': " + "The following complex fields do not support using struct field attributes: " + - "struct_array (struct_array.s1), struct_map (struct_map.key, struct_map.value.s1). " + + "struct_array (struct_array.f1), struct_map (struct_map.key, struct_map.value.f1). " + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types"); createModelAndValidate(joinLines("search test {", " document test {", - " struct s { field s1 type array<int> {} }", + " struct s { field f1 type array<int> {} }", " field struct_array type array<s> {", - " struct-field s1 { indexing: attribute }", + " struct-field f1 { indexing: attribute }", " }", " field struct_map type map<string,s> {", " struct-field key { indexing: attribute }", - " struct-field value.s1 { indexing: attribute }", + " struct-field value.f1 { indexing: attribute }", + " }", + " }", + "}")); + } + + @Test + public void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException { + createModelAndValidate(joinLines("search test {", + " document test {", + " struct s1 {", + " field f1 type string {}", + " field f2 type int {}", + " }", + " struct s2 {", + " field f3 type string {}", + " field f4 type array<int> {}", + " field f5 type array<s1> {}", + " }", + " field struct_array type array<s2> {", + " struct-field f3 { indexing: attribute }", + " }", + " field struct_map type map<string,s2> {", + " struct-field key { indexing: attribute }", + " struct-field value.f3 { indexing: attribute }", " }", " }", "}")); |