diff options
author | Geir Storli <geirst@oath.com> | 2018-06-14 17:00:57 +0200 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-06-14 17:00:57 +0200 |
commit | d99c687658dfc89522d6af953fb74abfbc820078 (patch) | |
tree | a7e6d6c2011e930dabe2fed6df1bc4076522997c /config-model | |
parent | 27b64a81a6a133976bd17385eb150e34added12d (diff) |
Validate that complex fields that have struct field attributes are supported.
This only applies for indexed search clusters.
Diffstat (limited to 'config-model')
4 files changed, 160 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index 857959d0678..72eb1c96e0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -31,6 +31,12 @@ public class ComplexAttributeFieldUtils { isMapOfPrimitiveType(field)); } + public static boolean isSupportedComplexField(DataType fieldType) { + return (isArrayOfSimpleStruct(fieldType) || + isMapOfSimpleStruct(fieldType) || + isMapOfPrimitiveType(fieldType)); + } + public static boolean isArrayOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { return isArrayOfSimpleStruct(field.getDataType(), Optional.of(docType)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java new file mode 100644 index 00000000000..005c1dd8b2e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -0,0 +1,85 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; +import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Validates that complex fields (of type struct or map) that have struct field attributes are supported. + * + * Only applies for indexed search clusters. + * + * @author geirst + */ +public class ComplexAttributeFieldsValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + List<AbstractSearchCluster> searchClusters = model.getSearchClusters(); + for (AbstractSearchCluster cluster : searchClusters) { + if (cluster.isStreaming()) { + continue; + } + SearchCluster searchCluster = (SearchCluster) cluster; + for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); + } + } + } + + private static void validateComplexFields(String clusterName, Search search) { + String unsupportedFields = search.allFields() + .filter(field -> isUnsupportedComplexField(field)) + .map(ComplexAttributeFieldsValidator::toString) + .collect(Collectors.joining(", ")); + + if (!unsupportedFields.isEmpty()) { + throw new IllegalArgumentException( + String.format("For cluster '%s', search '%s': The following complex fields do not support using struct field attributes: %s. " + + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types", + clusterName, search.getName(), unsupportedFields)); + } + } + + private static boolean isUnsupportedComplexField(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isSupportedComplexField(field) && + hasStructFieldAttributes(field.getStructFields())); + } + + private static boolean isSupportedComplexField(ImmutableSDField field) { + return (ComplexAttributeFieldUtils.isSupportedComplexField(field.getDataType()) || + field.getDataType().equals(PositionDataType.INSTANCE) || + field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + } + + private static String toString(ImmutableSDField field) { + return field.getName() + " (" + StringUtils.join(getStructFieldAttributes(field.getStructFields()), ", ") + ")"; + } + + private static boolean hasStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { + return !getStructFieldAttributes(structFields).isEmpty(); + } + + private static List<String> getStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { + List<String> result = new ArrayList<>(); + for (ImmutableSDField structField : structFields) { + structField.getAttributes().values().forEach(attr -> result.add(attr.getName())); + result.addAll(getStructFieldAttributes(structField.getStructFields())); + } + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 523ced52306..c08e81b250f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -52,6 +52,7 @@ public class Validation { } new ComponentValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); + new ComplexAttributeFieldsValidator().validate(model, deployState); new StreamingValidator().validate(model, deployState); new RankSetupValidator(force).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java new file mode 100644 index 00000000000..146369d1620 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +/** + * @author geirst + */ +public class ComplexAttributeFieldsValidatorTestCase { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_when_unsupported_complex_fields_have_struct_field_attributes() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For cluster 'mycluster', search 'test': " + + "The following complex fields do not support using struct field attributes: " + + "struct_array (struct_array.s1), struct_map (struct_map.key, struct_map.value.s1). " + + "Only supported for the following complex field types: array or map of struct with primitive types, map of primitive types"); + + createModelAndValidate(joinLines("search test {", + " document test {", + " struct s { field s1 type array<int> {} }", + " field struct_array type array<s> {", + " struct-field s1 { indexing: attribute }", + " }", + " field struct_map type map<string,s> {", + " struct-field key { indexing: attribute }", + " struct-field value.s1 { indexing: attribute }", + " }", + " }", + "}")); + } + + private static void createModelAndValidate(String searchDefinition) throws IOException, SAXException { + DeployState deployState = createDeployState(servicesXml(), searchDefinition); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + Validation.validate(model, false, false, deployState); + } + + private static DeployState createDeployState(String servicesXml, String searchDefinition) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withSearchDefinition(searchDefinition) + .build(); + return new DeployState.Builder().applicationPackage(app).build(true); + } + + private static String servicesXml() { + return joinLines("<services version='1.0'>", + new ContentClusterBuilder().getXml(), + "</services>"); + } +} |