diff options
Diffstat (limited to 'config-model/src/main')
5 files changed, 136 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java index cfe33ef019f..82b56f9c961 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; @@ -38,9 +39,22 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod } } + private static boolean isNestedFieldName(String fieldName) { + return fieldName.indexOf('.') != -1; + } + private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) { - if (field.targetField().doesAttributing()) { - builder.attribute.add(createAttributeBuilder(field)); + ImmutableSDField targetField = field.targetField(); + String targetFieldName = targetField.getName(); + if (!isNestedFieldName(targetFieldName)) { + if (targetField.doesAttributing()) { + builder.attribute.add(createAttributeBuilder(field)); + } + } else { + Attribute attribute = targetField.getAttributes().get(targetFieldName); + if (attribute != null) { + builder.attribute.add(createAttributeBuilder(field)); + } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java index 2192a7e7bb1..7c67fc422d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java @@ -12,12 +12,15 @@ import java.util.Map; public class ImportedFields { private final Map<String, ImportedField> fields; + private final Map<String, ImportedField> complexFields; - public ImportedFields(Map<String, ImportedField> fields) { + public ImportedFields(Map<String, ImportedField> fields, Map<String, ImportedField> complexFields) { this.fields = fields; + this.complexFields = complexFields; } public Map<String, ImportedField> fields() { return Collections.unmodifiableMap(fields); } + public Map<String, ImportedField> complexFields() { return Collections.unmodifiableMap(complexFields); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index fefb54a7fe3..c10b52eb1e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.searchdefinition.document.ImmutableImportedSDField; import java.util.stream.Stream; @@ -31,6 +32,11 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); + search.importedFields().map(fields -> fields.complexFields().values().stream()). + orElse(Stream.empty()). + map(ImmutableImportedSDField::new). + flatMap(this::getSummaryFieldsForImportedField). + forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); } private Stream<SummaryField> getSummaryFieldsForImportedField(ImmutableSDField importedField) { @@ -40,4 +46,8 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { private static void setAttributeTransform(SummaryField summaryField) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } + + private static void setAttributeCombinerTransform(SummaryField summaryField) { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index a3efd086c6a..bfb2fa9c120 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -2,11 +2,15 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; @@ -17,6 +21,10 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + /** * Iterates all imported fields from SD-parsing and validates and resolves them into concrete fields from referenced document types. * @@ -25,6 +33,7 @@ import java.util.Optional; public class ImportedFieldsResolver extends Processor { private final Map<String, ImportedField> importedFields = new LinkedHashMap<>(); + private final Map<String, ImportedField> importedComplexFields = new LinkedHashMap<>(); private final Optional<DocumentReferences> references; public ImportedFieldsResolver(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { @@ -35,12 +44,86 @@ public class ImportedFieldsResolver extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); - search.setImportedFields(new ImportedFields(importedFields)); + search.setImportedFields(new ImportedFields(importedFields, importedComplexFields)); } private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { DocumentReference reference = validateDocumentReference(importedField); - ImmutableSDField targetField = validateTargetField(importedField, reference, validate); + ImmutableSDField targetField = getTargetField(importedField, reference); + if (isArrayOfSimpleStruct(targetField)) { + resolveImportedArrayOfStructField(importedField, reference, targetField, validate); + } else if (isMapOfSimpleStruct(targetField)) { + resolveImportedMapOfStructField(importedField, reference, targetField, validate); + } else if (isMapOfPrimitiveType(targetField)) { + resolveImportedMapOfPrimitiveField(importedField, reference, targetField, validate); + } else { + resolveImportedNormalField(importedField, reference, targetField, validate); + } + } + + private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + resolveImportedNestedStructField(importedField, reference, targetField, validate); + makeImportedComplexField(importedField, reference, targetField); + } + + private void resolveImportedMapOfStructField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + resolveImportedNestedField(importedField, reference, targetField.getStructField("key"), validate); + resolveImportedNestedStructField(importedField, reference, targetField.getStructField("value"), validate); + makeImportedComplexField(importedField, reference, targetField); + } + + private void makeImportedComplexField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField) { + String name = importedField.fieldName(); + importedComplexFields.put(name, new ImportedField(name, reference, targetField)); + } + + private static String makeImportedNestedFieldName(TemporaryImportedField importedField, ImmutableSDField targetNestedField) { + return importedField.fieldName() + targetNestedField.getName().substring(importedField.targetFieldName().length()); + } + + private boolean resolveImportedNestedField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetNestedField, boolean requireAttribute) { + Attribute attribute = targetNestedField.getAttributes().get(targetNestedField.getName()); + String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); + if (attribute != null) { + importedFields.put(importedNestedFieldName, new ImportedField(importedNestedFieldName, reference, targetNestedField)); + } else if (requireAttribute) { + fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + + ": Is not an attribute field. Only attribute fields supported"); + } + return attribute != null; + } + + private void resolveImportedNestedStructField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetNestedField, boolean validate) { + boolean foundAttribute = false; + for (ImmutableSDField targetStructField : targetNestedField.getStructFields()) { + if (resolveImportedNestedField(importedField, reference, targetStructField, false)) { + foundAttribute = true; + }; + } + if (validate && !foundAttribute) { + String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); + fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + + ": Is not a struct containing an attribute field."); + } + } + + private void resolveImportedMapOfPrimitiveField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + resolveImportedNestedField(importedField, reference, targetField.getStructField("key"), validate); + resolveImportedNestedField(importedField, reference, targetField.getStructField("value"), validate); + makeImportedComplexField(importedField, reference, targetField); + } + + private void resolveImportedNormalField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + if (validate) { + validateTargetField(importedField, targetField, reference); + } importedFields.put(importedField.fieldName(), new ImportedField(importedField.fieldName(), reference, targetField)); } @@ -53,30 +136,31 @@ public class ImportedFieldsResolver extends Processor { return reference; } - private ImmutableSDField validateTargetField(TemporaryImportedField importedField, - DocumentReference reference, - boolean validate) { + private ImmutableSDField getTargetField(TemporaryImportedField importedField, + DocumentReference reference) { String targetFieldName = importedField.targetFieldName(); Search targetSearch = reference.targetSearch(); ImmutableSDField targetField = targetSearch.getField(targetFieldName); if (targetField == null) { fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Not found"); } - if (validate) { - if (!targetField.doesAttributing()) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + - ": Is not an attribute field. Only attribute fields supported"); - } else if (targetField.doesIndexing()) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + - ": Is an index field. Not supported"); - } else if (targetField.getDataType().equals(DataType.PREDICATE)) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + - ": Is of type 'predicate'. Not supported"); - } - } return targetField; } + private void validateTargetField(TemporaryImportedField importedField, + ImmutableSDField targetField, DocumentReference reference) { + if (!targetField.doesAttributing()) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is not an attribute field. Only attribute fields supported"); + } else if (targetField.doesIndexing()) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is an index field. Not supported"); + } else if (targetField.getDataType().equals(DataType.PREDICATE)) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is of type 'predicate'. Not supported"); + } + } + private static String targetFieldAsString(String targetFieldName, DocumentReference reference) { return "Field '" + targetFieldName + "' via reference field '" + reference.referenceField().getName() + "'"; } @@ -85,4 +169,8 @@ public class ImportedFieldsResolver extends Processor { throw new IllegalArgumentException("For search '" + search.getName() + "', import field '" + importedField.fieldName() + "': " + msg); } + private void fail(TemporaryImportedField importedField, String importedNestedFieldName, String msg) { + throw new IllegalArgumentException("For search '" + search.getName() + "', import field '" + + importedField.fieldName() + "' (nested to '" + importedNestedFieldName + "'): " + msg); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index c87801685bb..008b3182d8f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -51,6 +51,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { return isDocumentField(source) || (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) || (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) || + (search.importedFields().map(fields -> fields.complexFields().get(source) != null).orElse(false)) || (SummaryClass.DOCUMENT_ID_FIELD.equals(source)); } |