diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-01-22 14:00:56 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-01-31 11:33:04 +0100 |
commit | 228d880395e9ae07e9e55ca315f381a1de91cc15 (patch) | |
tree | 22106e62dd47981a81ce5b9003b24962407ba622 /config-model/src/main/java/com/yahoo | |
parent | dec299cb5de9111f29ae835cf8e7dbc6e0f88a4d (diff) |
Add imported fields to document configs
This adds the field names of all imported fields in a search definition
to the document configs associated with it (document manager and document
types config for Java and C++, respectively). This enables consumers of
these configs to distinguish between field references to imported fields
and fields that don't exist in the document schema itself.
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
8 files changed, 98 insertions, 9 deletions
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set<String> importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set<Name> documentReferences) { + public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set<Name> documentReferences) { + Set<Name> documentReferences, + Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..c736ca8e9dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,32 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; + +import java.util.Collections; +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List<Search> searchDefinitions; + + public ImportedFieldsEnumerator(List<Search> searchDefinitions) { + this.searchDefinitions = searchDefinitions; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.searchDefinitions.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, |