From 228d880395e9ae07e9e55ca315f381a1de91cc15 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 22 Jan 2020 14:00:56 +0100 Subject: Add imported fields to document configs This adds the field names of all imported fields in a search definition to the document configs associated with it (document manager and document types config for Java and C++, respectively). This enables consumers of these configs to distinguish between field references to imported fields and fields that don't exist in the document schema itself. --- .../com/yahoo/documentmodel/NewDocumentType.java | 23 +++- .../searchdefinition/DocumentModelBuilder.java | 12 +- .../searchdefinition/ImportedFieldsEnumerator.java | 32 +++++ .../com/yahoo/searchdefinition/SearchBuilder.java | 8 +- .../searchdefinition/document/SDDocumentType.java | 9 ++ .../processing/ImportedFieldsResolver.java | 4 +- .../configmodel/producers/DocumentManager.java | 10 ++ .../vespa/configmodel/producers/DocumentTypes.java | 9 ++ .../documentmanager_multiple_imported_fields.cfg | 104 +++++++++++++++ .../documenttypes_multiple_imported_fields.cfg | 141 +++++++++++++++++++++ .../com/yahoo/searchdefinition/AttributeUtils.java | 14 ++ .../DocumentReferenceResolverTest.java | 8 +- .../ImportedFieldsEnumeratorTest.java | 65 ++++++++++ .../AbstractReferenceFieldTestCase.java | 34 +++++ ...DocumentModelBuilderImportedFieldsTestCase.java | 54 ++++++++ .../DocumentModelBuilderReferenceTypeTestCase.java | 25 +--- .../search/DocumentTypeChangeValidatorTest.java | 1 + document/abi-spec.json | 38 +++++- .../src/vespa/document/config/documentmanager.def | 3 + .../src/vespa/document/config/documenttypes.def | 2 + 20 files changed, 555 insertions(+), 41 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java create mode 100644 config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg create mode 100644 config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg create mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java create mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set
fieldSets = new LinkedHashSet<>(); private final Set documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set documentReferences) { + public NewDocumentType(Name name, Set documentReferences, Set importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set documentReferences) { + Set documentReferences, + Set importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..c736ca8e9dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,32 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; + +import java.util.Collections; +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List searchDefinitions; + + public ImportedFieldsEnumerator(List searchDefinitions) { + this.searchDefinitions = searchDefinitions; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.searchDefinitions.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7b50176625d --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -0,0 +1,104 @@ +enablecompression false +datatype[0].id 1381038251 +datatype[0].structtype[0].name "position" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].compresstype NONE +datatype[0].structtype[0].compresslevel 0 +datatype[0].structtype[0].compressthreshold 95 +datatype[0].structtype[0].compressminsize 800 +datatype[0].structtype[0].field[0].name "x" +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" +datatype[0].structtype[0].field[1].name "y" +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" +datatype[1].id 595216861 +datatype[1].referencetype[0].target_type_id -1318255918 +datatype[2].id 542332920 +datatype[2].referencetype[0].target_type_id 443162583 +datatype[3].id 959075962 +datatype[3].structtype[0].name "ad.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "campaign_ref" +datatype[3].structtype[0].field[0].datatype 595216861 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "person_ref" +datatype[3].structtype[0].field[1].datatype 542332920 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[4].id -255288561 +datatype[4].structtype[0].name "ad.body" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id 2987301 +datatype[5].documenttype[0].name "ad" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct 959075962 +datatype[5].documenttype[0].bodystruct -255288561 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].documenttype[0].importedfield[0].name "my_cool_field" +datatype[5].documenttype[0].importedfield[1].name "my_swag_field" +datatype[5].documenttype[0].importedfield[2].name "my_name" +datatype[6].id -2041471955 +datatype[6].structtype[0].name "campaign.header" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].compresstype NONE +datatype[6].structtype[0].compresslevel 0 +datatype[6].structtype[0].compressthreshold 95 +datatype[6].structtype[0].compressminsize 800 +datatype[6].structtype[0].field[0].name "cool_field" +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].detailedtype "" +datatype[6].structtype[0].field[1].name "swag_field" +datatype[6].structtype[0].field[1].datatype 4 +datatype[6].structtype[0].field[1].detailedtype "" +datatype[7].id 1448849794 +datatype[7].structtype[0].name "campaign.body" +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].compresstype NONE +datatype[7].structtype[0].compresslevel 0 +datatype[7].structtype[0].compressthreshold 95 +datatype[7].structtype[0].compressminsize 800 +datatype[8].id -1318255918 +datatype[8].documenttype[0].name "campaign" +datatype[8].documenttype[0].version 0 +datatype[8].documenttype[0].inherits[0].name "document" +datatype[8].documenttype[0].inherits[0].version 0 +datatype[8].documenttype[0].headerstruct -2041471955 +datatype[8].documenttype[0].bodystruct 1448849794 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[9].id 3129224 +datatype[9].structtype[0].name "person.header" +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].compresstype NONE +datatype[9].structtype[0].compresslevel 0 +datatype[9].structtype[0].compressthreshold 95 +datatype[9].structtype[0].compressminsize 800 +datatype[9].structtype[0].field[0].name "name" +datatype[9].structtype[0].field[0].datatype 2 +datatype[9].structtype[0].field[0].detailedtype "" +datatype[10].id -2003767395 +datatype[10].structtype[0].name "person.body" +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].compresstype NONE +datatype[10].structtype[0].compresslevel 0 +datatype[10].structtype[0].compressthreshold 95 +datatype[10].structtype[0].compressminsize 800 +datatype[11].id 443162583 +datatype[11].documenttype[0].name "person" +datatype[11].documenttype[0].version 0 +datatype[11].documenttype[0].inherits[0].name "document" +datatype[11].documenttype[0].inherits[0].version 0 +datatype[11].documenttype[0].headerstruct 3129224 +datatype[11].documenttype[0].bodystruct -2003767395 +datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name" \ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7859703ffe0 --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -0,0 +1,141 @@ +enablecompression false +documenttype[0].id 2987301 +documenttype[0].name "ad" +documenttype[0].version 0 +documenttype[0].headerstruct 959075962 +documenttype[0].bodystruct -255288561 +documenttype[0].inherits[0].id 8 +documenttype[0].datatype[0].id 959075962 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ad.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 95 +documenttype[0].datatype[0].sstruct.compression.minsize 200 +documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" +documenttype[0].datatype[0].sstruct.field[0].id 23963250 +documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" +documenttype[0].datatype[0].sstruct.field[1].name "person_ref" +documenttype[0].datatype[0].sstruct.field[1].id 100779805 +documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" +documenttype[0].datatype[1].id -255288561 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "ad.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 95 +documenttype[0].datatype[1].sstruct.compression.minsize 200 +documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +documenttype[0].referencetype[0].id 595216861 +documenttype[0].referencetype[0].target_type_id -1318255918 +documenttype[0].referencetype[1].id 542332920 +documenttype[0].referencetype[1].target_type_id 443162583 +documenttype[0].importedfield[0].name "my_cool_field" +documenttype[0].importedfield[1].name "my_swag_field" +documenttype[0].importedfield[2].name "my_name" +documenttype[1].id -1318255918 +documenttype[1].name "campaign" +documenttype[1].version 0 +documenttype[1].headerstruct -2041471955 +documenttype[1].bodystruct 1448849794 +documenttype[1].inherits[0].id 8 +documenttype[1].datatype[0].id -2041471955 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "campaign.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 95 +documenttype[1].datatype[0].sstruct.compression.minsize 200 +documenttype[1].datatype[0].sstruct.field[0].name "cool_field" +documenttype[1].datatype[0].sstruct.field[0].id 1588702436 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" +documenttype[1].datatype[0].sstruct.field[1].name "swag_field" +documenttype[1].datatype[0].sstruct.field[1].id 1691224741 +documenttype[1].datatype[0].sstruct.field[1].datatype 4 +documenttype[1].datatype[0].sstruct.field[1].detailedtype "" +documenttype[1].datatype[1].id 1448849794 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "campaign.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 95 +documenttype[1].datatype[1].sstruct.compression.minsize 200 +documenttype[1].fieldsets{[document]}.fields[0] "cool_field" +documenttype[1].fieldsets{[document]}.fields[1] "swag_field" +documenttype[2].id 443162583 +documenttype[2].name "person" +documenttype[2].version 0 +documenttype[2].headerstruct 3129224 +documenttype[2].bodystruct -2003767395 +documenttype[2].inherits[0].id 8 +documenttype[2].datatype[0].id 3129224 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "person.header" +documenttype[2].datatype[0].sstruct.version 0 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 95 +documenttype[2].datatype[0].sstruct.compression.minsize 200 +documenttype[2].datatype[0].sstruct.field[0].name "name" +documenttype[2].datatype[0].sstruct.field[0].id 1160796772 +documenttype[2].datatype[0].sstruct.field[0].datatype 2 +documenttype[2].datatype[0].sstruct.field[0].detailedtype "" +documenttype[2].datatype[1].id -2003767395 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "person.body" +documenttype[2].datatype[1].sstruct.version 0 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 95 +documenttype[2].datatype[1].sstruct.compression.minsize 200 +documenttype[2].fieldsets{[document]}.fields[0] "name" \ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java new file mode 100644 index 00000000000..6ce5a430e92 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java @@ -0,0 +1,14 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 8378ec811a5..e46208c770d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to bar and add another field SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); @@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to non-existing document bar SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); fooDocument.addField(fooRefToBarField); @@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest { resolver.resolveReferences(fooDocument); } - private static void addAttributeAspect(SDField fooRefToBarField) { - fooRefToBarField.parseIndexingScript("{ attribute }"); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..d63f3dd903b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,65 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + Search parentSearch = new Search(); + SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch); + var parentField = new SDField("their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSearch.addDocument(parentDocument); + + Search fooSearch = new Search(); + SDField fooRefToParent = new SDField( + "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + var fooImports = fooSearch.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); + fooSearch.addDocument(fooDocument); + + Search barSearch = new Search(); + SDField barRefToParent = new SDField( + "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + var barImports = barSearch.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType("bar", barSearch); + barSearch.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java new file mode 100644 index 00000000000..b6498b48928 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -0,0 +1,34 @@ +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; + +/** + * Utility functions for testing generated configs for reference/imported fields. + */ +public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase { + + private static String TEST_FOLDER = "src/test/configmodel/types/references/"; + + protected void assertDocumentConfigs(DocumentModel model, + String cfgFileSpec) throws IOException { + assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); + assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); + } + + protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + + protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java new file mode 100644 index 00000000000..3faf4118b8c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.documentmodel; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase { + + @Test + public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException { + assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference { indexing: attribute }", + " field person_ref type reference { indexing: attribute }", + " }", + " import field campaign_ref.cool_field as my_cool_field {}", + " import field campaign_ref.swag_field as my_swag_field {}", + " import field person_ref.name as my_name {}", + "}")), + "multiple_imported_fields"); + } + + private static class TestDocumentModelBuilder { + private final SearchBuilder builder = new SearchBuilder(); + public TestDocumentModelBuilder addCampaign() throws ParseException { + builder.importString(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); + return this; + } + public TestDocumentModelBuilder addPerson() throws ParseException { + builder.importString(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + return this; + } + public DocumentModel build(String adSdContent) throws ParseException { + builder.importString(adSdContent); + builder.build(); + return builder.getModel(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index e9a7a6ed33e..55980ee5fea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -1,15 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; import java.io.IOException; @@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals; /** * @author geirst */ -public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase { @Test public void reference_fields_can_reference_other_document_types() throws ParseException, IOException { @@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT assertEquals(campaignRefType.getTargetType(), campaignType); } - private static String TEST_FOLDER = "src/test/configmodel/types/references/"; - - private void assertDocumentConfigs(DocumentModel model, - String cfgFileSpec) throws IOException { - assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); - assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); - } - - private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { - DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); - assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); - } - - private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { - DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); - assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); - } - private static class TestDocumentModelBuilder { private final SearchBuilder builder = new SearchBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4064e53dfb7..3dfcef70aba 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest { headerfields, new StructDataType("bodyfields"), new FieldSets(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/document/abi-spec.json b/document/abi-spec.json index e4917a174a6..717b0c9f188 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -712,6 +712,8 @@ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)", "public com.yahoo.document.DocumenttypesConfig$Documenttype build()" ], "fields": [ @@ -719,7 +721,8 @@ "public java.util.List datatype", "public java.util.List annotationtype", "public java.util.Map fieldsets", - "public java.util.List referencetype" + "public java.util.List referencetype", + "public java.util.List importedfield" ] }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": { @@ -1264,6 +1267,35 @@ ], "fields": [] }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void ()", + "public void (com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()" + ], + "fields": [] + }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void (com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public java.lang.String name()" + ], + "fields": [] + }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -1347,7 +1379,9 @@ "public java.util.Map fieldsets()", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)", "public java.util.List referencetype()", - "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)" + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)", + "public java.util.List importedfield()", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)" ], "fields": [] }, diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 1961f67d83c..092a29d9293 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int ## Field sets datatype[].documenttype[].fieldsets{}.fields[] string +## Imported fields (specified outside the document block in the schema) +datatype[].documenttype[].importedfield[].name string + ## Cross-document reference with ID of target document type datatype[].referencetype[].target_type_id int diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index fb3fcd8b123..a9ebde90bb6 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -104,3 +104,5 @@ documenttype[].referencetype[].id int ## Numeric ID of the document type instances of the reference point to. documenttype[].referencetype[].target_type_id int + +documenttype[].importedfield[].name string -- cgit v1.2.3