Merge pull request #12022 from vespa-engine/vekterli/support-imported-fields-in-gc-selections

Support using imported fields in document selection expressions

17 files changed, 519 insertions, 39 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index fc42864f1d0..41a30c4553d 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
     private final StructDataType body;
     private final Set<FieldSet> fieldSets = new LinkedHashSet<>();
     private final Set<Name> documentReferences;
+    // Imported fields are virtual and therefore exist outside of the SD's document field definition
+    // block itself. But for features like imported fields in a non-search context (e.g. GC selections)
+    // it is necessary to know that certain identifiers refer to imported fields instead of being unknown
+    // document fields. To achieve this, we track the names of imported fields as part of the document
+    // config itself.
+    private final Set<String> importedFieldNames;
 
     public NewDocumentType(Name name) {
         this(name, emptySet());
     }
 
-    public NewDocumentType(Name name, Set<Name> documentReferences) {
+    public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) {
         this(
                 name,
                 new StructDataType(name.getName() + ".header"),
                 new StructDataType(name.getName() + ".body"),
                 new FieldSets(),
-                documentReferences);
+                documentReferences,
+                importedFieldNames);
+    }
+
+    public NewDocumentType(Name name, Set<Name> documentReferences) {
+        this(name, documentReferences, emptySet());
     }
 
     public NewDocumentType(Name name,
                            StructDataType header,
                            StructDataType body,
                            FieldSets fs,
-                           Set<Name> documentReferences) {
+                           Set<Name> documentReferences,
+                           Set<String> importedFieldNames) {
         super(name.getName());
         this.name = name;
         this.header = header;
@@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
             }
         }
         this.documentReferences = documentReferences;
+        this.importedFieldNames = importedFieldNames;
     }
 
     public Name getFullName() {
@@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
         return documentReferences;
     }
 
+    public Set<String> getImportedFieldNames() {
+        return importedFieldNames;
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index f0b1b427531..d3a78321106 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
 import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
 import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
 import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -338,7 +340,8 @@ public class DocumentModelBuilder {
                                                  sdoc.getDocumentType().contentStruct(),
                                                  sdoc.getDocumentType().getBodyType(),
                                                  sdoc.getFieldSets(),
-                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()));
+                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()),
+                                                 convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields()));
         for (SDDocumentType n : sdoc.getInheritedTypes()) {
             NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
                 NewDocumentType inherited =  model.getDocumentManager().getDocumentType(name);
@@ -404,6 +407,13 @@ public class DocumentModelBuilder {
                 .collect(toSet());
     }
 
+    private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) {
+        if (importedFields == null) {
+            return emptySet();
+        }
+        return Collections.unmodifiableSet(importedFields.fields().keySet());
+    }
+
     private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
         for (Field f : fields) {
             DataType type = f.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
new file mode 100644
index 00000000000..ee4fc41f2f9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
@@ -0,0 +1,33 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Enumerates and emplaces a set of all imported fields into a SDDocumentType from
+ * its corresponding Search instance.
+ */
+public class ImportedFieldsEnumerator {
+
+    private final List<Search> searchDefinitions;
+
+    public ImportedFieldsEnumerator(List<Search> searchDefinitions) {
+        this.searchDefinitions = searchDefinitions;
+    }
+
+    public void enumerateImportedFields(SDDocumentType documentType) {
+        var search = this.searchDefinitions.stream()
+                .filter(s -> s.getDocument() != null)
+                .filter(s -> s.getDocument().getName().equals(documentType.getName()))
+                .findFirst();
+        if (search.isEmpty()) {
+            return; // No imported fields present.
+        }
+        search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 949539ff99f..eb68e6af203 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -230,20 +230,22 @@ public class SearchBuilder {
                 sdocs.add(search.getDocument());
             }
         }
-        SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+        var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
         orderer.process();
         for (SDDocumentType sdoc : orderer.getOrdered()) {
             new FieldOperationApplierForStructs().process(sdoc);
             new FieldOperationApplier().process(sdoc);
         }
 
-        DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList);
+        var resolver = new DocumentReferenceResolver(searchList);
         sdocs.forEach(resolver::resolveReferences);
+        var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
+        sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
 
         if (validate)
             new DocumentGraphValidator().validateDocumentGraph(sdocs);
 
-        DocumentModelBuilder builder = new DocumentModelBuilder(model);
+        var builder = new DocumentModelBuilder(model);
         for (Search search : new SearchOrderer().order(searchList)) {
             new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
             process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index 414a605c621..b3f98cc6f26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable {
     private FieldSets fieldSets;
     // Document references
     private Optional<DocumentReferences> documentReferences = Optional.empty();
+    private TemporaryImportedFields temporaryImportedFields;
 
     static {
         VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
@@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable {
         type.docType = docType.clone();
         type.inheritedTypes.putAll(inheritedTypes);
         type.structType = structType;
+        // TODO this isn't complete; should it be..?!
         return type;
     }
 
@@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable {
         this.documentReferences = Optional.of(documentReferences);
     }
 
+    public TemporaryImportedFields getTemporaryImportedFields() {
+        return temporaryImportedFields;
+    }
+
+    public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) {
+        this.temporaryImportedFields = temporaryImportedFields;
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index d6c334ee80b..6d3de23238d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
-                                                   ImmutableSDField targetField, boolean validate) {
+                                              ImmutableSDField targetField, boolean validate) {
         TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
                 reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
         ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
@@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void validateTargetField(TemporaryImportedField importedField,
-                                                 ImmutableSDField targetField, DocumentReference reference) {
+                                     ImmutableSDField targetField, DocumentReference reference) {
         if (!targetField.doesAttributing()) {
             fail(importedField, targetFieldAsString(targetField.getName(), reference) +
                     ": Is not an attribute field. Only attribute fields supported");
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 02d500931d7..e0d015cc8b2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -62,6 +62,7 @@ public class DocumentManager {
             }
         }
     }
+
     private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
         for (AnnotationType type : types) {
             DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
@@ -110,6 +111,7 @@ public class DocumentManager {
                 doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
             }
             buildConfig(dt.getFieldSets(), doc);
+            buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
         } else if (type instanceof TemporaryStructuredDataType) {
             //Ignored
         } else if (type instanceof StructDataType) {
@@ -164,4 +166,12 @@ public class DocumentManager {
         doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
 
+    private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) {
+        for (String fieldName : fieldNames) {
+            var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder();
+            ib.name(fieldName);
+            builder.importedfield(ib);
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index d4228b52746..e6bf826dccc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -58,6 +58,7 @@ public class DocumentTypes {
             buildConfig(annotation, atb);
         }
         buildConfig(documentType.getFieldSets(), db);
+        buildImportedFieldsConfig(documentType.getImportedFieldNames(), db);
         builder.documenttype(db);
     }
 
@@ -120,6 +121,14 @@ public class DocumentTypes {
         }
     }
 
+    private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder  builder) {
+        for (String fieldName : fieldNames) {
+            var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder();
+            ib.name(fieldName);
+            builder.importedfield(ib);
+        }
+    }
+
     private void buildConfig(StructDataType type,
                              DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
                              DocumenttypesConfig.Documenttype.Builder documentBuilder,
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7b50176625d
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
@@ -0,0 +1,104 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 542332920
+datatype[2].referencetype[0].target_type_id 443162583
+datatype[3].id 959075962
+datatype[3].structtype[0].name "ad.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "campaign_ref"
+datatype[3].structtype[0].field[0].datatype 595216861
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "person_ref"
+datatype[3].structtype[0].field[1].datatype 542332920
+datatype[3].structtype[0].field[1].detailedtype ""
+datatype[4].id -255288561
+datatype[4].structtype[0].name "ad.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[5].id 2987301
+datatype[5].documenttype[0].name "ad"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0].name "document"
+datatype[5].documenttype[0].inherits[0].version 0
+datatype[5].documenttype[0].headerstruct 959075962
+datatype[5].documenttype[0].bodystruct -255288561
+datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+datatype[5].documenttype[0].importedfield[0].name "my_cool_field"
+datatype[5].documenttype[0].importedfield[1].name "my_swag_field"
+datatype[5].documenttype[0].importedfield[2].name "my_name"
+datatype[6].id -2041471955
+datatype[6].structtype[0].name "campaign.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "cool_field"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].detailedtype ""
+datatype[6].structtype[0].field[1].name "swag_field"
+datatype[6].structtype[0].field[1].datatype 4
+datatype[6].structtype[0].field[1].detailedtype ""
+datatype[7].id 1448849794
+datatype[7].structtype[0].name "campaign.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[8].id -1318255918
+datatype[8].documenttype[0].name "campaign"
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0].name "document"
+datatype[8].documenttype[0].inherits[0].version 0
+datatype[8].documenttype[0].headerstruct -2041471955
+datatype[8].documenttype[0].bodystruct 1448849794
+datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field"
+datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field"
+datatype[9].id 3129224
+datatype[9].structtype[0].name "person.header"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "name"
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].detailedtype ""
+datatype[10].id -2003767395
+datatype[10].structtype[0].name "person.body"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].compresstype NONE
+datatype[10].structtype[0].compresslevel 0
+datatype[10].structtype[0].compressthreshold 95
+datatype[10].structtype[0].compressminsize 800
+datatype[11].id 443162583
+datatype[11].documenttype[0].name "person"
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0].name "document"
+datatype[11].documenttype[0].inherits[0].version 0
+datatype[11].documenttype[0].headerstruct 3129224
+datatype[11].documenttype[0].bodystruct -2003767395
+datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file
diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7859703ffe0
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
@@ -0,0 +1,141 @@
+enablecompression false
+documenttype[0].id 2987301
+documenttype[0].name "ad"
+documenttype[0].version 0
+documenttype[0].headerstruct 959075962
+documenttype[0].bodystruct -255288561
+documenttype[0].inherits[0].id 8
+documenttype[0].datatype[0].id 959075962
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "ad.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref"
+documenttype[0].datatype[0].sstruct.field[0].id 23963250
+documenttype[0].datatype[0].sstruct.field[0].datatype 595216861
+documenttype[0].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[0].datatype[0].sstruct.field[1].name "person_ref"
+documenttype[0].datatype[0].sstruct.field[1].id 100779805
+documenttype[0].datatype[0].sstruct.field[1].datatype 542332920
+documenttype[0].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[0].datatype[1].id -255288561
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "ad.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+documenttype[0].referencetype[0].id 595216861
+documenttype[0].referencetype[0].target_type_id -1318255918
+documenttype[0].referencetype[1].id 542332920
+documenttype[0].referencetype[1].target_type_id 443162583
+documenttype[0].importedfield[0].name "my_cool_field"
+documenttype[0].importedfield[1].name "my_swag_field"
+documenttype[0].importedfield[2].name "my_name"
+documenttype[1].id -1318255918
+documenttype[1].name "campaign"
+documenttype[1].version 0
+documenttype[1].headerstruct -2041471955
+documenttype[1].bodystruct 1448849794
+documenttype[1].inherits[0].id 8
+documenttype[1].datatype[0].id -2041471955
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "campaign.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 95
+documenttype[1].datatype[0].sstruct.compression.minsize 200
+documenttype[1].datatype[0].sstruct.field[0].name "cool_field"
+documenttype[1].datatype[0].sstruct.field[0].id 1588702436
+documenttype[1].datatype[0].sstruct.field[0].datatype 2
+documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[1].datatype[0].sstruct.field[1].name "swag_field"
+documenttype[1].datatype[0].sstruct.field[1].id 1691224741
+documenttype[1].datatype[0].sstruct.field[1].datatype 4
+documenttype[1].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[1].datatype[1].id 1448849794
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "campaign.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 95
+documenttype[1].datatype[1].sstruct.compression.minsize 200
+documenttype[1].fieldsets{[document]}.fields[0] "cool_field"
+documenttype[1].fieldsets{[document]}.fields[1] "swag_field"
+documenttype[2].id 443162583
+documenttype[2].name "person"
+documenttype[2].version 0
+documenttype[2].headerstruct 3129224
+documenttype[2].bodystruct -2003767395
+documenttype[2].inherits[0].id 8
+documenttype[2].datatype[0].id 3129224
+documenttype[2].datatype[0].type STRUCT
+documenttype[2].datatype[0].array.element.id 0
+documenttype[2].datatype[0].map.key.id 0
+documenttype[2].datatype[0].map.value.id 0
+documenttype[2].datatype[0].wset.key.id 0
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].annotationref.annotation.id 0
+documenttype[2].datatype[0].sstruct.name "person.header"
+documenttype[2].datatype[0].sstruct.version 0
+documenttype[2].datatype[0].sstruct.compression.type NONE
+documenttype[2].datatype[0].sstruct.compression.level 0
+documenttype[2].datatype[0].sstruct.compression.threshold 95
+documenttype[2].datatype[0].sstruct.compression.minsize 200
+documenttype[2].datatype[0].sstruct.field[0].name "name"
+documenttype[2].datatype[0].sstruct.field[0].id 1160796772
+documenttype[2].datatype[0].sstruct.field[0].datatype 2
+documenttype[2].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[2].datatype[1].id -2003767395
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].array.element.id 0
+documenttype[2].datatype[1].map.key.id 0
+documenttype[2].datatype[1].map.value.id 0
+documenttype[2].datatype[1].wset.key.id 0
+documenttype[2].datatype[1].wset.createifnonexistent false
+documenttype[2].datatype[1].wset.removeifzero false
+documenttype[2].datatype[1].annotationref.annotation.id 0
+documenttype[2].datatype[1].sstruct.name "person.body"
+documenttype[2].datatype[1].sstruct.version 0
+documenttype[2].datatype[1].sstruct.compression.type NONE
+documenttype[2].datatype[1].sstruct.compression.level 0
+documenttype[2].datatype[1].sstruct.compression.threshold 95
+documenttype[2].datatype[1].sstruct.compression.minsize 200
+documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
new file mode 100644
index 00000000000..2c13427760f
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
@@ -0,0 +1,15 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * Convenience class for tests that need to set attribute properties on fields.
+ */
+public class AttributeUtils {
+
+    public static void addAttributeAspect(SDField field) {
+        field.parseIndexingScript("{ attribute }");
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index 8378ec811a5..e46208c770d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest {
         // Create foo document with document reference to bar and add another field
         SDField fooRefToBarField = new SDField
                 ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
-        addAttributeAspect(fooRefToBarField);
+        AttributeUtils.addAttributeAspect(fooRefToBarField);
         SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
         Search fooSearch = new Search();
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
@@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest {
         // Create foo document with document reference to non-existing document bar
         SDField fooRefToBarField = new SDField(
                 "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
-        addAttributeAspect(fooRefToBarField);
+        AttributeUtils.addAttributeAspect(fooRefToBarField);
         Search fooSearch = new Search();
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
         fooDocument.addField(fooRefToBarField);
@@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest {
         resolver.resolveReferences(fooDocument);
     }
 
-    private static void addAttributeAspect(SDField fooRefToBarField) {
-        fooRefToBarField.parseIndexingScript("{ attribute }");
-    }
-
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
new file mode 100644
index 00000000000..fcbb89b5c42
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -0,0 +1,66 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedField;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ImportedFieldsEnumeratorTest {
+
+    @Test
+    public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() {
+        Search parentSearch = new Search();
+        SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch);
+        var parentField = new SDField("their_field", DataType.INT);
+        AttributeUtils.addAttributeAspect(parentField);
+        parentDocument.addField(parentField);
+        parentSearch.addDocument(parentDocument);
+
+        Search fooSearch = new Search();
+        SDField fooRefToParent = new SDField(
+                "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+        AttributeUtils.addAttributeAspect(fooRefToParent);
+        var fooImports = fooSearch.temporaryImportedFields().get();
+        fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
+        fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
+        SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
+        fooSearch.addDocument(fooDocument);
+
+        Search barSearch = new Search();
+        SDField barRefToParent = new SDField(
+                "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+        AttributeUtils.addAttributeAspect(barRefToParent);
+        var barImports = barSearch.temporaryImportedFields().get();
+        barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
+        SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+        barSearch.addDocument(barDocument);
+
+        var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch));
+
+        enumerator.enumerateImportedFields(parentDocument);
+        assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent
+
+        enumerator.enumerateImportedFields(fooDocument);
+        assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import"));
+
+        enumerator.enumerateImportedFields(barDocument);
+        assertImportedFieldsAre(barDocument, List.of("my_cool_import"));
+    }
+
+    private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) {
+        assertNotNull(documentType.getTemporaryImportedFields());
+        var actualNames = documentType.getTemporaryImportedFields().fields().keySet();
+        var expectedNameSet = new HashSet<>(expectedNames);
+        assertEquals(expectedNameSet, actualNames);
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
new file mode 100644
index 00000000000..d0ee0523489
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+
+import java.io.IOException;
+
+/**
+ * Utility functions for testing generated configs for reference/imported fields.
+ */
+public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase {
+
+    private static String TEST_FOLDER = "src/test/configmodel/types/references/";
+
+    protected void assertDocumentConfigs(DocumentModel model,
+                                         String cfgFileSpec) throws IOException {
+        assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
+        assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
+    }
+
+    protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
+        DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
+        assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
+    }
+
+    protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
+        DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
+        assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
new file mode 100644
index 00000000000..599ae77a456
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+
+public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase {
+
+    @Test
+    public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException {
+        assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines(
+                "search ad {",
+                "  document ad {",
+                "    field campaign_ref type reference<campaign> { indexing: attribute }",
+                "    field person_ref type reference<person> { indexing: attribute }",
+                "  }",
+                "  import field campaign_ref.cool_field as my_cool_field {}",
+                "  import field campaign_ref.swag_field as my_swag_field {}",
+                "  import field person_ref.name as my_name {}",
+                "}")),
+                "multiple_imported_fields");
+    }
+
+    private static class TestDocumentModelBuilder {
+        private final SearchBuilder builder = new SearchBuilder();
+        public TestDocumentModelBuilder addCampaign() throws ParseException {
+            builder.importString(joinLines("search campaign {",
+                    "  document campaign {",
+                    "    field cool_field type string { indexing: attribute }",
+                    "    field swag_field type long { indexing: attribute }",
+                    "  }",
+                    "}"));
+            return this;
+        }
+        public TestDocumentModelBuilder addPerson() throws ParseException {
+            builder.importString(joinLines("search person {",
+                    "  document person {",
+                    "    field name type string { indexing: attribute }",
+                    "  }",
+                    "}"));
+            return this;
+        }
+        public DocumentModel build(String adSdContent) throws ParseException {
+            builder.importString(adSdContent);
+            builder.build();
+            return builder.getModel();
+        }
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
index e9a7a6ed33e..55980ee5fea 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
@@ -1,15 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.documentmodel;
 
-import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.ReferenceDataType;
-import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
-import com.yahoo.vespa.configmodel.producers.DocumentManager;
-import com.yahoo.vespa.configmodel.producers.DocumentTypes;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author geirst
  */
-public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase {
 
     @Test
     public void reference_fields_can_reference_other_document_types() throws ParseException, IOException {
@@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT
         assertEquals(campaignRefType.getTargetType(), campaignType);
     }
 
-    private static String TEST_FOLDER = "src/test/configmodel/types/references/";
-
-    private void assertDocumentConfigs(DocumentModel model,
-                                       String cfgFileSpec) throws IOException {
-        assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
-        assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
-    }
-
-    private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
-        DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
-        assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
-    }
-
-    private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
-        DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
-        assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
-    }
-
     private static class TestDocumentModelBuilder {
         private final SearchBuilder builder = new SearchBuilder();
         public TestDocumentModelBuilder addCampaign() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
index 4064e53dfb7..3dfcef70aba 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
@@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest {
                 headerfields,
                 new StructDataType("bodyfields"),
                 new FieldSets(),
+                Collections.emptySet(),
                 Collections.emptySet());
     }