Add imported fields to document configs

This adds the field names of all imported fields in a search definition
to the document configs associated with it (document manager and document
types config for Java and C++, respectively). This enables consumers of
these configs to distinguish between field references to imported fields
and fields that don't exist in the document schema itself.

5 files changed, 59 insertions, 6 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index f0b1b427531..d3a78321106 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
 import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
 import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
 import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -338,7 +340,8 @@ public class DocumentModelBuilder {
                                                  sdoc.getDocumentType().contentStruct(),
                                                  sdoc.getDocumentType().getBodyType(),
                                                  sdoc.getFieldSets(),
-                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()));
+                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()),
+                                                 convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields()));
         for (SDDocumentType n : sdoc.getInheritedTypes()) {
             NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
                 NewDocumentType inherited =  model.getDocumentManager().getDocumentType(name);
@@ -404,6 +407,13 @@ public class DocumentModelBuilder {
                 .collect(toSet());
     }
 
+    private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) {
+        if (importedFields == null) {
+            return emptySet();
+        }
+        return Collections.unmodifiableSet(importedFields.fields().keySet());
+    }
+
     private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
         for (Field f : fields) {
             DataType type = f.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
new file mode 100644
index 00000000000..c736ca8e9dc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
@@ -0,0 +1,32 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Enumerates and emplaces a set of all imported fields into a SDDocumentType from
+ * its corresponding Search instance.
+ */
+public class ImportedFieldsEnumerator {
+
+    private final List<Search> searchDefinitions;
+
+    public ImportedFieldsEnumerator(List<Search> searchDefinitions) {
+        this.searchDefinitions = searchDefinitions;
+    }
+
+    public void enumerateImportedFields(SDDocumentType documentType) {
+        var search = this.searchDefinitions.stream()
+                .filter(s -> s.getDocument() != null)
+                .filter(s -> s.getDocument().getName().equals(documentType.getName()))
+                .findFirst();
+        if (search.isEmpty()) {
+            return; // No imported fields present.
+        }
+        search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 949539ff99f..eb68e6af203 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -230,20 +230,22 @@ public class SearchBuilder {
                 sdocs.add(search.getDocument());
             }
         }
-        SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+        var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
         orderer.process();
         for (SDDocumentType sdoc : orderer.getOrdered()) {
             new FieldOperationApplierForStructs().process(sdoc);
             new FieldOperationApplier().process(sdoc);
         }
 
-        DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList);
+        var resolver = new DocumentReferenceResolver(searchList);
         sdocs.forEach(resolver::resolveReferences);
+        var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
+        sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
 
         if (validate)
             new DocumentGraphValidator().validateDocumentGraph(sdocs);
 
-        DocumentModelBuilder builder = new DocumentModelBuilder(model);
+        var builder = new DocumentModelBuilder(model);
         for (Search search : new SearchOrderer().order(searchList)) {
             new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
             process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index 414a605c621..b3f98cc6f26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable {
     private FieldSets fieldSets;
     // Document references
     private Optional<DocumentReferences> documentReferences = Optional.empty();
+    private TemporaryImportedFields temporaryImportedFields;
 
     static {
         VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
@@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable {
         type.docType = docType.clone();
         type.inheritedTypes.putAll(inheritedTypes);
         type.structType = structType;
+        // TODO this isn't complete; should it be..?!
         return type;
     }
 
@@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable {
         this.documentReferences = Optional.of(documentReferences);
     }
 
+    public TemporaryImportedFields getTemporaryImportedFields() {
+        return temporaryImportedFields;
+    }
+
+    public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) {
+        this.temporaryImportedFields = temporaryImportedFields;
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index d6c334ee80b..6d3de23238d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
-                                                   ImmutableSDField targetField, boolean validate) {
+                                              ImmutableSDField targetField, boolean validate) {
         TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
                 reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
         ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
@@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void validateTargetField(TemporaryImportedField importedField,
-                                                 ImmutableSDField targetField, DocumentReference reference) {
+                                     ImmutableSDField targetField, DocumentReference reference) {
         if (!targetField.doesAttributing()) {
             fail(importedField, targetFieldAsString(targetField.getName(), reference) +
                     ": Is not an attribute field. Only attribute fields supported");