Merge pull request #21659 from vespa-engine/arnej/on-demand-struct-fields

Arnej/on demand struct fields

35 files changed, 185 insertions, 209 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index d98869e9dd3..4ff54d7ff1c 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -4,8 +4,10 @@ package com.yahoo.documentmodel;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.Field;
+import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.document.annotation.AnnotationTypeRegistry;
 import com.yahoo.document.datatypes.FieldValue;
@@ -383,4 +385,18 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
 
     }
 
+    private ReferenceDataType refToThis = null;
+
+    @SuppressWarnings("deprecation")
+    public ReferenceDataType getReferenceDataType() {
+        if (refToThis == null) {
+            // super ugly, the APIs for this are horribly inconsistent
+            var tmptmp = TemporaryStructuredDataType.create(getName());
+            var tmp = ReferenceDataType.createWithInferredId(tmptmp);
+            tmp.setTargetType((StructuredDataType) this);
+            refToThis = tmp;
+        }
+        return refToThis;
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
index 16eef798acd..64688a7e70d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
@@ -84,7 +84,6 @@ public class Application {
 
         List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas);
         for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) {
-            new FieldOperationApplierForStructs().processSchemaFields(schema);
             new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list?
             new Processing(properties).process(schema,
                                                logger,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index 4a449dc898f..2d9c81085fe 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -292,17 +292,8 @@ public class DocumentModelBuilder {
         else if (type instanceof ReferenceDataType) {
             ReferenceDataType t = (ReferenceDataType) type;
             var tt = t.getTargetType();
-            if (tt instanceof TemporaryStructuredDataType) {
-                DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
-                t.setTargetType((StructuredDataType) targetType);
-            } else if (tt instanceof DocumentType) {
-                DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
-                // super ugly, the APIs for this are horribly inconsistent
-                var tmptmp = TemporaryStructuredDataType.create(tt.getName());
-                var tmp = new ReferenceDataType(tmptmp, t.getId());
-                tmp.setTargetType((StructuredDataType) targetType);
-                type = tmp;
-            }
+            var doc = getDocumentType(docs, tt.getId());
+            type = doc.getReferenceDataType();
         }
         if (type != original) {
             replacements.add(new TypeReplacement(original, type));
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
index 5e5623e2319..4a5a858f828 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -20,47 +20,8 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier {
         for (SDDocumentType type : sdoc.getAllTypes()) {
             if (type.isStruct()) {
                 apply(type);
-                copyFields(type, sdoc);
             }
         }
     }
 
-    @SuppressWarnings("deprecation")
-    private void copyFields(SDDocumentType structType, SDDocumentType sdoc) {
-        //find all fields in OTHER types that have this type:
-        List<SDDocumentType> list = new ArrayList<>();
-        list.add(sdoc);
-        list.addAll(sdoc.getTypes());
-        for (SDDocumentType anyType : list) {
-            Iterator<Field> fields = anyType.fieldIterator();
-            while (fields.hasNext()) {
-                SDField field = (SDField) fields.next();
-                maybePopulateField(sdoc, field, structType);
-            }
-        }
-    }
-
-    private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) {
-        DataType structUsedByField = field.getFirstStructRecursive();
-        if (structUsedByField == null) {
-            return;
-        }
-        if (structUsedByField.getName().equals(structType.getName())) {
-            //this field is using this type!!
-            field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0);
-            field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching());
-        }
-    }
-
-    public void processSchemaFields(Schema schema) {
-        var sdoc = schema.getDocument();
-        if (sdoc == null) return;
-        for (SDDocumentType type : sdoc.getAllTypes()) {
-            if (type.isStruct()) {
-                for (SDField field : schema.allExtraFields()) {
-                    maybePopulateField(sdoc, field, type);
-                }
-            }
-        }
-    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index a037d5046a7..b87bdd8907e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -268,8 +268,8 @@ public class SDDocumentType implements Cloneable, Serializable {
         return field;
     }
 
-    public Field addField(String string, DataType dataType, boolean header, int code) {
-        SDField field = new SDField(this, string, code, dataType, header);
+    public Field addField(String fName, DataType dataType, boolean header, int code) {
+        SDField field = new SDField(this, fName, code, dataType);
         addField(field);
         return field;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index bd6625d4bde..8263352e87f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -111,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
     /** Struct fields defined in this field */
     private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0);
 
-    /** The document that this field was declared in, or null*/
-    private SDDocumentType ownerDocType = null;
+    /** The document that this field was declared in, or null */
+    private SDDocumentType repoDocType = null;
 
     /** The aliases declared for this field. May pertain to indexes or attributes */
     private final Map<String, String> aliasToName = new HashMap<>();
@@ -130,25 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      * @param name the name of the field
      * @param dataType the datatype of the field
     */
-    protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) {
+    public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
         super(name, id, dataType);
-        populate(populate, repo, name, dataType);
+        this.repoDocType = repo;
+        populate(name, dataType);
     }
 
-    public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
-        this(repo, name, id, dataType, true);
+    public SDField(String name, DataType dataType) {
+        this(null, name, dataType);
     }
 
     /** Creates a new field */
-    public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) {
-        super(name, dataType);
-        populate(populate, repo, name, dataType);
+    public SDField(SDDocumentType repo, String name, DataType dataType) {
+        this(repo, name, dataType, null);
     }
 
     /** Creates a new field */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) {
-        super(name, dataType, owner == null ? null : owner.getDocumentType());
-        populate(populate, repo, name, dataType);
+    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
+        this(repo, name, dataType, owner, null, 0);
     }
 
     /**
@@ -159,27 +158,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      * @param owner the owning document (used to check for id collisions)
      * @param fieldMatching the matching object to set for the field
      */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner,
-                      Matching fieldMatching, boolean populate, int recursion) {
+    protected SDField(SDDocumentType repo,
+                      String name,
+                      DataType dataType,
+                      SDDocumentType owner,
+                      Matching fieldMatching,
+                      int recursion)
+    {
         super(name, dataType, owner == null ? null : owner.getDocumentType());
+        this.repoDocType = repo;
+        this.structFieldDepth = recursion;
         if (fieldMatching != null)
             this.setMatching(fieldMatching);
-        populate(populate, repo, name, dataType, fieldMatching, recursion);
+        populate(name, dataType);
     }
 
-    public SDField(SDDocumentType repo, String name, DataType dataType) {
-        this(repo, name, dataType, true);
-    }
+    private int structFieldDepth = 0;
 
-    public SDField(String name, DataType dataType) {
-        this(null, name, dataType);
-    }
-
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) {
-        populate(populate, repo, name, dataType, null, 0);
-    }
-
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching,  int recursion) {
+    private void populate(String name, DataType dataType) {
         if (dataType instanceof TensorDataType) {
             TensorType type = ((TensorDataType)dataType).getTensorType();
             if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
@@ -194,10 +190,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         else {
             addQueryCommand("type " + dataType.getName());
         }
-        if (populate || (dataType instanceof MapDataType)) {
-            populateWithStructFields(repo, name, dataType, recursion);
-            populateWithStructMatching(repo, dataType, fieldMatching);
-        }
     }
 
     public void setIsExtraField(boolean isExtra) {
@@ -273,17 +265,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         }
     }
 
+    private boolean doneStructFields = false;
+
     @SuppressWarnings("deprecation")
-    public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) {
-        DataType dt = getFirstStructOrMapRecursive();
-        if (dt == null) return;
+    private void actuallyMakeStructFields() {
+        if (doneStructFields) return;
+        if (getFirstStructOrMapRecursive() == null) {
+            doneStructFields = true;
+            return;
+        }
+        var sdoc = repoDocType;
+        var dataType = getDataType();
 
         java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> {
             if (structFields.containsKey(fieldName)) return;
-            String subName = name.concat(".").concat(fieldName);
-            var subField = new SDField(sdoc, subName, fieldType,
-                                       ownerDocType, new Matching(),
-                                       true, recursion + 1);
+            String subName = getName().concat(".").concat(fieldName);
+            var subField = new SDField(sdoc, subName, fieldType, null,
+                                       null, structFieldDepth + 1);
             structFields.put(fieldName, subField);
         };
 
@@ -292,15 +290,16 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
             supplyStructField.accept("key", mdt.getKeyType());
             supplyStructField.accept("value", mdt.getValueType());
         } else {
-            if (recursion >= 10) return;
+            if (structFieldDepth >= 10) {
+                // too risky, infinite recursion
+                doneStructFields = true;
+                return;
+            }
             if (dataType instanceof CollectionDataType) {
                 dataType = ((CollectionDataType)dataType).getNestedType();
             }
-            if (dataType instanceof TemporaryStructuredDataType) {
-                SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
-                if (subType == null) {
-                    throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'.");
-                }
+            SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+            if (dataType instanceof TemporaryStructuredDataType && subType != null) {
                 for (Field field : subType.fieldSet()) {
                     supplyStructField.accept(field.getName(), field.getDataType());
                 }
@@ -310,37 +309,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                     supplyStructField.accept(field.getName(), field.getDataType());
                 }
             }
-        }
-    }
-
-    public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) {
-        if (sdoc == null) return;
-        if (superFieldMatching == null) return;
-        DataType dt = getFirstStructOrMapRecursive();
-        if (dt == null) return;
-
-        if (dataType instanceof MapDataType) {
-            // old code here would never do anything useful, should we do something here?
-            return;
-        } else {
-            if (dataType instanceof CollectionDataType) {
-                dataType = ((CollectionDataType)dataType).getNestedType();
+            if ((subType == null) && (structFields.size() > 0)) {
+                throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in "
+                                                   + this + " [" + getDataType() + getDataType().getClass() +
+                                                   "] with " + structFields.size() + " struct fields");
             }
-            if (dataType instanceof StructDataType) {
-                SDDocumentType subType = sdoc.getType(dataType.getName());
-                if (subType == null) {
-                    throw new IllegalArgumentException("Could not find struct " + dataType.getName());
-                }
+            // populate struct fields with matching
+            if (subType != null) {
                 for (Field f : subType.fieldSet()) {
                     if (f instanceof SDField) {
                         SDField field = (SDField) f;
                         Matching subFieldMatching = new Matching();
-                        subFieldMatching.merge(superFieldMatching);
+                        subFieldMatching.merge(this.matching);
                         subFieldMatching.merge(field.getMatching());
                         SDField subField = structFields.get(field.getName());
                         if (subField != null) {
-                            subFieldMatching.merge(subField.getMatching());
-                            subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching);
+                            // we just made this with no matching, so nop:
+                            // subFieldMatching.merge(subField.getMatching());
                             subField.setMatching(subFieldMatching);
                         }
                     } else {
@@ -349,8 +334,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                 }
             }
         }
+        doneStructFields = true;
     }
 
+    private Matching matchingForStructFields = null;
+
     public void addOperation(FieldOperation op) {
         pendingOperations.add(op);
     }
@@ -723,7 +711,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
 
     /** Returns list of static struct fields */
     @Override
-    public Collection<SDField> getStructFields() { return structFields.values(); }
+    public Collection<SDField> getStructFields() {
+        actuallyMakeStructFields();
+        return structFields.values();
+    }
 
     /**
      * Returns a struct field defined in this field,
@@ -732,6 +723,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      */
     @Override
     public SDField getStructField(String name) {
+        actuallyMakeStructFields();
         if (name.contains(".")) {
             String superFieldName = name.substring(0,name.indexOf("."));
             String subFieldName = name.substring(name.indexOf(".")+1);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
index 4ced104fa55..8c17b607f94 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -8,12 +8,12 @@ import com.yahoo.document.DataType;
  */
 public class TemporarySDField extends SDField {
 
-    public TemporarySDField(String name, DataType dataType, SDDocumentType owner) {
-        super(owner, name, dataType, owner, false);
+    public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
+        super(repo, name, dataType, owner);
     }
 
-    public TemporarySDField(String name, DataType dataType) {
-        super(null, name, dataType, false);
+    public TemporarySDField(SDDocumentType repo, String name, DataType dataType) {
+        super(repo, name, dataType);
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
index fe3ac11af27..a5f5f961ab5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
@@ -24,6 +24,8 @@ public class IndexingOperation implements FieldOperation {
         this.script = script;
     }
 
+    public ScriptExpression getScript() { return script; }
+
     public void apply(SDField field) {
         field.setIndexingScript(script);
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
index 36c11b33b23..caeebd65f4f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
@@ -291,13 +291,12 @@ public class ConvertParsedFields {
         schema.addIndex(index);
     }
 
-    SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) {
+    SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) {
         // TODO - can we cleanup this mess
         var structProxy = new SDDocumentType(parsed.name(), schema);
-        structProxy.setStruct(context.resolveStruct(parsed));
         for (var parsedField : parsed.getFields()) {
             var fieldType = context.resolveType(parsedField.getType());
-            var field = new SDField(structProxy, parsedField.name(), fieldType);
+            var field = new SDField(document, parsedField.name(), fieldType);
             convertCommonFieldSettings(field, parsedField);
             structProxy.addField(field);
             if (parsedField.hasIdOverride()) {
@@ -307,6 +306,7 @@ public class ConvertParsedFields {
         for (String inherit : parsed.getInherited()) {
             structProxy.inherit(new DataTypeName(inherit));                
         }
+        structProxy.setStruct(context.resolveStruct(parsed));
         return structProxy;
     }
 
@@ -314,7 +314,7 @@ public class ConvertParsedFields {
         var annType = context.resolveAnnotation(parsed.name());
         var payload = parsed.getStruct();
         if (payload.isPresent()) {
-            var structProxy = convertStructDeclaration(schema, payload.get());
+            var structProxy = convertStructDeclaration(schema, document, payload.get());
             document.addType(structProxy);
         }
         document.addAnnotation(annType);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
index d32fae99f5a..2d9a788cfef 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
@@ -33,7 +33,9 @@ import com.yahoo.vespa.documentmodel.DocumentSummary;
 import com.yahoo.vespa.documentmodel.SummaryField;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -122,6 +124,8 @@ public class ConvertSchemaCollection {
         typeConverter.convert(true);
     }
 
+    private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap();
+
     public List<Schema> convertToSchemas() {
         typeConverter = new ConvertParsedTypes(orderedInput, docMan);
         typeConverter.convert(false);
@@ -150,10 +154,12 @@ public class ConvertSchemaCollection {
     {
         SDDocumentType document = new SDDocumentType(parsed.name());
         for (String inherit : parsed.getInherited()) {
-            document.inherit(new DataTypeName(inherit));
+            var parent = convertedDocuments.get(inherit);
+            assert(parent != null);
+            document.inherit(parent);
         }
         for (var struct : parsed.getStructs()) {
-            var structProxy = fieldConverter.convertStructDeclaration(schema, struct);
+            var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct);
             document.addType(structProxy);
         }
         for (var annotation : parsed.getAnnotations()) {
@@ -165,6 +171,7 @@ public class ConvertSchemaCollection {
                 document.setFieldId(sdf, field.idOverride());
             }
         }
+        convertedDocuments.put(parsed.name(), document);
         schema.addDocument(document);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
index 536caf55111..23b5195486b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
@@ -12,7 +12,7 @@ import com.yahoo.yolean.Exceptions;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -27,7 +27,7 @@ public class IntermediateCollection {
     private final DeployLogger deployLogger;
     private final ModelContext.Properties modelProperties;
 
-    private Map<String, ParsedSchema> parsedSchemas = new HashMap<>();
+    private Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>();
 
     IntermediateCollection() {
         this.deployLogger = new BaseDeployLogger();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
index 51defffa00b..0be48d1fd25 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -70,7 +70,7 @@ public class AddExtraFieldsToDocument extends Processor {
         if (docField == null) {
             ImmutableSDField existingField = schema.getField(field.getName());
             if (existingField == null) {
-                SDField newField = new SDField(document, field.getName(), field.getDataType(), true);
+                SDField newField = new SDField(document, field.getName(), field.getDataType());
                 newField.setIsExtraField(true);
                 document.addField(newField);
             } else if (!existingField.isImportedField()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
index 0bb1b7da769..d7882c7f8fb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -10,6 +10,7 @@ import com.yahoo.document.PositionDataType;
 import com.yahoo.searchdefinition.Schema;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.GeoPos;
+import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.vespa.documentmodel.SummaryField;
 import com.yahoo.vespa.documentmodel.SummaryTransform;
@@ -35,8 +36,11 @@ import java.util.logging.Level;
  */
 public class CreatePositionZCurve extends Processor {
 
+    private final SDDocumentType repo;
+
     public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
+        this.repo = schema.getDocument();
     }
 
     private boolean useV8GeoPositions = false;
@@ -105,7 +109,7 @@ public class CreatePositionZCurve extends Processor {
                                                     "' already created.");
         }
         boolean isArray = inputField.getDataType() instanceof ArrayDataType;
-        SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
+        SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
         Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY :
                                                                             Attribute.CollectionType.SINGLE);
         attribute.setPosition(true);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
index 84dc6d369fc..7397f9a289c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -61,7 +61,7 @@ public class UriHack extends Processor {
             String partName = uriName + "." + suffix;
             // I wonder if this is explicit in qrs or implicit in backend?
             // search.addFieldSetItem(uriName, partName);
-            SDField partField = new SDField(partName, generatedType);
+            SDField partField = new SDField(schema.getDocument(), partName, generatedType);
             partField.setIndexStructureField(uriField.doesIndexing());
             partField.setRankType(uriField.getRankType());
             partField.setStemming(Stemming.NONE);
diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index 7447eee5cec..ba955f071b2 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -958,10 +958,7 @@ void indexingOperation(ParsedField field, boolean multiLine) : { }
 {
     {
         IndexingOperation oldOp = newIndexingOperation(multiLine);
-        // TODO conversion via SDField is very ugly
-        SDField tmpField = new SDField("temp", com.yahoo.document.DataType.STRING);
-        oldOp.apply(tmpField);
-        ParsedIndexingOp newOp = new ParsedIndexingOp(tmpField.getIndexingScript());
+        ParsedIndexingOp newOp = new ParsedIndexingOp(oldOp.getScript());
         field.setIndexingOperation(newOp);
     }
 }
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 1a0b518bbb2..e90df2776e0 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) :
         if (name != null && Schema.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, document);
+        field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document);
     }
     lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE>
     {
@@ -668,7 +668,7 @@ void fieldSet(Schema schema) :
     |
       ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
     |
-      ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
+      ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
     )(<NL>)*)+
   <RBRACE>
   {
@@ -698,11 +698,13 @@ void annotationOutside(Schema schema) :
     <ANNOTATION> name = identifier()
     {
         type = new SDAnnotationType(name.trim());
+        if (schema.getDocument() == null) {
+            throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
+        }
     }
     [ inheritsAnnotation(type) (<NL>)* ]
-    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE>
     {
-        if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
         schema.addAnnotation(type);
     }
 }
@@ -723,7 +725,7 @@ void annotation(Schema schema, SDDocumentType document) :
         type = new SDAnnotationType(name.trim());
     }
     [ inheritsAnnotation(type) (<NL>)* ]
-    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    lbrace() (type = annotationBody(schema, document, type)) <RBRACE>
     {
         document.addAnnotation(type);
     }
@@ -737,12 +739,12 @@ void annotation(Schema schema, SDDocumentType document) :
  * @param type     the type being built
  * @return a modified or new AnnotationType instance
  */
-SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) :
+SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) :
 {
     SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema);
 }
 {
-    (structFieldDefinition(struct) (<NL>)*)*
+    (structFieldDefinition(repo, struct) (<NL>)*)*
     {
         if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField.
             type = new SDAnnotationType(type.getName(), struct, type.getInherits());
@@ -809,9 +811,13 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
     SDDocumentType struct;
 }
 {
-    ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); }
+    ( <STRUCT> name = identifier() (<NL>)* {
+        if (repo == null)
+            throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
+        struct = new SDDocumentType(name, schema);
+    }
     [ inheritsDocument(struct) (<NL>)* ]
-    lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> )
+    lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> )
     {
         try {
             docMan.getDataType(name);
@@ -819,7 +825,6 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
         } catch (IllegalArgumentException e) {
             // empty
         }
-        if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
         SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName());
         DataType stype = sdtype != null
                          ? sdtype.getStruct()
@@ -828,7 +833,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
         return struct;
     }
 }
-
+        
 /**
  * This rule consumes a data type block from within a field element.
  *
@@ -921,7 +926,7 @@ DataType wildCardType() :
  *
  * @param struct The struct to modify.
  */
-void structFieldDefinition(SDDocumentType struct) :
+void structFieldDefinition(SDDocumentType document, SDDocumentType struct) :
 {
     String name;
     SDField field;
@@ -932,7 +937,7 @@ void structFieldDefinition(SDDocumentType struct) :
         if (name != null && Schema.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, struct);
+        field = new TemporarySDField(document, name, type, struct);
         struct.addField(field);
     }
     lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg
index 876ed00d0c4..ef1bb4c5ad4 100644
--- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg
+++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg
@@ -47,8 +47,6 @@ doctype[1].fieldsets{[document]}.fields[0] "campaign_ref"
 doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref"
 doctype[1].documentref[0].idx 10017
 doctype[1].documentref[0].targettype 10018
-doctype[1].documentref[1].idx 10019
-doctype[1].documentref[1].targettype 10018
 doctype[1].structtype[0].idx 10016
 doctype[1].structtype[0].name "ad.header"
 doctype[1].structtype[0].field[0].name "campaign_ref"
@@ -56,10 +54,10 @@ doctype[1].structtype[0].field[0].internalid 23963250
 doctype[1].structtype[0].field[0].type 10017
 doctype[1].structtype[0].field[1].name "other_campaign_ref"
 doctype[1].structtype[0].field[1].internalid 874751172
-doctype[1].structtype[0].field[1].type 10019
+doctype[1].structtype[0].field[1].type 10017
 doctype[2].name "campaign"
 doctype[2].idx 10018
 doctype[2].inherits[0].idx 10000
-doctype[2].contentstruct 10020
-doctype[2].structtype[0].idx 10020
+doctype[2].contentstruct 10019
+doctype[2].structtype[0].idx 10019
 doctype[2].structtype[0].name "campaign.header"
diff --git a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd
index 32823e46592..0636e7a537e 100644
--- a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd
+++ b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd
@@ -21,6 +21,7 @@ search streamingstruct {
       # Allow default matchtypes in struct. Can be overridden.
       # No index/attribute related stuff. It is only a datatype definition.
     }
+
     struct ns1 {
       field nf1 type s1 { }
       field nf1s type s1 { match: substring }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
index 81a44261daf..8edbd789a27 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
@@ -156,8 +156,8 @@ public class DocumentGraphValidatorTest {
 
     @SuppressWarnings("deprecation")
     private static void createDocumentReference(Schema from, Schema to, String refFieldName) {
-        SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName())));
         SDDocumentType fromDocument = from.getDocument();
+        SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName())));
         fromDocument.addField(refField);
         Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap();
         HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index 66f1850bd10..4fa145afae9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -38,12 +38,12 @@ public class DocumentReferenceResolverTest {
         barSchema.addDocument(barDocument);
 
         // Create foo document with document reference to bar and add another field
-        SDField fooRefToBarField = new SDField
-                ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
-        AttributeUtils.addAttributeAspect(fooRefToBarField);
-        SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
         Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty());
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema);
+        SDField fooRefToBarField = new SDField
+            (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
+        AttributeUtils.addAttributeAspect(fooRefToBarField);
+        SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT);
         fooDocument.addField(fooRefToBarField);
         fooDocument.addField(irrelevantField);
         fooSchema.addDocument(fooDocument);
@@ -62,11 +62,12 @@ public class DocumentReferenceResolverTest {
     @Test
     public void throws_user_friendly_exception_if_referenced_document_does_not_exist() {
         // Create foo document with document reference to non-existing document bar
+        Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty());
+        SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema);
         SDField fooRefToBarField = new SDField(
+                fooDocument,
                 "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
         AttributeUtils.addAttributeAspect(fooRefToBarField);
-        Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty());
-        SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema);
         fooDocument.addField(fooRefToBarField);
         fooSchema.addDocument(fooDocument);
 
@@ -86,10 +87,10 @@ public class DocumentReferenceResolverTest {
         barSchema.addDocument(barDocument);
 
         // Create foo document with document reference to bar
-        SDField fooRefToBarField = new SDField
-                ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
         Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty());
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema);
+        SDField fooRefToBarField = new SDField
+            (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
         fooDocument.addField(fooRefToBarField);
         fooSchema.addDocument(fooDocument);
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
index 7d2386030da..7e708f93a96 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -22,16 +22,18 @@ public class ImportedFieldsEnumeratorTest {
         String PARENT = "parent";
         Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty());
         SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema);
-        var parentField = new SDField("their_field", DataType.INT);
+        var parentField = new SDField(parentDocument, "their_field", DataType.INT);
         AttributeUtils.addAttributeAspect(parentField);
         parentDocument.addField(parentField);
         parentSchema.addDocument(parentDocument);
 
         String FOO = "foo";
         Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty());
+        /*
         SDField fooRefToParent = new SDField(
                 "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
         AttributeUtils.addAttributeAspect(fooRefToParent);
+        */
         var fooImports = fooSchema.temporaryImportedFields().get();
         fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
         fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
@@ -40,9 +42,11 @@ public class ImportedFieldsEnumeratorTest {
 
         String BAR = "bar";
         Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty());
+        /*
         SDField barRefToParent = new SDField(
                 "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
         AttributeUtils.addAttributeAspect(barRefToParent);
+        */
         var barImports = barSchema.temporaryImportedFields().get();
         barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
         SDDocumentType barDocument = new SDDocumentType(BAR, barSchema);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java
index c7ec2d9e9d1..652a06a6025 100755
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java
@@ -37,22 +37,22 @@ public class SDDocumentTypeOrdererTestCase {
         g.inherit(new TemporarySDDocumentType(new DataTypeName("e")));
         g.inherit(new TemporarySDDocumentType(new DataTypeName("c")));
 
-        SDField aFieldTypeB = new TemporarySDField("atypeb", DataType.STRING);
+        SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING);
         a.addField(aFieldTypeB);
 
-        SDField bFieldTypeC = new TemporarySDField("btypec", DataType.STRING);
+        SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING);
         b.addField(bFieldTypeC);
 
-        SDField cFieldTypeG = new TemporarySDField("ctypeg", DataType.STRING);
+        SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING);
         c.addField(cFieldTypeG);
 
-        SDField gFieldTypeF = new TemporarySDField("gtypef", DataType.STRING);
+        SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING);
         g.addField(gFieldTypeF);
 
-        SDField fFieldTypeC = new TemporarySDField("ftypec", DataType.STRING);
+        SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING);
         f.addField(fFieldTypeC);
 
-        SDField dFieldTypeE = new TemporarySDField("dtypee", DataType.STRING);
+        SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING);
         d.addField(dFieldTypeE);
 
         types.add(a);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
index 237e6c8c992..a3123550efa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
@@ -25,11 +25,11 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase {
         RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema);
         SDDocumentType doc = new SDDocumentType("test");
         schema.addDocument(doc);
-        doc.addField(new SDField("a", DataType.STRING));
-        SDField field = new SDField("b", DataType.STRING);
+        doc.addField(new SDField(doc, "a", DataType.STRING));
+        SDField field = new SDField(doc, "b", DataType.STRING);
         field.setLiteralBoost(500);
         doc.addField(field);
-        doc.addField(new SDField("c", DataType.STRING));
+        doc.addField(new SDField(doc, "c", DataType.STRING));
 
         schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry());
         new DerivedConfiguration(schema, rankProfileRegistry);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
index 440f067dd00..ba485b7b96b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
@@ -29,7 +29,7 @@ public class IdTestCase extends AbstractExportingTestCase {
         Schema schema = new Schema("test", MockApplicationPackage.createEmpty());
         SDDocumentType document = new SDDocumentType("test");
         schema.addDocument(document);
-        SDField uri = new SDField("URI", DataType.URI);
+        SDField uri = new SDField(document, "URI", DataType.URI);
         uri.parseIndexingScript("{ summary | index }");
         document.addField(uri);
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java
index 63e3df2205f..275c94465ae 100755
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java
@@ -54,12 +54,8 @@ public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testMapWithStructSettings() throws IOException, ParseException {
-        /*
-         * does not work
-         * does not pick up settings from struct declaration
         assertCorrectDeriving("matchsettings_map_wss",
                               new TestProperties().setExperimentalSdParsing(false));
-        */
         assertCorrectDeriving("matchsettings_map_wss",
                               new TestProperties().setExperimentalSdParsing(true));
     }
@@ -74,11 +70,8 @@ public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testMapAfter() throws IOException, ParseException {
-        /* fails with:
-        java.lang.IllegalArgumentException: Could not find struct 'elem'.
         assertCorrectDeriving("matchsettings_map_after",
                               new TestProperties().setExperimentalSdParsing(false));
-        */
         assertCorrectDeriving("matchsettings_map_after",
                               new TestProperties().setExperimentalSdParsing(true));
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
index 34d33a00d9e..2522f8f56e2 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
@@ -91,8 +91,8 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase {
 
     @SuppressWarnings("deprecation")
     private static void createDocumentReference(Schema from, Schema to, String refFieldName) {
-        SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName())));
         SDDocumentType fromDocument = from.getDocument();
+        SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName())));
         fromDocument.addField(refField);
         Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap();
         HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
index dbb32e61144..62a79e49146 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
@@ -30,7 +30,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase {
         RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema);
         SDDocumentType document = new SDDocumentType("test");
         schema.addDocument(document);
-        SDField a = new SDField("a", DataType.STRING);
+        SDField a = new SDField(document, "a", DataType.STRING);
         a.parseIndexingScript("{ index }");
         document.addField(a);
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
index 5ab5a8057e8..9c974225605 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
@@ -25,8 +25,9 @@ public class VsmFieldsTestCase {
     @Test
     public void reference_type_field_is_unsearchable() {
         Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties());
-        schema.addDocument(new SDDocumentType("test"));
-        SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type")));
+        var sdoc = new SDDocumentType("test");
+        schema.addDocument(sdoc);
+        SDField refField = new TemporarySDField(sdoc, "ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type")));
         refField.parseIndexingScript("{ summary }");
         schema.getDocument().addField(refField);
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
index 9ca97f4dbc7..0adaba4cf68 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
@@ -59,8 +59,9 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest {
     }
 
     private static ImportedFields createSingleImportedField(String fieldName) {
-        Schema targetSchema = createSearch("target_doc");
-        SDField targetField = new SDField("target_field", DataType.INT);
+        Schema targetSchema = createSearchWithDocument("target_doc");
+        var doc = targetSchema.getDocument();
+        SDField targetField = new SDField(doc, "target_field", DataType.INT);
         DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema);
         ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField);
         return new ImportedFields(Collections.singletonMap(fieldName, importedField));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java
index 5d2590a420d..926e26451d7 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java
@@ -186,9 +186,10 @@ public class AdjustPositionSummaryFieldsTestCase {
 
         private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) {
             String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }";
-            schema.getDocument().addField(createField("pos", PositionDataType.INSTANCE, ilScript));
+            var doc = schema.getDocument();
+            doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript));
             if (setupBadAttr) {
-                schema.getDocument().addField(createField("pos_zcurve", DataType.LONG, "{ attribute }"));
+                doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }"));
             }
         }
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
index 324010f9e83..d1d3f4489ce 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
@@ -110,13 +110,14 @@ public class ImportedFieldsResolverTestCase {
         public SearchModel() {
             super();
             grandParentSchema = createSearch("grandparent");
-            grandParentSchema.getDocument().addField(createField("ancient_field", DataType.INT, "{ attribute }"));
-
-            parentSchema.getDocument().addField(createField("attribute_field", DataType.INT, "{ attribute }"));
-            parentSchema.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }"));
-            parentSchema.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT));
-            parentSchema.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }"));
-            parentSchema.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }"));
+            var grandParentDoc = grandParentSchema.getDocument();
+            grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }"));
+            var parentDoc = parentSchema.getDocument();
+            parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }"));
+            parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }"));
+            parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT));
+            parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }"));
+            parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }"));
             addRefField(parentSchema, grandParentSchema, "ref");
             addImportedField(parentSchema, "ancient_field", "ref", "ancient_field");
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
index b14c7287537..3b4612ee87a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
@@ -37,19 +37,19 @@ public class ParentChildSearchModel {
         return result;
     }
 
-    protected static TemporarySDField createField(String name, DataType dataType, String indexingScript) {
-        TemporarySDField result = new TemporarySDField(name, dataType);
+    protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) {
+        TemporarySDField result = new TemporarySDField(repo, name, dataType);
         result.parseIndexingScript(indexingScript);
         return result;
     }
 
     @SuppressWarnings("deprecation")
-    protected static SDField createRefField(String parentType, String fieldName) {
-        return new TemporarySDField(fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType)));
+    protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) {
+        return new TemporarySDField(repo, fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType)));
     }
 
     protected static void addRefField(Schema child, Schema parent, String fieldName) {
-        SDField refField = createRefField(parent.getName(), fieldName);
+        SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName);
         child.getDocument().addField(refField);
         child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(),
                 new DocumentReference(refField, parent))));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
index 22fd4e45c4a..8c801d9deaf 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
@@ -65,8 +65,8 @@ public class ValidateFieldTypesTest {
     }
 
     private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) {
-        Schema targetSchema = createSearch("target_doc");
-        SDField targetField = new SDField("target_field", dataType);
+        Schema targetSchema = createSearchWithDocument("target_doc");
+        SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType);
         DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema);
         ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField);
         return new ImportedFields(Collections.singletonMap(fieldName, importedField));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
index e58c29cc4fd..405c1127842 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
@@ -42,7 +42,7 @@ public class SchemaClusterTest {
         // sd1
         SDDocumentType sdt1 = new SDDocumentType("s1");
         Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty());
-        SDField f1 = new SDField("f1", DataType.STRING);
+        SDField f1 = new SDField(sdt1, "f1", DataType.STRING);
         f1.addAttribute(new Attribute("f1", DataType.STRING));
         f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1"))));
         sdt1.addField(f1);
@@ -51,7 +51,7 @@ public class SchemaClusterTest {
         // sd2
         SDDocumentType sdt2 = new SDDocumentType("s2");
         Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty());
-        SDField f2=new SDField("f2", DataType.STRING);
+        SDField f2=new SDField(sdt2, "f2", DataType.STRING);
         f2.addAttribute(new Attribute("f2", DataType.STRING));
         f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2"))));
         sdt2.addField(f2);
diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java
index e47c60863ff..2fcbe7333b4 100644
--- a/document/src/main/java/com/yahoo/document/DataType.java
+++ b/document/src/main/java/com/yahoo/document/DataType.java
@@ -62,6 +62,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com
     static {
         TAG.setTag(true);
     }
+
     public static int lastPredefinedDataTypeId() {
         return 21;
     }