Merge pull request #21659 from vespa-engine/arnej/on-demand-struct-fields

Arnej/on demand struct fields

16 files changed, 127 insertions, 153 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index d98869e9dd3..4ff54d7ff1c 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -4,8 +4,10 @@ package com.yahoo.documentmodel;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.Field;
+import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.document.annotation.AnnotationTypeRegistry;
 import com.yahoo.document.datatypes.FieldValue;
@@ -383,4 +385,18 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
 
     }
 
+    private ReferenceDataType refToThis = null;
+
+    @SuppressWarnings("deprecation")
+    public ReferenceDataType getReferenceDataType() {
+        if (refToThis == null) {
+            // super ugly, the APIs for this are horribly inconsistent
+            var tmptmp = TemporaryStructuredDataType.create(getName());
+            var tmp = ReferenceDataType.createWithInferredId(tmptmp);
+            tmp.setTargetType((StructuredDataType) this);
+            refToThis = tmp;
+        }
+        return refToThis;
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
index 16eef798acd..64688a7e70d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java
@@ -84,7 +84,6 @@ public class Application {
 
         List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas);
         for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) {
-            new FieldOperationApplierForStructs().processSchemaFields(schema);
             new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list?
             new Processing(properties).process(schema,
                                                logger,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index 4a449dc898f..2d9c81085fe 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -292,17 +292,8 @@ public class DocumentModelBuilder {
         else if (type instanceof ReferenceDataType) {
             ReferenceDataType t = (ReferenceDataType) type;
             var tt = t.getTargetType();
-            if (tt instanceof TemporaryStructuredDataType) {
-                DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
-                t.setTargetType((StructuredDataType) targetType);
-            } else if (tt instanceof DocumentType) {
-                DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
-                // super ugly, the APIs for this are horribly inconsistent
-                var tmptmp = TemporaryStructuredDataType.create(tt.getName());
-                var tmp = new ReferenceDataType(tmptmp, t.getId());
-                tmp.setTargetType((StructuredDataType) targetType);
-                type = tmp;
-            }
+            var doc = getDocumentType(docs, tt.getId());
+            type = doc.getReferenceDataType();
         }
         if (type != original) {
             replacements.add(new TypeReplacement(original, type));
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
index 5e5623e2319..4a5a858f828 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -20,47 +20,8 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier {
         for (SDDocumentType type : sdoc.getAllTypes()) {
             if (type.isStruct()) {
                 apply(type);
-                copyFields(type, sdoc);
             }
         }
     }
 
-    @SuppressWarnings("deprecation")
-    private void copyFields(SDDocumentType structType, SDDocumentType sdoc) {
-        //find all fields in OTHER types that have this type:
-        List<SDDocumentType> list = new ArrayList<>();
-        list.add(sdoc);
-        list.addAll(sdoc.getTypes());
-        for (SDDocumentType anyType : list) {
-            Iterator<Field> fields = anyType.fieldIterator();
-            while (fields.hasNext()) {
-                SDField field = (SDField) fields.next();
-                maybePopulateField(sdoc, field, structType);
-            }
-        }
-    }
-
-    private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) {
-        DataType structUsedByField = field.getFirstStructRecursive();
-        if (structUsedByField == null) {
-            return;
-        }
-        if (structUsedByField.getName().equals(structType.getName())) {
-            //this field is using this type!!
-            field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0);
-            field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching());
-        }
-    }
-
-    public void processSchemaFields(Schema schema) {
-        var sdoc = schema.getDocument();
-        if (sdoc == null) return;
-        for (SDDocumentType type : sdoc.getAllTypes()) {
-            if (type.isStruct()) {
-                for (SDField field : schema.allExtraFields()) {
-                    maybePopulateField(sdoc, field, type);
-                }
-            }
-        }
-    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index a037d5046a7..b87bdd8907e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -268,8 +268,8 @@ public class SDDocumentType implements Cloneable, Serializable {
         return field;
     }
 
-    public Field addField(String string, DataType dataType, boolean header, int code) {
-        SDField field = new SDField(this, string, code, dataType, header);
+    public Field addField(String fName, DataType dataType, boolean header, int code) {
+        SDField field = new SDField(this, fName, code, dataType);
         addField(field);
         return field;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index bd6625d4bde..8263352e87f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -111,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
     /** Struct fields defined in this field */
     private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0);
 
-    /** The document that this field was declared in, or null*/
-    private SDDocumentType ownerDocType = null;
+    /** The document that this field was declared in, or null */
+    private SDDocumentType repoDocType = null;
 
     /** The aliases declared for this field. May pertain to indexes or attributes */
     private final Map<String, String> aliasToName = new HashMap<>();
@@ -130,25 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      * @param name the name of the field
      * @param dataType the datatype of the field
     */
-    protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) {
+    public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
         super(name, id, dataType);
-        populate(populate, repo, name, dataType);
+        this.repoDocType = repo;
+        populate(name, dataType);
     }
 
-    public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
-        this(repo, name, id, dataType, true);
+    public SDField(String name, DataType dataType) {
+        this(null, name, dataType);
     }
 
     /** Creates a new field */
-    public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) {
-        super(name, dataType);
-        populate(populate, repo, name, dataType);
+    public SDField(SDDocumentType repo, String name, DataType dataType) {
+        this(repo, name, dataType, null);
     }
 
     /** Creates a new field */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) {
-        super(name, dataType, owner == null ? null : owner.getDocumentType());
-        populate(populate, repo, name, dataType);
+    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
+        this(repo, name, dataType, owner, null, 0);
     }
 
     /**
@@ -159,27 +158,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      * @param owner the owning document (used to check for id collisions)
      * @param fieldMatching the matching object to set for the field
      */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner,
-                      Matching fieldMatching, boolean populate, int recursion) {
+    protected SDField(SDDocumentType repo,
+                      String name,
+                      DataType dataType,
+                      SDDocumentType owner,
+                      Matching fieldMatching,
+                      int recursion)
+    {
         super(name, dataType, owner == null ? null : owner.getDocumentType());
+        this.repoDocType = repo;
+        this.structFieldDepth = recursion;
         if (fieldMatching != null)
             this.setMatching(fieldMatching);
-        populate(populate, repo, name, dataType, fieldMatching, recursion);
+        populate(name, dataType);
     }
 
-    public SDField(SDDocumentType repo, String name, DataType dataType) {
-        this(repo, name, dataType, true);
-    }
+    private int structFieldDepth = 0;
 
-    public SDField(String name, DataType dataType) {
-        this(null, name, dataType);
-    }
-
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) {
-        populate(populate, repo, name, dataType, null, 0);
-    }
-
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching,  int recursion) {
+    private void populate(String name, DataType dataType) {
         if (dataType instanceof TensorDataType) {
             TensorType type = ((TensorDataType)dataType).getTensorType();
             if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
@@ -194,10 +190,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         else {
             addQueryCommand("type " + dataType.getName());
         }
-        if (populate || (dataType instanceof MapDataType)) {
-            populateWithStructFields(repo, name, dataType, recursion);
-            populateWithStructMatching(repo, dataType, fieldMatching);
-        }
     }
 
     public void setIsExtraField(boolean isExtra) {
@@ -273,17 +265,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         }
     }
 
+    private boolean doneStructFields = false;
+
     @SuppressWarnings("deprecation")
-    public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) {
-        DataType dt = getFirstStructOrMapRecursive();
-        if (dt == null) return;
+    private void actuallyMakeStructFields() {
+        if (doneStructFields) return;
+        if (getFirstStructOrMapRecursive() == null) {
+            doneStructFields = true;
+            return;
+        }
+        var sdoc = repoDocType;
+        var dataType = getDataType();
 
         java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> {
             if (structFields.containsKey(fieldName)) return;
-            String subName = name.concat(".").concat(fieldName);
-            var subField = new SDField(sdoc, subName, fieldType,
-                                       ownerDocType, new Matching(),
-                                       true, recursion + 1);
+            String subName = getName().concat(".").concat(fieldName);
+            var subField = new SDField(sdoc, subName, fieldType, null,
+                                       null, structFieldDepth + 1);
             structFields.put(fieldName, subField);
         };
 
@@ -292,15 +290,16 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
             supplyStructField.accept("key", mdt.getKeyType());
             supplyStructField.accept("value", mdt.getValueType());
         } else {
-            if (recursion >= 10) return;
+            if (structFieldDepth >= 10) {
+                // too risky, infinite recursion
+                doneStructFields = true;
+                return;
+            }
             if (dataType instanceof CollectionDataType) {
                 dataType = ((CollectionDataType)dataType).getNestedType();
             }
-            if (dataType instanceof TemporaryStructuredDataType) {
-                SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
-                if (subType == null) {
-                    throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'.");
-                }
+            SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+            if (dataType instanceof TemporaryStructuredDataType && subType != null) {
                 for (Field field : subType.fieldSet()) {
                     supplyStructField.accept(field.getName(), field.getDataType());
                 }
@@ -310,37 +309,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                     supplyStructField.accept(field.getName(), field.getDataType());
                 }
             }
-        }
-    }
-
-    public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) {
-        if (sdoc == null) return;
-        if (superFieldMatching == null) return;
-        DataType dt = getFirstStructOrMapRecursive();
-        if (dt == null) return;
-
-        if (dataType instanceof MapDataType) {
-            // old code here would never do anything useful, should we do something here?
-            return;
-        } else {
-            if (dataType instanceof CollectionDataType) {
-                dataType = ((CollectionDataType)dataType).getNestedType();
+            if ((subType == null) && (structFields.size() > 0)) {
+                throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in "
+                                                   + this + " [" + getDataType() + getDataType().getClass() +
+                                                   "] with " + structFields.size() + " struct fields");
             }
-            if (dataType instanceof StructDataType) {
-                SDDocumentType subType = sdoc.getType(dataType.getName());
-                if (subType == null) {
-                    throw new IllegalArgumentException("Could not find struct " + dataType.getName());
-                }
+            // populate struct fields with matching
+            if (subType != null) {
                 for (Field f : subType.fieldSet()) {
                     if (f instanceof SDField) {
                         SDField field = (SDField) f;
                         Matching subFieldMatching = new Matching();
-                        subFieldMatching.merge(superFieldMatching);
+                        subFieldMatching.merge(this.matching);
                         subFieldMatching.merge(field.getMatching());
                         SDField subField = structFields.get(field.getName());
                         if (subField != null) {
-                            subFieldMatching.merge(subField.getMatching());
-                            subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching);
+                            // we just made this with no matching, so nop:
+                            // subFieldMatching.merge(subField.getMatching());
                             subField.setMatching(subFieldMatching);
                         }
                     } else {
@@ -349,8 +334,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                 }
             }
         }
+        doneStructFields = true;
     }
 
+    private Matching matchingForStructFields = null;
+
     public void addOperation(FieldOperation op) {
         pendingOperations.add(op);
     }
@@ -723,7 +711,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
 
     /** Returns list of static struct fields */
     @Override
-    public Collection<SDField> getStructFields() { return structFields.values(); }
+    public Collection<SDField> getStructFields() {
+        actuallyMakeStructFields();
+        return structFields.values();
+    }
 
     /**
      * Returns a struct field defined in this field,
@@ -732,6 +723,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      */
     @Override
     public SDField getStructField(String name) {
+        actuallyMakeStructFields();
         if (name.contains(".")) {
             String superFieldName = name.substring(0,name.indexOf("."));
             String subFieldName = name.substring(name.indexOf(".")+1);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
index 4ced104fa55..8c17b607f94 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -8,12 +8,12 @@ import com.yahoo.document.DataType;
  */
 public class TemporarySDField extends SDField {
 
-    public TemporarySDField(String name, DataType dataType, SDDocumentType owner) {
-        super(owner, name, dataType, owner, false);
+    public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
+        super(repo, name, dataType, owner);
     }
 
-    public TemporarySDField(String name, DataType dataType) {
-        super(null, name, dataType, false);
+    public TemporarySDField(SDDocumentType repo, String name, DataType dataType) {
+        super(repo, name, dataType);
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
index fe3ac11af27..a5f5f961ab5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
@@ -24,6 +24,8 @@ public class IndexingOperation implements FieldOperation {
         this.script = script;
     }
 
+    public ScriptExpression getScript() { return script; }
+
     public void apply(SDField field) {
         field.setIndexingScript(script);
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
index 36c11b33b23..caeebd65f4f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
@@ -291,13 +291,12 @@ public class ConvertParsedFields {
         schema.addIndex(index);
     }
 
-    SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) {
+    SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) {
         // TODO - can we cleanup this mess
         var structProxy = new SDDocumentType(parsed.name(), schema);
-        structProxy.setStruct(context.resolveStruct(parsed));
         for (var parsedField : parsed.getFields()) {
             var fieldType = context.resolveType(parsedField.getType());
-            var field = new SDField(structProxy, parsedField.name(), fieldType);
+            var field = new SDField(document, parsedField.name(), fieldType);
             convertCommonFieldSettings(field, parsedField);
             structProxy.addField(field);
             if (parsedField.hasIdOverride()) {
@@ -307,6 +306,7 @@ public class ConvertParsedFields {
         for (String inherit : parsed.getInherited()) {
             structProxy.inherit(new DataTypeName(inherit));                
         }
+        structProxy.setStruct(context.resolveStruct(parsed));
         return structProxy;
     }
 
@@ -314,7 +314,7 @@ public class ConvertParsedFields {
         var annType = context.resolveAnnotation(parsed.name());
         var payload = parsed.getStruct();
         if (payload.isPresent()) {
-            var structProxy = convertStructDeclaration(schema, payload.get());
+            var structProxy = convertStructDeclaration(schema, document, payload.get());
             document.addType(structProxy);
         }
         document.addAnnotation(annType);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
index d32fae99f5a..2d9a788cfef 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
@@ -33,7 +33,9 @@ import com.yahoo.vespa.documentmodel.DocumentSummary;
 import com.yahoo.vespa.documentmodel.SummaryField;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -122,6 +124,8 @@ public class ConvertSchemaCollection {
         typeConverter.convert(true);
     }
 
+    private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap();
+
     public List<Schema> convertToSchemas() {
         typeConverter = new ConvertParsedTypes(orderedInput, docMan);
         typeConverter.convert(false);
@@ -150,10 +154,12 @@ public class ConvertSchemaCollection {
     {
         SDDocumentType document = new SDDocumentType(parsed.name());
         for (String inherit : parsed.getInherited()) {
-            document.inherit(new DataTypeName(inherit));
+            var parent = convertedDocuments.get(inherit);
+            assert(parent != null);
+            document.inherit(parent);
         }
         for (var struct : parsed.getStructs()) {
-            var structProxy = fieldConverter.convertStructDeclaration(schema, struct);
+            var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct);
             document.addType(structProxy);
         }
         for (var annotation : parsed.getAnnotations()) {
@@ -165,6 +171,7 @@ public class ConvertSchemaCollection {
                 document.setFieldId(sdf, field.idOverride());
             }
         }
+        convertedDocuments.put(parsed.name(), document);
         schema.addDocument(document);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
index 536caf55111..23b5195486b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java
@@ -12,7 +12,7 @@ import com.yahoo.yolean.Exceptions;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -27,7 +27,7 @@ public class IntermediateCollection {
     private final DeployLogger deployLogger;
     private final ModelContext.Properties modelProperties;
 
-    private Map<String, ParsedSchema> parsedSchemas = new HashMap<>();
+    private Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>();
 
     IntermediateCollection() {
         this.deployLogger = new BaseDeployLogger();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
index 51defffa00b..0be48d1fd25 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -70,7 +70,7 @@ public class AddExtraFieldsToDocument extends Processor {
         if (docField == null) {
             ImmutableSDField existingField = schema.getField(field.getName());
             if (existingField == null) {
-                SDField newField = new SDField(document, field.getName(), field.getDataType(), true);
+                SDField newField = new SDField(document, field.getName(), field.getDataType());
                 newField.setIsExtraField(true);
                 document.addField(newField);
             } else if (!existingField.isImportedField()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
index 0bb1b7da769..d7882c7f8fb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -10,6 +10,7 @@ import com.yahoo.document.PositionDataType;
 import com.yahoo.searchdefinition.Schema;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.GeoPos;
+import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.vespa.documentmodel.SummaryField;
 import com.yahoo.vespa.documentmodel.SummaryTransform;
@@ -35,8 +36,11 @@ import java.util.logging.Level;
  */
 public class CreatePositionZCurve extends Processor {
 
+    private final SDDocumentType repo;
+
     public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
+        this.repo = schema.getDocument();
     }
 
     private boolean useV8GeoPositions = false;
@@ -105,7 +109,7 @@ public class CreatePositionZCurve extends Processor {
                                                     "' already created.");
         }
         boolean isArray = inputField.getDataType() instanceof ArrayDataType;
-        SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
+        SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
         Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY :
                                                                             Attribute.CollectionType.SINGLE);
         attribute.setPosition(true);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
index 84dc6d369fc..7397f9a289c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -61,7 +61,7 @@ public class UriHack extends Processor {
             String partName = uriName + "." + suffix;
             // I wonder if this is explicit in qrs or implicit in backend?
             // search.addFieldSetItem(uriName, partName);
-            SDField partField = new SDField(partName, generatedType);
+            SDField partField = new SDField(schema.getDocument(), partName, generatedType);
             partField.setIndexStructureField(uriField.doesIndexing());
             partField.setRankType(uriField.getRankType());
             partField.setStemming(Stemming.NONE);
diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index 7447eee5cec..ba955f071b2 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -958,10 +958,7 @@ void indexingOperation(ParsedField field, boolean multiLine) : { }
 {
     {
         IndexingOperation oldOp = newIndexingOperation(multiLine);
-        // TODO conversion via SDField is very ugly
-        SDField tmpField = new SDField("temp", com.yahoo.document.DataType.STRING);
-        oldOp.apply(tmpField);
-        ParsedIndexingOp newOp = new ParsedIndexingOp(tmpField.getIndexingScript());
+        ParsedIndexingOp newOp = new ParsedIndexingOp(oldOp.getScript());
         field.setIndexingOperation(newOp);
     }
 }
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 1a0b518bbb2..e90df2776e0 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) :
         if (name != null && Schema.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, document);
+        field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document);
     }
     lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE>
     {
@@ -668,7 +668,7 @@ void fieldSet(Schema schema) :
     |
       ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
     |
-      ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
+      ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
     )(<NL>)*)+
   <RBRACE>
   {
@@ -698,11 +698,13 @@ void annotationOutside(Schema schema) :
     <ANNOTATION> name = identifier()
     {
         type = new SDAnnotationType(name.trim());
+        if (schema.getDocument() == null) {
+            throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
+        }
     }
     [ inheritsAnnotation(type) (<NL>)* ]
-    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE>
     {
-        if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
         schema.addAnnotation(type);
     }
 }
@@ -723,7 +725,7 @@ void annotation(Schema schema, SDDocumentType document) :
         type = new SDAnnotationType(name.trim());
     }
     [ inheritsAnnotation(type) (<NL>)* ]
-    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    lbrace() (type = annotationBody(schema, document, type)) <RBRACE>
     {
         document.addAnnotation(type);
     }
@@ -737,12 +739,12 @@ void annotation(Schema schema, SDDocumentType document) :
  * @param type     the type being built
  * @return a modified or new AnnotationType instance
  */
-SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) :
+SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) :
 {
     SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema);
 }
 {
-    (structFieldDefinition(struct) (<NL>)*)*
+    (structFieldDefinition(repo, struct) (<NL>)*)*
     {
         if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField.
             type = new SDAnnotationType(type.getName(), struct, type.getInherits());
@@ -809,9 +811,13 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
     SDDocumentType struct;
 }
 {
-    ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); }
+    ( <STRUCT> name = identifier() (<NL>)* {
+        if (repo == null)
+            throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
+        struct = new SDDocumentType(name, schema);
+    }
     [ inheritsDocument(struct) (<NL>)* ]
-    lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> )
+    lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> )
     {
         try {
             docMan.getDataType(name);
@@ -819,7 +825,6 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
         } catch (IllegalArgumentException e) {
             // empty
         }
-        if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
         SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName());
         DataType stype = sdtype != null
                          ? sdtype.getStruct()
@@ -828,7 +833,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
         return struct;
     }
 }
-
+        
 /**
  * This rule consumes a data type block from within a field element.
  *
@@ -921,7 +926,7 @@ DataType wildCardType() :
  *
  * @param struct The struct to modify.
  */
-void structFieldDefinition(SDDocumentType struct) :
+void structFieldDefinition(SDDocumentType document, SDDocumentType struct) :
 {
     String name;
     SDField field;
@@ -932,7 +937,7 @@ void structFieldDefinition(SDDocumentType struct) :
         if (name != null && Schema.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, struct);
+        field = new TemporarySDField(document, name, type, struct);
         struct.addField(field);
     }
     lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {