Merge pull request #21780 from vespa-engine/arnej/split-temporary-structured-type

Arnej/split temporary structured type

16 files changed, 132 insertions, 23 deletions

diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java
index 65c282e01e2..702ab835dd4 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java
@@ -5,7 +5,6 @@ import com.yahoo.document.DataType;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.StructuredDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.ReferenceFieldValue;
 
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index dd8edbdde6c..ff5930f8f6a 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -7,7 +7,6 @@ import com.yahoo.document.Field;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.StructuredDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.document.annotation.AnnotationTypeRegistry;
 import com.yahoo.document.datatypes.FieldValue;
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java b/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java
new file mode 100644
index 00000000000..536c10ee242
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java
@@ -0,0 +1,41 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.StructDataType;
+
+/**
+ * Proxy for a struct type declared in a specific document
+ *
+ * @author arnej
+ **/
+public final class OwnedTemporaryType extends StructDataType implements OwnedType {
+
+    private final String ownerName;
+    private final String uniqueName;
+
+    public OwnedTemporaryType(String name, DocumentType document) {
+        this(name, document.getName());
+    }
+
+    public OwnedTemporaryType(String name, String owner) {
+        super(name);
+        this.ownerName = owner;
+        this.uniqueName = name + "@" + owner;
+    }
+
+    @Override
+    public String getOwnerName() {
+        return ownerName;
+    }
+
+    @Override
+    public String getUniqueName() {
+        return uniqueName;
+    }
+
+    @Override
+    public String toString() {
+        return "{OwnedTemporaryType "+uniqueName+" id="+getId()+" uid="+getUniqueId()+"}";
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java b/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java
new file mode 100644
index 00000000000..e3a91692ca8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java
@@ -0,0 +1,15 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+/**
+ * API for a type declared in a specific document
+ *
+ * @author arnej
+ **/
+public interface OwnedType {
+    String getOwnerName();
+    String getUniqueName();
+    default int getUniqueId() {
+        return getUniqueName().hashCode();
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java b/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java
new file mode 100644
index 00000000000..66f6354b3f5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java
@@ -0,0 +1,21 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.StructDataType;
+
+/**
+ * Proxy for an unknown type (must resolve to struct or document)
+ *
+ * @author arnej
+ **/
+public final class TemporaryUnknownType extends StructDataType {
+
+    public TemporaryUnknownType(String name) {
+        super(name);
+    }
+
+    @Override
+    public String toString() {
+        return "{TemporaryUnknownType "+getName()+" id="+getId()+"}";
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index 4660b81ff72..1ac3f80c743 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -7,7 +7,6 @@ import com.yahoo.document.DataType;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
-import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.StructuredDataType;
 import com.yahoo.document.TemporaryStructuredDataType;
@@ -15,7 +14,10 @@ import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.annotation.AnnotationReferenceDataType;
 import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.documentmodel.DataTypeCollection;
+import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.OwnedTemporaryType;
+import com.yahoo.documentmodel.TemporaryUnknownType;
 import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDDocumentType;
@@ -232,13 +234,26 @@ public class DocumentModelBuilder {
         }
         DataType original = type;
         if (type instanceof TemporaryStructuredDataType) {
+            throw new IllegalArgumentException("Cannot handle temporary: " + type);
+        }
+        if (type instanceof TemporaryUnknownType) {
+            // must be a known struct or document type
             DataType other = repo.getDataType(type.getId());
             if (other == null || other == type) {
+                // maybe it is the name of a document type:
                 other = getDocumentType(docs, type.getName());
             }
-            if (other != null) {
-                type = other;
+            if (other == null) {
+                throw new IllegalArgumentException("No replacement found for temporary type: " + type);
+            }
+            type = other;
+        } else if (type instanceof OwnedTemporaryType) {
+            // must be replaced with the real struct type
+            DataType other = repo.getDataType(type.getId());
+            if (other == null || other == type) {
+                throw new IllegalArgumentException("No replacement found for temporary type: " + type);
             }
+            type = other;
         } else if (type instanceof DocumentType) {
             DataType other = getDocumentType(docs, type.getName());
             if (other != null) {
@@ -532,11 +547,13 @@ public class DocumentModelBuilder {
 
         @SuppressWarnings("deprecation")
         private StructDataType handleStruct(SDDocumentType type) {
+            // System.err.println("handle struct " + type + " for doc " + targetDt.getName());
             if (type.isStruct()) {
                 var st = type.getStruct();
                 if (st.getName().equals(type.getName()) &&
                     (st instanceof StructDataType) &&
-                    ! (st instanceof TemporaryStructuredDataType))
+                    (! (st instanceof TemporaryUnknownType)) &&
+                    (! (st instanceof OwnedTemporaryType)))
                     {
                         return handleStruct((StructDataType) st);
                     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index c592e4842a9..b466a78c69b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -8,7 +8,6 @@ import com.yahoo.document.DocumentType;
 import com.yahoo.document.PrimitiveDataType;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.document.StructuredDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.datatypes.BoolFieldValue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index 621e7ce8571..49ae00d0663 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -7,9 +7,10 @@ import com.yahoo.document.DocumentType;
 import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.StructDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.documentmodel.OwnedTemporaryType;
+import com.yahoo.documentmodel.TemporaryUnknownType;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
 import com.yahoo.language.simple.SimpleLinguistics;
@@ -307,7 +308,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                 return;
             }
             SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
-            if (dataType instanceof TemporaryStructuredDataType && subType != null) {
+            if (dataType instanceof TemporaryUnknownType && subType != null) {
+                for (Field field : subType.fieldSet()) {
+                    supplyStructField.accept(field.getName(), field.getDataType());
+                }
+            } else if (dataType instanceof OwnedTemporaryType && subType != null) {
                 for (Field field : subType.fieldSet()) {
                     supplyStructField.accept(field.getName(), field.getDataType());
                 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index ff311795089..8867ab63e2b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -9,6 +9,8 @@ import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.documentmodel.DataTypeCollection;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.OwnedTemporaryType;
+import com.yahoo.documentmodel.TemporaryUnknownType;
 import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.FieldSet;
 import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -83,6 +85,12 @@ public class DocumentManager {
             if (dataType instanceof TemporaryStructuredDataType) {
                 throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName());
             }
+            if (dataType instanceof TemporaryUnknownType) {
+                throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName());
+            }
+            if (dataType instanceof OwnedTemporaryType) {
+                throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName());
+            }
             if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) {
                 Datatype.Builder dataTypeBuilder = new Datatype.Builder();
                 documentConfigBuilder.datatype(dataTypeBuilder);
@@ -144,8 +152,6 @@ public class DocumentManager {
             }
             buildConfig(dt.getFieldSets(), doc);
             buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
-        } else if (type instanceof TemporaryStructuredDataType) {
-            throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
         } else if (type instanceof StructDataType) {
             StructDataType structType = (StructDataType) type;
             Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder();
@@ -348,7 +354,11 @@ public class DocumentManager {
         indexMap.setDone(type);
         if (type instanceof TemporaryStructuredDataType) {
             throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
-        } if (type instanceof StructDataType) {
+        } else if (type instanceof TemporaryUnknownType) {
+            throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
+        } else if (type instanceof OwnedTemporaryType) {
+            throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
+        } else if (type instanceof StructDataType) {
             docTypeBuildOneType((StructDataType) type, documentBuilder, indexMap);
         } else if (type instanceof ArrayDataType) {
             docTypeBuildOneType((ArrayDataType) type, documentBuilder, indexMap);
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index 1240dac8bf1..e1a28c8114f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -8,6 +8,8 @@ import com.yahoo.document.annotation.AnnotationType;
 import com.yahoo.documentmodel.DataTypeCollection;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
 import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.OwnedTemporaryType;
+import com.yahoo.documentmodel.TemporaryUnknownType;
 import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.FieldSet;
 import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -111,6 +113,12 @@ public class DocumentTypes {
             built.add(type.getId());
             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder();
             dataTypeBuilder.id(type.getId());
+            if (type instanceof TemporaryUnknownType) {
+                throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
+            }
+            if (type instanceof OwnedTemporaryType) {
+                throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
+            }
             if (type instanceof StructDataType) {
                 buildConfig((StructDataType) type, dataTypeBuilder, documentBuilder, built);
             } else if (type instanceof ArrayDataType) {
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 018531616fb..ab0cdefc355 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -826,9 +826,10 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
             // empty
         }
         SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName());
-        DataType stype = sdtype != null
-                         ? sdtype.getStruct()
-                         : TemporaryStructuredDataType.create(struct.getName());
+        if (sdtype != null) {
+            throw new ParseException("Struct '" + name + "' is already defined.");
+        }
+        DataType stype = new OwnedTemporaryType(name, repo.getName());
         struct.setStruct(stype);
         return struct;
     }
@@ -847,7 +848,6 @@ DataType dataType() :
     DataType arrayType = null;
     DataType wsetType = null;
     TensorType tensorType;
-    TemporaryStructuredDataType referenceType;
     String referencedDoc;
 }
 {
@@ -863,9 +863,9 @@ DataType dataType() :
         DataType type = VespaDocumentType.INSTANCE.getDataType(typeName);
 
         if (type == null) {
-            // we are basically creating TemporaryStructDataType instances for ANYTHING here!!
+            // we are basically creating TemporaryUnknownType instances for ANYTHING here!!
             // we must do this and clean them up later.
-            type = TemporaryStructuredDataType.create(typeName);
+            type = new TemporaryUnknownType(typeName);
         }
 
         if (isArrayOldStyle) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
index 8c8cb59e9d9..5bbb751585b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
@@ -3,7 +3,6 @@ package com.yahoo.searchdefinition;
 
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.document.TemporarySDField;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index b23ad4eb5b6..fe1b19be64b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -4,7 +4,6 @@ package com.yahoo.searchdefinition;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.document.DataType;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import org.junit.Rule;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
index ed3f5cb0ba6..11e21c7915d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java
@@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.derived;
 
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.DocumentReference;
 import com.yahoo.searchdefinition.DocumentReferences;
 import com.yahoo.searchdefinition.Schema;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
index 2d6b3acc4dd..9855ca30ebc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
@@ -5,7 +5,6 @@ import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.Application;
 import com.yahoo.searchdefinition.Schema;
 import com.yahoo.searchdefinition.document.SDDocumentType;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
index 0b615595794..6ddacd066b1 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
@@ -7,7 +7,6 @@ import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.document.DataType;
 import com.yahoo.documentmodel.NewDocumentReferenceDataType;
-import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.Application;
 import com.yahoo.searchdefinition.DocumentReference;
 import com.yahoo.searchdefinition.DocumentReferences;