Merge pull request #13875 from vespa-engine/geirst/inherited-reference-fields

Ensure that reference fields can be inherited from super document types.

11 files changed, 281 insertions, 1 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
index 0d0da71bd0f..7f1a9933188 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
@@ -6,6 +6,7 @@ import com.yahoo.document.ReferenceDataType;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -30,10 +31,24 @@ public class DocumentReferenceResolver {
     }
 
     public void resolveReferences(SDDocumentType documentType) {
-        DocumentReferences references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType));
+        var references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType));
         documentType.setDocumentReferences(references);
     }
 
+    public void resolveInheritedReferences(SDDocumentType documentType) {
+        resolveInheritedReferencesRecursive(documentType, documentType.getInheritedTypes());
+    }
+
+    private void resolveInheritedReferencesRecursive(SDDocumentType documentType,
+                                                     Collection<SDDocumentType> inheritedTypes) {
+        for (var inheritedType : inheritedTypes) {
+            documentType.getDocumentReferences().get().mergeFrom(inheritedType.getDocumentReferences().get());
+        }
+        for (var inheritedType : inheritedTypes) {
+            resolveInheritedReferencesRecursive(documentType, inheritedType.getInheritedTypes());
+        }
+    }
+
     private Map<String, DocumentReference> createFieldToDocumentReferenceMapping(SDDocumentType documentType) {
         return fieldStream(documentType)
                 .filter(field -> field.getDataType() instanceof ReferenceDataType)
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java
index 37f5ab1bbde..4a3995b2d40 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java
@@ -18,6 +18,10 @@ public class DocumentReferences implements Iterable<Map.Entry<String, DocumentRe
         this.references = references;
     }
 
+    public void mergeFrom(DocumentReferences other) {
+        references.putAll(other.references);
+    }
+
     @Override
     public Iterator<Map.Entry<String, DocumentReference>> iterator() {
         return Collections.unmodifiableSet(references.entrySet()).iterator();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index eb68e6af203..1fab30f9ea4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -239,6 +239,7 @@ public class SearchBuilder {
 
         var resolver = new DocumentReferenceResolver(searchList);
         sdocs.forEach(resolver::resolveReferences);
+        sdocs.forEach(resolver::resolveInheritedReferences);
         var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
         sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
 
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg
new file mode 100644
index 00000000000..1f4d5619248
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg
@@ -0,0 +1,108 @@
+attribute[].name "ref_from_a"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "ref_from_b"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "from_a_int_field"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "from_b_int_field"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd
new file mode 100644
index 00000000000..75c16d1eefe
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd
@@ -0,0 +1,8 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+schema child_a {
+  document child_a {
+    field ref_from_a type reference<parent> {
+      indexing: attribute
+    }
+  }
+}
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd
new file mode 100644
index 00000000000..b4349fcc65d
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd
@@ -0,0 +1,8 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+schema child_b {
+  document child_b inherits child_a {
+    field ref_from_b type reference<parent> {
+      indexing: attribute
+    }
+  }
+}
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd
new file mode 100644
index 00000000000..f798b8c3446
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd
@@ -0,0 +1,7 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+schema child_c {
+  document child_c inherits child_b {
+  }
+  import field ref_from_a.int_field as from_a_int_field {}
+  import field ref_from_b.int_field as from_b_int_field {}
+}
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg
new file mode 100644
index 00000000000..ca490b053f7
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg
@@ -0,0 +1,110 @@
+enablecompression false
+documenttype[].id -94853056
+documenttype[].name "child_a"
+documenttype[].version 0
+documenttype[].headerstruct 867409663
+documenttype[].bodystruct 0
+documenttype[].inherits[].id 8
+documenttype[].datatype[].id 867409663
+documenttype[].datatype[].type STRUCT
+documenttype[].datatype[].array.element.id 0
+documenttype[].datatype[].map.key.id 0
+documenttype[].datatype[].map.value.id 0
+documenttype[].datatype[].wset.key.id 0
+documenttype[].datatype[].wset.createifnonexistent false
+documenttype[].datatype[].wset.removeifzero false
+documenttype[].datatype[].annotationref.annotation.id 0
+documenttype[].datatype[].sstruct.name "child_a.header"
+documenttype[].datatype[].sstruct.version 0
+documenttype[].datatype[].sstruct.compression.type NONE
+documenttype[].datatype[].sstruct.compression.level 0
+documenttype[].datatype[].sstruct.compression.threshold 95
+documenttype[].datatype[].sstruct.compression.minsize 200
+documenttype[].datatype[].sstruct.field[].name "ref_from_a"
+documenttype[].datatype[].sstruct.field[].id 300427062
+documenttype[].datatype[].sstruct.field[].datatype 427398467
+documenttype[].datatype[].sstruct.field[].detailedtype ""
+documenttype[].fieldsets{[]}.fields[] "ref_from_a"
+documenttype[].referencetype[].id 427398467
+documenttype[].referencetype[].target_type_id 1175161836
+documenttype[].id -94852095
+documenttype[].name "child_b"
+documenttype[].version 0
+documenttype[].headerstruct 670896158
+documenttype[].bodystruct 0
+documenttype[].inherits[].id 8
+documenttype[].inherits[].id -94853056
+documenttype[].datatype[].id 670896158
+documenttype[].datatype[].type STRUCT
+documenttype[].datatype[].array.element.id 0
+documenttype[].datatype[].map.key.id 0
+documenttype[].datatype[].map.value.id 0
+documenttype[].datatype[].wset.key.id 0
+documenttype[].datatype[].wset.createifnonexistent false
+documenttype[].datatype[].wset.removeifzero false
+documenttype[].datatype[].annotationref.annotation.id 0
+documenttype[].datatype[].sstruct.name "child_b.header"
+documenttype[].datatype[].sstruct.version 0
+documenttype[].datatype[].sstruct.compression.type NONE
+documenttype[].datatype[].sstruct.compression.level 0
+documenttype[].datatype[].sstruct.compression.threshold 95
+documenttype[].datatype[].sstruct.compression.minsize 200
+documenttype[].datatype[].sstruct.field[].name "ref_from_b"
+documenttype[].datatype[].sstruct.field[].id 185778735
+documenttype[].datatype[].sstruct.field[].datatype 427398467
+documenttype[].datatype[].sstruct.field[].detailedtype ""
+documenttype[].fieldsets{[]}.fields[] "ref_from_a"
+documenttype[].fieldsets{[]}.fields[] "ref_from_b"
+documenttype[].id -94851134
+documenttype[].name "child_c"
+documenttype[].version 0
+documenttype[].headerstruct 474382653
+documenttype[].bodystruct 0
+documenttype[].inherits[].id 8
+documenttype[].inherits[].id -94852095
+documenttype[].datatype[].id 474382653
+documenttype[].datatype[].type STRUCT
+documenttype[].datatype[].array.element.id 0
+documenttype[].datatype[].map.key.id 0
+documenttype[].datatype[].map.value.id 0
+documenttype[].datatype[].wset.key.id 0
+documenttype[].datatype[].wset.createifnonexistent false
+documenttype[].datatype[].wset.removeifzero false
+documenttype[].datatype[].annotationref.annotation.id 0
+documenttype[].datatype[].sstruct.name "child_c.header"
+documenttype[].datatype[].sstruct.version 0
+documenttype[].datatype[].sstruct.compression.type NONE
+documenttype[].datatype[].sstruct.compression.level 0
+documenttype[].datatype[].sstruct.compression.threshold 95
+documenttype[].datatype[].sstruct.compression.minsize 200
+documenttype[].fieldsets{[]}.fields[] "ref_from_a"
+documenttype[].fieldsets{[]}.fields[] "ref_from_b"
+documenttype[].importedfield[].name "from_a_int_field"
+documenttype[].importedfield[].name "from_b_int_field"
+documenttype[].id 1175161836
+documenttype[].name "parent"
+documenttype[].version 0
+documenttype[].headerstruct 836075987
+documenttype[].bodystruct 0
+documenttype[].inherits[].id 8
+documenttype[].datatype[].id 836075987
+documenttype[].datatype[].type STRUCT
+documenttype[].datatype[].array.element.id 0
+documenttype[].datatype[].map.key.id 0
+documenttype[].datatype[].map.value.id 0
+documenttype[].datatype[].wset.key.id 0
+documenttype[].datatype[].wset.createifnonexistent false
+documenttype[].datatype[].wset.removeifzero false
+documenttype[].datatype[].annotationref.annotation.id 0
+documenttype[].datatype[].sstruct.name "parent.header"
+documenttype[].datatype[].sstruct.version 0
+documenttype[].datatype[].sstruct.compression.type NONE
+documenttype[].datatype[].sstruct.compression.level 0
+documenttype[].datatype[].sstruct.compression.threshold 95
+documenttype[].datatype[].sstruct.compression.minsize 200
+documenttype[].datatype[].sstruct.field[].name "int_field"
+documenttype[].datatype[].sstruct.field[].id 2128822283
+documenttype[].datatype[].sstruct.field[].datatype 0
+documenttype[].datatype[].sstruct.field[].detailedtype ""
+documenttype[].fieldsets{[]}.fields[] "int_field"
+
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg
new file mode 100644
index 00000000000..e574bfa2393
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg
@@ -0,0 +1,6 @@
+attribute[].name "from_a_int_field"
+attribute[].referencefield "ref_from_a"
+attribute[].targetfield "int_field"
+attribute[].name "from_b_int_field"
+attribute[].referencefield "ref_from_b"
+attribute[].targetfield "int_field"
diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd
new file mode 100644
index 00000000000..0509f1a8565
--- /dev/null
+++ b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd
@@ -0,0 +1,8 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+schema parent {
+  document parent {
+    field int_field type int {
+      indexing: attribute
+    }
+  }
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
index b06745f9a75..a0dd89229dd 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
@@ -30,4 +30,9 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase {
     public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException {
         assertCorrectDeriving("imported_position_field_summary", "child");
     }
+
+    @Test
+    public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException {
+        assertCorrectDeriving("imported_fields_inherited_reference", "child_c");
+    }
 }