Add deriving of reference data type in documentmanager config.

6 files changed, 324 insertions, 0 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
index 2a723596255..372a8d480e9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
@@ -122,6 +122,8 @@ public class SDDocumentTypeOrderer {
             //do nothing
         } else if (type instanceof TensorDataType) {
             //do nothing
+        } else if (type instanceof ReferenceDataType) {
+            //do nothing
         } else {
             deployLogger.log(Level.WARNING, "Unknown type : " + type);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 705ec4dc4a1..dc8517e7852 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -145,6 +145,9 @@ public class DocumentManager {
             // Nothing to do; the type of the tensor is instead stored in each field as detailed type information
             // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
             // without changing the field type and thus requiring data refeed
+        } else if (type instanceof ReferenceDataType) {
+            ReferenceDataType refType = (ReferenceDataType) type;
+            builder.referencetype(new Datatype.Referencetype.Builder().target_type_id(refType.getTargetType().getId()));
         } else {
             throw new IllegalArgumentException("Can not create config for data type '" + type.getName());
         }
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg
new file mode 100644
index 00000000000..3d8c7139aa0
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg
@@ -0,0 +1,47 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id -1895788438
+datatype[1].referencetype[0].target_type_id 2987301
+datatype[2].id 959075962
+datatype[2].structtype[0].name "ad.header"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].compresstype NONE
+datatype[2].structtype[0].compresslevel 0
+datatype[2].structtype[0].compressthreshold 95
+datatype[2].structtype[0].compressminsize 800
+datatype[2].structtype[0].field[0].name "self_ref"
+datatype[2].structtype[0].field[0].datatype -1895788438
+datatype[2].structtype[0].field[0].detailedtype ""
+datatype[2].structtype[0].field[1].name "rankfeatures"
+datatype[2].structtype[0].field[1].datatype 2
+datatype[2].structtype[0].field[1].detailedtype ""
+datatype[2].structtype[0].field[2].name "summaryfeatures"
+datatype[2].structtype[0].field[2].datatype 2
+datatype[2].structtype[0].field[2].detailedtype ""
+datatype[3].id -255288561
+datatype[3].structtype[0].name "ad.body"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[4].id 2987301
+datatype[4].documenttype[0].name "ad"
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0].name "document"
+datatype[4].documenttype[0].inherits[0].version 0
+datatype[4].documenttype[0].headerstruct 959075962
+datatype[4].documenttype[0].bodystruct -255288561
+datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "self_ref"
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg
new file mode 100644
index 00000000000..81498f03139
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg
@@ -0,0 +1,107 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 542332920
+datatype[2].referencetype[0].target_type_id 443162583
+datatype[3].id 959075962
+datatype[3].structtype[0].name "ad.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "campaign_ref"
+datatype[3].structtype[0].field[0].datatype 595216861
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "person_ref"
+datatype[3].structtype[0].field[1].datatype 542332920
+datatype[3].structtype[0].field[1].detailedtype ""
+datatype[3].structtype[0].field[2].name "rankfeatures"
+datatype[3].structtype[0].field[2].datatype 2
+datatype[3].structtype[0].field[2].detailedtype ""
+datatype[3].structtype[0].field[3].name "summaryfeatures"
+datatype[3].structtype[0].field[3].datatype 2
+datatype[3].structtype[0].field[3].detailedtype ""
+datatype[4].id -255288561
+datatype[4].structtype[0].name "ad.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[5].id 2987301
+datatype[5].documenttype[0].name "ad"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0].name "document"
+datatype[5].documenttype[0].inherits[0].version 0
+datatype[5].documenttype[0].headerstruct 959075962
+datatype[5].documenttype[0].bodystruct -255288561
+datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+datatype[6].id -2041471955
+datatype[6].structtype[0].name "campaign.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "rankfeatures"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].detailedtype ""
+datatype[6].structtype[0].field[1].name "summaryfeatures"
+datatype[6].structtype[0].field[1].datatype 2
+datatype[6].structtype[0].field[1].detailedtype ""
+datatype[7].id 1448849794
+datatype[7].structtype[0].name "campaign.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[8].id -1318255918
+datatype[8].documenttype[0].name "campaign"
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0].name "document"
+datatype[8].documenttype[0].inherits[0].version 0
+datatype[8].documenttype[0].headerstruct -2041471955
+datatype[8].documenttype[0].bodystruct 1448849794
+datatype[9].id 3129224
+datatype[9].structtype[0].name "person.header"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "rankfeatures"
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].detailedtype ""
+datatype[9].structtype[0].field[1].name "summaryfeatures"
+datatype[9].structtype[0].field[1].datatype 2
+datatype[9].structtype[0].field[1].detailedtype ""
+datatype[10].id -2003767395
+datatype[10].structtype[0].name "person.body"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].compresstype NONE
+datatype[10].structtype[0].compresslevel 0
+datatype[10].structtype[0].compressthreshold 95
+datatype[10].structtype[0].compressminsize 800
+datatype[11].id 443162583
+datatype[11].documenttype[0].name "person"
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0].name "document"
+datatype[11].documenttype[0].inherits[0].version 0
+datatype[11].documenttype[0].headerstruct 3129224
+datatype[11].documenttype[0].bodystruct -2003767395
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg
new file mode 100644
index 00000000000..181da48eac4
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg
@@ -0,0 +1,78 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 959075962
+datatype[2].structtype[0].name "ad.header"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].compresstype NONE
+datatype[2].structtype[0].compresslevel 0
+datatype[2].structtype[0].compressthreshold 95
+datatype[2].structtype[0].compressminsize 800
+datatype[2].structtype[0].field[0].name "campaign_ref"
+datatype[2].structtype[0].field[0].datatype 595216861
+datatype[2].structtype[0].field[0].detailedtype ""
+datatype[2].structtype[0].field[1].name "other_campaign_ref"
+datatype[2].structtype[0].field[1].datatype 595216861
+datatype[2].structtype[0].field[1].detailedtype ""
+datatype[2].structtype[0].field[2].name "rankfeatures"
+datatype[2].structtype[0].field[2].datatype 2
+datatype[2].structtype[0].field[2].detailedtype ""
+datatype[2].structtype[0].field[3].name "summaryfeatures"
+datatype[2].structtype[0].field[3].datatype 2
+datatype[2].structtype[0].field[3].detailedtype ""
+datatype[3].id -255288561
+datatype[3].structtype[0].name "ad.body"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[4].id 2987301
+datatype[4].documenttype[0].name "ad"
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0].name "document"
+datatype[4].documenttype[0].inherits[0].version 0
+datatype[4].documenttype[0].headerstruct 959075962
+datatype[4].documenttype[0].bodystruct -255288561
+datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref"
+datatype[5].id -2041471955
+datatype[5].structtype[0].name "campaign.header"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].compresstype NONE
+datatype[5].structtype[0].compresslevel 0
+datatype[5].structtype[0].compressthreshold 95
+datatype[5].structtype[0].compressminsize 800
+datatype[5].structtype[0].field[0].name "rankfeatures"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].detailedtype ""
+datatype[5].structtype[0].field[1].name "summaryfeatures"
+datatype[5].structtype[0].field[1].datatype 2
+datatype[5].structtype[0].field[1].detailedtype ""
+datatype[6].id 1448849794
+datatype[6].structtype[0].name "campaign.body"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[7].id -1318255918
+datatype[7].documenttype[0].name "campaign"
+datatype[7].documenttype[0].version 0
+datatype[7].documenttype[0].inherits[0].name "document"
+datatype[7].documenttype[0].inherits[0].version 0
+datatype[7].documenttype[0].headerstruct -2041471955
+datatype[7].documenttype[0].bodystruct 1448849794
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
new file mode 100644
index 00000000000..77631012972
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
@@ -0,0 +1,87 @@
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.searchdefinition.TestUtils.joinLines;
+
+/**
+ * @author geirst
+ */
+public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase {
+
+    @Test
+    public void reference_fields_can_reference_other_document_types() throws ParseException, IOException {
+        assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines(
+                "search ad {",
+                "  document ad {",
+                "    field campaign_ref type reference<campaign> {}",
+                "    field person_ref type reference<person> {}",
+                "  }",
+                "}")),
+                "documentmanager_refs_to_other_types.cfg");
+    }
+
+    @Test
+    public void reference_fields_can_reference_same_document_type_multiple_times() throws ParseException, IOException {
+        assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().build(joinLines(
+                "search ad {",
+                "  document ad {",
+                "    field campaign_ref type reference<campaign> {}",
+                "    field other_campaign_ref type reference<campaign> {}",
+                "  }",
+                "}")),
+                "documentmanager_refs_to_same_type.cfg");
+    }
+
+    @Test
+    public void reference_field_can_reference_self_document_type() throws ParseException, IOException {
+        assertDocumentConfigs(new TestDocumentModelBuilder().build(joinLines(
+                "search ad {",
+                "  document ad {",
+                "    field self_ref type reference<ad> {}",
+                "  }",
+                "}")),
+                "documentmanager_ref_to_self_type.cfg");
+    }
+
+    private static String TEST_FOLDER = "src/test/configmodel/types/references/";
+
+    private void assertDocumentConfigs(DocumentModel model,
+                                       String documentmanagerCfgFile) throws IOException {
+        assertDocumentmanagerCfg(model, documentmanagerCfgFile);
+    }
+
+    private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
+        DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
+        assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
+    }
+
+    private static class TestDocumentModelBuilder {
+        private final SearchBuilder builder = new SearchBuilder();
+        public TestDocumentModelBuilder addCampaign() throws ParseException {
+            builder.importString(joinLines("search campaign {",
+                    "  document campaign {}",
+                    "}"));
+            return this;
+        }
+        public TestDocumentModelBuilder addPerson() throws ParseException {
+            builder.importString(joinLines("search person {",
+                    "  document person {}",
+                    "}"));
+            return this;
+        }
+        public DocumentModel build(String adSdContent) throws ParseException {
+            builder.importString(adSdContent);
+            builder.build();
+            return builder.getModel();
+        }
+    }
+
+}