Support references in concrete document types

6 files changed, 145 insertions, 20 deletions

diff --git a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java
index 5d65b25499c..459681d52ae 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java
@@ -151,4 +151,15 @@ public class ReferenceFieldValue extends FieldValue {
     public void deserialize(Field field, FieldReader reader) {
         reader.read(field, this);
     }
+
+    /**
+     * Expose target document ID as this value's wrapped value. Primarily implemented to
+     * allow for transparent interoperability when used in concrete document types.
+     *
+     * @return reference DocumentId, or null if none has been set
+     */
+    @Override
+    public Object getWrappedValue() {
+        return documentId.orElse(null);
+    }
 }
diff --git a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java
index daa65d6e95e..a1d69238463 100644
--- a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java
+++ b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java
@@ -13,6 +13,7 @@ import org.junit.rules.ExpectedException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -189,4 +190,16 @@ public class ReferenceFieldValueTestCase {
         value.assign(newId);
     }
 
+    @Test
+    public void exposed_wrapped_value_is_null_for_empty_reference() {
+        ReferenceFieldValue nullRef = new ReferenceFieldValue(referenceTypeFoo());
+        assertNull(nullRef.getWrappedValue());
+    }
+
+    @Test
+    public void expose_wrapped_value_is_doc_id_for_non_empty_reference() {
+        ReferenceFieldValue idRef = new ReferenceFieldValue(referenceTypeFoo(), docId("id:ns:foo::toad"));
+        assertEquals(docId("id:ns:foo::toad"), idRef.getWrappedValue());
+    }
+
 }
diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd
index 56a9904ca8b..7f9992d8014 100644
--- a/documentgen-test/etc/complex/book.sd
+++ b/documentgen-test/etc/complex/book.sd
@@ -66,6 +66,9 @@ search book {
         field myraw type raw {
 
         }
+        field ref type reference<parent> {
+
+        }
     }
     field sw1 type float {
     }
diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd
new file mode 100644
index 00000000000..35bb650153a
--- /dev/null
+++ b/documentgen-test/etc/complex/parent.sd
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# This type represents a target for a parent-child reference in the `book` type.
+search parent {
+    document parent {
+        field dummy type string {
+
+        }
+    }
+}
\ No newline at end of file
diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
index 59e10bbce00..ff4b3adb1ad 100644
--- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
+++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
@@ -32,6 +32,7 @@ import java.lang.reflect.Modifier;
 import java.nio.ByteBuffer;
 import java.util.*;
 
+import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.junit.Assert.*;
 
 /**
@@ -328,6 +329,61 @@ public class DocumentGenPluginTest {
     }
 
     @Test
+    public void concrete_reference_id_is_initially_null() {
+        final Book book = getBook();
+        assertNull(book.getRef());
+    }
+
+    @Test
+    public void can_set_and_get_concrete_reference_document_id() {
+        final Book book = getBook();
+        final DocumentId docId = new DocumentId("id:ns:parent::foo");
+        book.setRef(docId); // TODO currently no validation of ID upon setRef time
+        assertEquals(docId, book.getRef());
+    }
+
+    @Test
+    public void clearing_document_nulls_out_reference_id() {
+        final Book book = getBook();
+        book.setRef(new DocumentId("id:ns:parent::foo"));
+        book.clear();
+
+        assertNull(book.getRef());
+    }
+
+    @Test
+    public void reference_field_has_correct_reference_data_type() {
+        final Book book = getBook();
+        final Field field = book.getField("ref");
+        assertEquals(ReferenceDataType.createWithInferredId(Parent.type), field.getDataType());
+    }
+
+    @Test
+    public void concrete_reference_id_can_be_transparently_converted_to_field_value() {
+        final Book book = getBook();
+        final DocumentId docId = new DocumentId("id:ns:parent::bar");
+        book.setRef(docId);
+
+        final Field field = book.getField("ref");
+        final FieldValue value = book.getFieldValue(field);
+        assertThat(value, instanceOf(ReferenceFieldValue.class));
+        final ReferenceFieldValue refValue = (ReferenceFieldValue)value;
+        assertEquals(field.getDataType(), refValue.getDataType());
+        assertTrue(refValue.getDocumentId().isPresent());
+        assertEquals(docId, refValue.getDocumentId().get());
+    }
+
+    @Test
+    public void reference_field_value_can_be_transparently_converted_to_concrete_reference_id() {
+        final Book book = getBook();
+        final DocumentId docId = new DocumentId("id:ns:parent::bar");
+        final Field field = book.getField("ref");
+        book.setFieldValue(field, new ReferenceFieldValue((ReferenceDataType)field.getDataType(), docId));
+
+        assertEquals(docId, book.getRef());
+    }
+
+    @Test
     public void testPackDocFromGenericDoc() {
         DocumentType bookGeneric = new DocumentType("book");
         DocumentType somethingElse = new DocumentType("somethingElse");
@@ -379,11 +435,20 @@ public class DocumentGenPluginTest {
         }
     }
 
+    private DocumentTypeManager typeManagerFromSDs(String... files) {
+        final DocumentTypeManager mgr = new DocumentTypeManager();
+        mgr.configure("raw:" + getDocumentConfig(Arrays.asList(files)));
+        return mgr;
+    }
+
+    private DocumentTypeManager typeManagerForBookType() {
+        return typeManagerFromSDs("etc/complex/common.sd", "etc/complex/parent.sd", "etc/complex/book.sd");
+    }
+
     @Test
     @Ignore // Just to test memory usage
     public void testMemUseGeneric() {
-        DocumentTypeManager mgr = new DocumentTypeManager();
-        mgr.configure("raw:" + getDocumentConfig(new ArrayList<String>() {{ add("etc/complex/common.sd"); add("etc/complex/book.sd"); }}));
+        final DocumentTypeManager mgr = typeManagerForBookType();
         DocumentType bookT=mgr.getDocumentType("book");
         List<Document> manyGenericBooks = new ArrayList<>();
         for (int i = 0; i < NUM_BOOKS; i++) {
@@ -399,8 +464,7 @@ public class DocumentGenPluginTest {
     @Test
     @Ignore // Just to test memory usage
     public void testMemUseConcrete() {
-        DocumentTypeManager mgr = new DocumentTypeManager();
-        mgr.configure("raw:" + getDocumentConfig(new ArrayList<String>() {{ add("etc/complex/common.sd"); add("etc/complex/book.sd"); }}));
+        final DocumentTypeManager mgr = typeManagerForBookType();
         List<Book> manyConcreteBooks = new ArrayList<>();
         for (int i = 0; i < NUM_BOOKS; i++) {
             manyConcreteBooks.add(newBookConcrete(i));
@@ -417,9 +481,6 @@ public class DocumentGenPluginTest {
         book.titleSpanTrees().put(t.getName(), t);
         book.setTitle("Moby Dick");
         book.setYear(1851);
-        //Array myAs1 = new Array(DataType.getArray(DataType.STRING));
-        //myAs1.add("as1_1");
-        //myAs1.add("as1_2");
         book.setMystruct(new Ss1().setSs01(new Ss0().setS0("My s0").setD0(99d)).setS1("My s1").setL1(89l));//.setAl1(myAs1));
         Map<Float, Integer> wsFloat = new HashMap<>();
         wsFloat.put(56f, 55);
@@ -488,9 +549,8 @@ public class DocumentGenPluginTest {
 
     @Test
     public void testPackComplex() {
-        DocumentTypeManager mgr = new DocumentTypeManager();
-        mgr.configure("raw:" + getDocumentConfig(new ArrayList<String>() {{ add("etc/complex/common.sd"); add("etc/complex/book.sd"); }}));
-        DocumentType bookT=mgr.getDocumentType("book");
+        final DocumentTypeManager mgr = typeManagerForBookType();
+        DocumentType bookT = mgr.getDocumentType("book");
         Document bookGeneric = new Document(bookT, new DocumentId("doc:book:0"));
         bookGeneric.setFieldValue("author", new StringFieldValue("Melville"));
         StringFieldValue title = new StringFieldValue("Moby Dick");
@@ -591,8 +651,9 @@ public class DocumentGenPluginTest {
         assertEquals(b.getMystruct().getFieldValue("d1").getWrappedValue(), 678d);
         assertEquals(b.getMystruct().getD1(), (Double)678d);
 
-        assertEquals(ConcreteDocumentFactory.documentTypeObjects.size(), 9);
+        assertEquals(ConcreteDocumentFactory.documentTypeObjects.size(), 10);
         assertEquals(ConcreteDocumentFactory.documentTypeObjects.get("music"), Music.type);
+        assertEquals(ConcreteDocumentFactory.documentTypeObjects.get("parent"), Parent.type);
         assertEquals(ConcreteDocumentFactory.documentTypeObjects.get("common"), Common.type);
     }
 
@@ -712,9 +773,17 @@ public class DocumentGenPluginTest {
         assertTrue(Modifier.isAbstract(Emptyannotation.class.getModifiers()));
     }
 
+    private static Document roundtripSerialize(Document docToSerialize, DocumentTypeManager mgr) {
+        DocumentSerializer serializer = DocumentSerializerFactory.create42();
+        serializer.write(docToSerialize);
+        serializer.getBuf().flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(mgr, serializer.getBuf());
+        return new Document(deserializer);
+    }
+
     @Test
     public void testSerialization() {
-        Book book = getBook();
+        final Book book = getBook();
         assertEquals(book.getMystruct().getD1(), (Double)56.777);
         assertEquals(book.getMystruct().getCompressionType(), CompressionType.NONE);
         assertEquals(book.getBody().getFieldCount(), 3);
@@ -723,13 +792,7 @@ public class DocumentGenPluginTest {
         assertEquals(book.getContent().get(0), 3);
         assertEquals(book.getContent().get(1), 4);
         assertEquals(book.getContent().get(2), 5);
-        DocumentSerializer serializer = DocumentSerializerFactory.create42();
-        serializer.write(book);
-        serializer.getBuf().flip();
-        DocumentTypeManager dtm = new DocumentTypeManager();
-        dtm.configure("raw:" + getDocumentConfig(new ArrayList<String>() {{ add("etc/complex/common.sd"); add("etc/complex/book.sd"); }}));
-        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(dtm, serializer.getBuf());
-        Document des = new Document(deserializer);
+        final Document des = roundtripSerialize(book, typeManagerForBookType());
         assertEquals(des.getBody().getFieldCount(), 3);
         assertEquals(des.getHeader().getFieldCount(), 10);
         assertEquals(des.getDataType().getName(), "book");
@@ -770,6 +833,22 @@ public class DocumentGenPluginTest {
     }
 
     @Test
+    public void concrete_reference_fields_can_be_roundtrip_serialized() {
+        final Book book = getBook();
+        final DocumentId id = new DocumentId("id:ns:parent::baz");
+        book.setRef(id);
+
+        final Document doc = roundtripSerialize(book, typeManagerForBookType());
+        final ReferenceFieldValue refValue = (ReferenceFieldValue) doc.getFieldValue(doc.getField("ref"));
+        assertTrue(refValue.getDocumentId().isPresent());
+        assertEquals(id, refValue.getDocumentId().get());
+
+        final Book bookCopy = (Book)new ConcreteDocumentFactory().getDocumentCopy(
+                "book", doc, new DocumentId("id:ns:book::helloworld"));
+        assertEquals(id, bookCopy.getRef());
+    }
+
+    @Test
     public void testInheritanceOfGeneratedTypes() {
         assertEquals(Music3.class.getSuperclass(), Document.class);
         assertEquals(Music4.class.getSuperclass(), Document.class);
diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
index a1ba05eacb3..67627ca9410 100644
--- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
+++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
@@ -330,7 +330,8 @@ public class DocumentGenMojo extends AbstractMojo {
             out.write("}\n");
             annotationTypes.put(annType.getName(), packageName+".annotation."+className);
         } catch (IOException e) {
-            throw new RuntimeException("Could not export sources for annotation type '"+annType.getName()+"'", e);        }
+            throw new RuntimeException("Could not export sources for annotation type '"+annType.getName()+"'", e);
+        }
     }
 
     /**
@@ -837,6 +838,9 @@ public class DocumentGenMojo extends AbstractMojo {
         if (dt instanceof ArrayDataType) return "java.util.List<"+toJavaType(((ArrayDataType)dt).getNestedType())+">";
         if (dt instanceof MapDataType) return "java.util.Map<"+toJavaType(((MapDataType)dt).getKeyType())+","+toJavaType(((MapDataType)dt).getValueType())+">";
         if (dt instanceof AnnotationReferenceDataType) return className(((AnnotationReferenceDataType) dt).getAnnotationType().getName());
+        if (dt instanceof ReferenceDataType) {
+            return "com.yahoo.document.DocumentId";
+        }
         return "byte[]";
     }
 
@@ -862,6 +866,11 @@ public class DocumentGenMojo extends AbstractMojo {
         // For annotation references and generated types, the references are to the actual objects of the correct types, so most likely this is never needed,
         // but there might be scenarios where we want to look up the AnnotationType in the AnnotationTypeRegistry here instead.
         if (dt instanceof AnnotationReferenceDataType) return "new com.yahoo.document.annotation.AnnotationReferenceDataType(new com.yahoo.document.annotation.AnnotationType(\""+((AnnotationReferenceDataType)dt).getAnnotationType().getName()+"\"))";
+        if (dt instanceof ReferenceDataType) {
+            // All concrete document types have a public `type` constant with their DocumentType.
+            return String.format("new com.yahoo.document.ReferenceDataType(%s.type, %d)",
+                    className(((ReferenceDataType) dt).getTargetType().getName()), dt.getId());
+        }
         return "DataType.RAW";
     }