Merge pull request #6965 from vespa-engine/revert-6964-revert-6960-balder/ensure-the-extra-fields-are-included-in-the-concrete-documents-too

Revert "Revert "Use a normal SearchBuilder to get all sanity checks and preprocessing…""

12 files changed, 69 insertions, 37 deletions

diff --git a/documentgen-test/etc/complex/music.sd b/documentgen-test/etc/complex/music.sd
index 6dbb7b76862..b95edd2b4f3 100644
--- a/documentgen-test/etc/complex/music.sd
+++ b/documentgen-test/etc/complex/music.sd
@@ -46,12 +46,12 @@ search music {
 
    field sw1 type float {
       indexing {
-        input weight * 6 + input w1 + input w2 | summary;
+        input weight_src * 6 + input w1_src + input w2_src | summary;
       }
    }
 
    field didinteger type array<int> {
-      indexing: input did | split " " | attribute
+      indexing: input did | split " " | for_each { to_int } | attribute
    }
 
    rank-profile default {
diff --git a/documentgen-test/etc/complex/music2.sd b/documentgen-test/etc/complex/music2.sd
index e608225bf38..4cc9db0651e 100644
--- a/documentgen-test/etc/complex/music2.sd
+++ b/documentgen-test/etc/complex/music2.sd
@@ -51,12 +51,12 @@ search music2 {
 
    field sw1 type float {
       indexing {
-        input weight * 6 + input w1 + input w2 | summary;
+        input weight_src * 6 + input w1_src + input w2_src | summary;
       }
    }
 
    field didinteger type array<int> {
-      indexing: input did | split " " | attribute
+      indexing: input did | split " " | for_each { to_int } | attribute
    }
 
    rank-profile default {
diff --git a/documentgen-test/etc/complex/video.sd b/documentgen-test/etc/complex/video.sd
index fc7f58298c1..0749daa01aa 100644
--- a/documentgen-test/etc/complex/video.sd
+++ b/documentgen-test/etc/complex/video.sd
@@ -32,7 +32,7 @@ search video {
 
    field sw1 type float {
       indexing {
-        input weight * 6 + input w1 + input w2 | summary;
+        input weight_src * 6 + input w1_src + input w2_src | summary;
       }
    }
 
diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd
index 2635ebe9881..16bf4447979 100644
--- a/vespa-documentgen-plugin/etc/complex/book.sd
+++ b/vespa-documentgen-plugin/etc/complex/book.sd
@@ -30,7 +30,7 @@ search book {
         attribute: prefetch
       }
 
-      field mynestedwsfloat type weightedset<weightedset<float>> {}
+      field mynestedwsfloat type weightedset<float> {}
 
       field myarrayint type array<int> {
         indexing: attribute
diff --git a/vespa-documentgen-plugin/etc/complex/common.sd b/vespa-documentgen-plugin/etc/complex/common.sd
index 0764421ac8d..e0505eba05b 100644
--- a/vespa-documentgen-plugin/etc/complex/common.sd
+++ b/vespa-documentgen-plugin/etc/complex/common.sd
@@ -17,19 +17,13 @@ search common {
          indexing: summary
       }
       field weight type float {
-         indexing {
-            input weight * 10 | attribute | summary;
-         }
+        indexing: attribute | summary
       }
       field w1 type float {
-         indexing {
-            input weight * 6 + input w1 | summary;
-         }
+        indexing: summary
       }
       field w2 type float {
-         indexing {
-            input w2 + input weight | summary;
-         }
+        indexing: summary
       }
       field did type string {
          indexing: attribute|index|summary
diff --git a/vespa-documentgen-plugin/etc/complex/common2.sd b/vespa-documentgen-plugin/etc/complex/common2.sd
new file mode 100644
index 00000000000..e32d3ed6751
--- /dev/null
+++ b/vespa-documentgen-plugin/etc/complex/common2.sd
@@ -0,0 +1,9 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search common2 {
+    document {
+        field com2 type string {
+
+        }
+    }
+}
+
diff --git a/vespa-documentgen-plugin/etc/complex/music2.sd b/vespa-documentgen-plugin/etc/complex/music2.sd
index 5657580e622..2e2d96ecdec 100644
--- a/vespa-documentgen-plugin/etc/complex/music2.sd
+++ b/vespa-documentgen-plugin/etc/complex/music2.sd
@@ -56,7 +56,7 @@ search music2 {
    }
 
    field didinteger type array<int> {
-      indexing: input did | split " " | attribute
+      indexing: input did | split " " | for_each { to_int } | attribute
    }
 
    rank-profile default {
diff --git a/vespa-documentgen-plugin/etc/complex/music3.sd b/vespa-documentgen-plugin/etc/complex/music3.sd
new file mode 100644
index 00000000000..65f37029d04
--- /dev/null
+++ b/vespa-documentgen-plugin/etc/complex/music3.sd
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search music3 {
+    document music3 inherits music2, common2 {
+        field mu3 type string {
+
+        }
+    }
+}
diff --git a/vespa-documentgen-plugin/etc/localapp/common.sd b/vespa-documentgen-plugin/etc/localapp/common.sd
index ada7ce7436a..724897b4e7f 100644
--- a/vespa-documentgen-plugin/etc/localapp/common.sd
+++ b/vespa-documentgen-plugin/etc/localapp/common.sd
@@ -17,19 +17,13 @@ search common {
          indexing: summary
       }
       field weight type float {
-         indexing {
-            input weight * 10 | attribute | summary;
-         }
+         indexing: attribute|summary
       }
       field w1 type float {
-         indexing {
-            input weight * 6 + input w1 | summary;
-         }
+         indexing: summary
       }
       field w2 type float {
-         indexing {
-            input w2 + input weight | summary;
-         }
+         indexing: summary
       }
       field did type string {
          indexing: attribute|index|summary
diff --git a/vespa-documentgen-plugin/etc/localapp/music.sd b/vespa-documentgen-plugin/etc/localapp/music.sd
index 0cfe5cf923a..e00e046f511 100644
--- a/vespa-documentgen-plugin/etc/localapp/music.sd
+++ b/vespa-documentgen-plugin/etc/localapp/music.sd
@@ -51,7 +51,7 @@ search music {
    }
 
    field didinteger type array<int> {
-      indexing: input did | split " " | attribute
+      indexing: input did | split " " | for_each { to_int } | attribute
    }
 
    rank-profile default {
diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
index eab3983dc69..acefa3fa461 100644
--- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
+++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.document.*;
 import com.yahoo.document.annotation.AnnotationReferenceDataType;
 import com.yahoo.document.annotation.AnnotationType;
@@ -109,7 +110,7 @@ public class DocumentGenMojo extends AbstractMojo {
             public boolean accept(File dir, String name) {
                 return name.endsWith(".sd");
             }});
-        SearchBuilder builder = new UnprocessingSearchBuilder();
+        SearchBuilder builder = new SearchBuilder();
         for (File f : sdFiles) {
             try {
                 long modTime = f.lastModified();
@@ -405,7 +406,9 @@ public class DocumentGenMojo extends AbstractMojo {
      */
     private void exportDocumentClass(NewDocumentType docType, Writer out, String packageName) throws IOException {
         String className = className(docType.getName());
-        String superType = javaSuperType(docType);
+        Pair<String, Boolean> extendInfo = javaSuperType(docType);
+        String superType = extendInfo.getFirst();
+        Boolean multiExtends = extendInfo.getSecond();
         out.write(
                 "package "+packageName+";\n\n" +
                 exportInnerImportsFromSuperTypes(docType, packageName) +
@@ -441,12 +444,14 @@ public class DocumentGenMojo extends AbstractMojo {
         exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType");
         exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType");
 
-        exportExtendedStructTypeGetter(className, docType.getName(), docType.getAllFields(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType");
-        exportCopyConstructor(className, docType.getAllFields(), out, 1, true);
-        exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? docType.getAllFields() : docType.getFields(), out, 1, true);
-        exportDocumentMethods(docType.getAllFields(), out, 1);
-        exportHashCode(docType.getAllFields(), out, 1, "(getDataType() != null ? getDataType().hashCode() : 0) + getId().hashCode()");
-        exportEquals(className, docType.getAllFields(), out, 1);
+        Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields());
+        exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, out, 1, "getDocumentType", "com.yahoo.document.DocumentType");
+        exportCopyConstructor(className, allUniqueFields, out, 1, true);
+
+        exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true);
+        exportDocumentMethods(allUniqueFields, out, 1);
+        exportHashCode(allUniqueFields, out, 1, "(getDataType() != null ? getDataType().hashCode() : 0) + getId().hashCode()");
+        exportEquals(className, allUniqueFields, out, 1);
         Set<DataType> exportedStructs = exportStructTypes(docType.getTypes(), out, 1, null);
         docTypes.put(docType.getName(), packageName+"."+className);
         for (DataType exportedStruct : exportedStructs) {
@@ -455,15 +460,36 @@ public class DocumentGenMojo extends AbstractMojo {
         out.write("}\n");
     }
 
+    private Collection<Field> getAllUniqueFields(Boolean multipleInheritance, Collection<Field> allFields) {
+        if (multipleInheritance) {
+            Map<String, Field> seen = new HashMap<>();
+            List<Field> unique = new ArrayList<>(allFields.size());
+            for (Field f : allFields) {
+                if (seen.containsKey(f.getName())) {
+                    if ( ! f.equals(seen.get(f.getName()))) {
+                        throw new IllegalArgumentException("Field '" + f.getName() + "' has conflicting definitions in multiple inheritance." +
+                                "First defined as '" + seen.get(f.getName()) + "', then as '" + f + "'.");
+                    }
+                } else {
+                    unique.add(f);
+                    seen.put(f.getName(), f);
+                }
+            }
+            return unique;
+        }
+        return allFields;
+    }
+
     /**
      * The Java class the class of the given type should inherit from. If the input type inherits from _one_
      * other type, use that, otherwise Document.
      */
-    private static String javaSuperType(NewDocumentType docType) {
+    private static Pair<String,Boolean> javaSuperType(NewDocumentType docType) {
         String ret = "com.yahoo.document.Document";
         Collection<NewDocumentType> specInheriteds = specificInheriteds(docType);
-        if (!specInheriteds.isEmpty() && singleInheritance(specInheriteds)) ret = className(specInheriteds.iterator().next().getName());
-        return ret;
+        boolean singleExtends = singleInheritance(specInheriteds);
+        if (!specInheriteds.isEmpty() && singleExtends) ret = className(specInheriteds.iterator().next().getName());
+        return new Pair<>(ret, !singleExtends);
     }
 
     private static boolean singleInheritance(Collection<NewDocumentType> specInheriteds) {
diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java
index a9a5893cf96..b21f38c586a 100644
--- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java
+++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java
@@ -33,6 +33,7 @@ public class DocumentGenTest {
         mojo.execute(new File("etc/complex/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document");
         Map<String, Search> searches = mojo.getSearches();
         assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT);
+        assertEquals(searches.get("book").getDocument("book").getField("sw1").getDataType(), DataType.FLOAT);
         assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType);
         assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType);
         assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT);