Merge pull request #7823 from vespa-engine/toregge/derive-config-for-imported-pos-field

Derive config for imported position field.

11 files changed, 221 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
index 9ddb18ecc8d..4f71bd90830 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -88,7 +88,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
 
     private void deriveImportedComplexField(ImportedField field) {
         String fieldName = field.fieldName();
-        addIndexCommand(fieldName, CMD_MULTIVALUE);
+        if (isPositionField(field.targetField())) {
+            addIndexCommand(fieldName, CMD_DEFAULT_POSITION);
+            if (isPositionArrayField(field.targetField())) {
+                addIndexCommand(fieldName, CMD_MULTIVALUE);
+            }
+        } else {
+            addIndexCommand(fieldName, CMD_MULTIVALUE);
+        }
         addIndexCommand(fieldName, CMD_INDEX);
     }
 
@@ -103,6 +110,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         return b.toString();
     }
 
+    private static boolean isPositionArrayField(ImmutableSDField field) {
+        return field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE));
+    }
+
+    private static boolean isPositionField(ImmutableSDField field) {
+        return field.getDataType().equals(PositionDataType.INSTANCE) || isPositionArrayField(field);
+    }
+
     @Override
     protected void derive(ImmutableSDField field, Search search) {
         derive(field, search, false);
@@ -127,8 +142,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             String name = e.getValue();
             addIndexAlias(alias, name);
         }
-        boolean isPosition = field.getDataType().equals(PositionDataType.INSTANCE) ||
-                field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE));
+        boolean isPosition = isPositionField(field);
         if (field.usesStructOrMap()) {
             for (ImmutableSDField structField : field.getStructFields()) {
                 derive(structField, search, isPosition); // Recursion
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index bfb2fa9c120..c8ecc31103a 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
 import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.searchdefinition.DocumentReference;
 import com.yahoo.searchdefinition.DocumentReferences;
@@ -50,7 +51,10 @@ public class ImportedFieldsResolver extends Processor {
     private void resolveImportedField(TemporaryImportedField importedField, boolean validate) {
         DocumentReference reference = validateDocumentReference(importedField);
         ImmutableSDField targetField = getTargetField(importedField, reference);
-        if (isArrayOfSimpleStruct(targetField)) {
+        if (targetField.getDataType().equals(PositionDataType.INSTANCE) ||
+                targetField.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
+            resolveImportedPositionField(importedField, reference, targetField, validate);
+        } else if (isArrayOfSimpleStruct(targetField)) {
             resolveImportedArrayOfStructField(importedField, reference, targetField, validate);
         } else if (isMapOfSimpleStruct(targetField)) {
             resolveImportedMapOfStructField(importedField, reference, targetField, validate);
@@ -61,6 +65,15 @@ public class ImportedFieldsResolver extends Processor {
         }
     }
 
+    private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
+                                                   ImmutableSDField targetField, boolean validate) {
+        TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
+                reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
+        ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
+        resolveImportedNormalField(importedZCurveField, reference, targetZCurveField, validate);
+        makeImportedComplexField(importedField, reference, targetField);
+    }
+
     private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference,
                                                    ImmutableSDField targetField, boolean validate) {
         resolveImportedNestedStructField(importedField, reference, targetField, validate);
@@ -74,6 +87,14 @@ public class ImportedFieldsResolver extends Processor {
         makeImportedComplexField(importedField, reference, targetField);
     }
 
+    private void makeImportedNormalField(TemporaryImportedField importedField, String name, DocumentReference reference,
+                                         ImmutableSDField targetField) {
+        if (importedFields.get(name) != null) {
+            fail(importedField, name, targetFieldAsString(targetField.getName(), reference) +": Field already imported");
+        }
+        importedFields.put(name, new ImportedField(name, reference, targetField));
+    }
+
     private void makeImportedComplexField(TemporaryImportedField importedField, DocumentReference reference,
                                                  ImmutableSDField targetField) {
         String name = importedField.fieldName();
@@ -89,7 +110,7 @@ public class ImportedFieldsResolver extends Processor {
         Attribute attribute = targetNestedField.getAttributes().get(targetNestedField.getName());
         String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField);
         if (attribute != null) {
-            importedFields.put(importedNestedFieldName, new ImportedField(importedNestedFieldName, reference, targetNestedField));
+            makeImportedNormalField(importedField, importedNestedFieldName, reference, targetNestedField);
         } else if (requireAttribute) {
             fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) +
                     ": Is not an attribute field. Only attribute fields supported");
@@ -124,7 +145,7 @@ public class ImportedFieldsResolver extends Processor {
         if (validate) {
             validateTargetField(importedField, targetField, reference);
         }
-        importedFields.put(importedField.fieldName(), new ImportedField(importedField.fieldName(), reference, targetField));
+        makeImportedNormalField(importedField, importedField.fieldName(), reference, targetField);
     }
 
     private DocumentReference validateDocumentReference(TemporaryImportedField importedField) {
@@ -170,6 +191,9 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void fail(TemporaryImportedField importedField, String importedNestedFieldName, String msg) {
+        if (importedField.fieldName().equals(importedNestedFieldName)) {
+            fail(importedField, msg);
+        }
         throw new IllegalArgumentException("For search '" + search.getName() + "', import field '" +
                 importedField.fieldName() + "' (nested to '" + importedNestedFieldName + "'): " + msg);
     }
diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg
new file mode 100644
index 00000000000..db2280a7846
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/attributes.cfg
@@ -0,0 +1,42 @@
+attribute[0].name "parent_ref"
+attribute[0].datatype REFERENCE
+attribute[0].collectiontype SINGLE
+attribute[0].removeifzero false
+attribute[0].createifnonexistent false
+attribute[0].fastsearch false
+attribute[0].huge false
+attribute[0].ismutable false
+attribute[0].sortascending true
+attribute[0].sortfunction UCA
+attribute[0].sortstrength PRIMARY
+attribute[0].sortlocale ""
+attribute[0].enablebitvectors false
+attribute[0].enableonlybitvector false
+attribute[0].fastaccess false
+attribute[0].arity 8
+attribute[0].lowerbound -9223372036854775808
+attribute[0].upperbound 9223372036854775807
+attribute[0].densepostinglistthreshold 0.4
+attribute[0].tensortype ""
+attribute[0].imported false
+attribute[1].name "my_pos_zcurve"
+attribute[1].datatype INT64
+attribute[1].collectiontype SINGLE
+attribute[1].removeifzero false
+attribute[1].createifnonexistent false
+attribute[1].fastsearch true
+attribute[1].huge false
+attribute[1].ismutable false
+attribute[1].sortascending true
+attribute[1].sortfunction UCA
+attribute[1].sortstrength PRIMARY
+attribute[1].sortlocale ""
+attribute[1].enablebitvectors false
+attribute[1].enableonlybitvector false
+attribute[1].fastaccess false
+attribute[1].arity 8
+attribute[1].lowerbound -9223372036854775808
+attribute[1].upperbound 9223372036854775807
+attribute[1].densepostinglistthreshold 0.4
+attribute[1].tensortype ""
+attribute[1].imported true
\ No newline at end of file
diff --git a/config-model/src/test/derived/imported_position_field/child.sd b/config-model/src/test/derived/imported_position_field/child.sd
new file mode 100644
index 00000000000..29997b855bd
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/child.sd
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search child {
+  document child {
+    field parent_ref type reference<parent> {
+      indexing: attribute | summary
+    }
+  }
+  import field parent_ref.pos as my_pos {}
+}
diff --git a/config-model/src/test/derived/imported_position_field/imported-fields.cfg b/config-model/src/test/derived/imported_position_field/imported-fields.cfg
new file mode 100644
index 00000000000..51b371827c5
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/imported-fields.cfg
@@ -0,0 +1,5 @@
+attribute[0].name "my_pos_zcurve"
+attribute[0].referencefield "parent_ref"
+attribute[0].targetfield "pos_zcurve"
+attribute[0].datatype NONE
+attribute[0].collectiontype SINGLE
\ No newline at end of file
diff --git a/config-model/src/test/derived/imported_position_field/index-info.cfg b/config-model/src/test/derived/imported_position_field/index-info.cfg
new file mode 100644
index 00000000000..d8fffbdfb2d
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/index-info.cfg
@@ -0,0 +1,27 @@
+indexinfo[0].name "child"
+indexinfo[0].command[0].indexname "sddocname"
+indexinfo[0].command[0].command "index"
+indexinfo[0].command[1].indexname "sddocname"
+indexinfo[0].command[1].command "word"
+indexinfo[0].command[2].indexname "parent_ref"
+indexinfo[0].command[2].command "index"
+indexinfo[0].command[3].indexname "parent_ref"
+indexinfo[0].command[3].command "attribute"
+indexinfo[0].command[4].indexname "parent_ref"
+indexinfo[0].command[4].command "word"
+indexinfo[0].command[5].indexname "rankfeatures"
+indexinfo[0].command[5].command "index"
+indexinfo[0].command[6].indexname "summaryfeatures"
+indexinfo[0].command[6].command "index"
+indexinfo[0].command[7].indexname "my_pos_zcurve"
+indexinfo[0].command[7].command "index"
+indexinfo[0].command[8].indexname "my_pos_zcurve"
+indexinfo[0].command[8].command "attribute"
+indexinfo[0].command[9].indexname "my_pos_zcurve"
+indexinfo[0].command[9].command "fast-search"
+indexinfo[0].command[10].indexname "my_pos_zcurve"
+indexinfo[0].command[10].command "numerical"
+indexinfo[0].command[11].indexname "my_pos"
+indexinfo[0].command[11].command "default-position"
+indexinfo[0].command[12].indexname "my_pos"
+indexinfo[0].command[12].command "index"
\ No newline at end of file
diff --git a/config-model/src/test/derived/imported_position_field/parent.sd b/config-model/src/test/derived/imported_position_field/parent.sd
new file mode 100644
index 00000000000..493e105e5fd
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/parent.sd
@@ -0,0 +1,8 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search parent {
+    document parent {
+        field pos type position {
+            indexing: attribute | summary
+        }
+    }
+}
diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg
new file mode 100644
index 00000000000..8da2598a5fc
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/summary.cfg
@@ -0,0 +1,17 @@
+defaultsummaryid 1570252291
+classes[0].id 1570252291
+classes[0].name "default"
+classes[0].fields[0].name "parent_ref"
+classes[0].fields[0].type "longstring"
+classes[0].fields[1].name "rankfeatures"
+classes[0].fields[1].type "featuredata"
+classes[0].fields[2].name "summaryfeatures"
+classes[0].fields[2].type "featuredata"
+classes[0].fields[3].name "documentid"
+classes[0].fields[3].type "longstring"
+classes[1].id 1274088866
+classes[1].name "attributeprefetch"
+classes[1].fields[0].name "rankfeatures"
+classes[1].fields[0].type "featuredata"
+classes[1].fields[1].name "summaryfeatures"
+classes[1].fields[1].type "featuredata"
\ No newline at end of file
diff --git a/config-model/src/test/derived/imported_position_field/summarymap.cfg b/config-model/src/test/derived/imported_position_field/summarymap.cfg
new file mode 100644
index 00000000000..42b6e811ee6
--- /dev/null
+++ b/config-model/src/test/derived/imported_position_field/summarymap.cfg
@@ -0,0 +1,7 @@
+defaultoutputclass -1
+override[0].field "rankfeatures"
+override[0].command "rankfeatures"
+override[0].arguments ""
+override[1].field "summaryfeatures"
+override[1].command "summaryfeatures"
+override[1].arguments ""
\ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
index f7641f97df6..b4c42c248ef 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
@@ -20,4 +20,9 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase {
     public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException {
         assertCorrectDeriving("imported_struct_fields", "child");
     }
+
+    @Test
+    public void configs_for_imported_position_field_are_derived() throws IOException, ParseException {
+        assertCorrectDeriving("imported_position_field", "child");
+    }
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java
index 536a52dd1b6..48b79dade1f 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java
@@ -111,7 +111,7 @@ public class ImportedFieldsTestCase {
     @Test
     public void check_illegal_struct_import_missing_array_of_struct_attributes() throws ParseException {
         exception.expect(IllegalArgumentException.class);
-        exception.expectMessage("For search 'child', import field 'my_elem_array' (nested to 'my_elem_array'): Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field.");
+        exception.expectMessage("For search 'child', import field 'my_elem_array': Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field.");
         checkStructImport(new ParentSdBuilder().elem_array_name_attr(false).elem_array_weight_attr(false));
     }
 
@@ -235,6 +235,62 @@ public class ImportedFieldsTestCase {
         return builder.getSearch("child");
     }
 
+    private static class ParentPosSdBuilder {
+        public String build() {
+            return joinLines("search parent {",
+                    "  document parent {",
+                    "field pos type position {",
+                    "indexing: attribute | summary",
+                    "    }",
+                    "  }",
+                    "}");
+        }
+    }
+
+    private static class ChildPosSdBuilder {
+        private boolean import_pos_zcurve_before;
+
+        public ChildPosSdBuilder() {
+            import_pos_zcurve_before = false;
+        }
+
+        ChildPosSdBuilder import_pos_zcurve_before(boolean v) { import_pos_zcurve_before = v; return this; }
+
+        public String build() {
+            return joinLines("search child {",
+                    "  document child {",
+                    "    field parent_ref type reference<parent> {",
+                    "      indexing: attribute | summary",
+                    "    }",
+                    "  }",
+                    importPosZCurve(import_pos_zcurve_before),
+                    "  import field parent_ref.pos as my_pos {}",
+                    "}");
+        }
+
+        private static String importPosZCurve(boolean doImport) {
+            return doImport ? "import field parent_ref.pos_zcurve as my_pos_zcurve {}" : "";
+        }
+    }
+
+    private static void checkPosImport(ParentPosSdBuilder parentBuilder, ChildPosSdBuilder childBuilder) throws ParseException {
+        Search search = buildChildSearch(parentBuilder.build(), childBuilder.build());
+        assertEquals(1, search.importedFields().get().fields().size());
+        assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", search);
+    }
+
+    @Test
+    public void check_pos_import() throws ParseException {
+        checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder());
+    }
+
+    @Test
+    public void check_pos_import_after_pos_zcurve_import() throws ParseException {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("For search 'child', import field 'my_pos_zcurve': Field 'pos_zcurve' via reference field 'parent_ref': Field already imported");
+        checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true));
+    }
+
     private static void assertSearchNotContainsImportedField(String fieldName, Search search) {
         ImportedField importedField = search.importedFields().get().fields().get(fieldName);
         assertNull(importedField);