Merge pull request #6041 from vespa-engine/geirst/derive-map-of-struct-field-into-array-attributes

Support that map of simple struct field is derived into array attribu…

8 files changed, 181 insertions, 27 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index c7ca1a33ff2..fc75f7a19fc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -5,6 +5,7 @@ import com.yahoo.config.subscription.ConfigInstanceUtil;
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
 import com.yahoo.document.PositionDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.searchdefinition.Search;
@@ -44,39 +45,59 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
     /** Derives everything from a field */
     @Override
     protected void derive(ImmutableSDField field, Search search) {
-        boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field);
-        if (field.usesStructOrMap() &&
-            !fieldIsArrayOfSimpleStruct &&
-            !field.getDataType().equals(PositionDataType.INSTANCE) &&
-            !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
-            return; // Ignore struct fields for indexed search (only implemented for streaming search)
+        if (unsupportedFieldType(field)) {
+            return; // Ignore majority of struct fields for indexed search (only implemented for streaming search)
         }
         if (field.isImportedField()) {
             deriveImportedAttributes(field);
-        } else if (fieldIsArrayOfSimpleStruct) {
+        } else if (isArrayOfSimpleStruct(field)) {
             deriveArrayOfSimpleStruct(field);
+        } else if (isMapOfSimpleStruct(field)) {
+            deriveMapOfSimpleStruct(field);
         } else {
             deriveAttributes(field);
         }
     }
 
+    private static boolean unsupportedFieldType(ImmutableSDField field) {
+        return (field.usesStructOrMap() &&
+                !isArrayOfSimpleStruct(field) &&
+                !isMapOfSimpleStruct(field) &&
+                !field.getDataType().equals(PositionDataType.INSTANCE) &&
+                !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)));
+    }
+
     private static boolean isArrayOfSimpleStruct(ImmutableSDField field) {
         DataType fieldType = field.getDataType();
         if (fieldType instanceof ArrayDataType) {
             ArrayDataType arrayType = (ArrayDataType)fieldType;
-            DataType nestedType = arrayType.getNestedType();
-            if (nestedType instanceof StructDataType &&
-                !(nestedType.equals(PositionDataType.INSTANCE))) {
-                StructDataType structType = (StructDataType)nestedType;
-                for (Field innerField : structType.getFields()) {
-                    if (!isPrimitiveType(innerField.getDataType())) {
-                        return false;
-                    }
+            return isSimpleStruct(arrayType.getNestedType());
+        } else {
+            return false;
+        }
+    }
+
+    private static boolean isMapOfSimpleStruct(ImmutableSDField field) {
+        DataType fieldType = field.getDataType();
+        if (fieldType instanceof MapDataType) {
+            MapDataType mapType = (MapDataType)fieldType;
+            return isPrimitiveType(mapType.getKeyType()) &&
+                    isSimpleStruct(mapType.getValueType());
+        } else {
+            return false;
+        }
+    }
+
+    private static boolean isSimpleStruct(DataType type) {
+        if (type instanceof StructDataType &&
+                !(type.equals(PositionDataType.INSTANCE))) {
+            StructDataType structType = (StructDataType) type;
+            for (Field innerField : structType.getFields()) {
+                if (!isPrimitiveType(innerField.getDataType())) {
+                    return false;
                 }
-                return true;
-            } else {
-                return false;
             }
+            return true;
         } else {
             return false;
         }
@@ -138,14 +159,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
 
     private void deriveArrayOfSimpleStruct(ImmutableSDField field) {
         for (ImmutableSDField structField : field.getStructFields()) {
-            for (Attribute attribute : structField.getAttributes().values()) {
-                if (structField.getName().equals(attribute.getName())) {
-                    attributes.put(attribute.getName(), attribute.convertToArray());
-                }
+            deriveAttributesAsArrayType(structField);
+        }
+    }
+
+    private void deriveAttributesAsArrayType(ImmutableSDField field) {
+        for (Attribute attribute : field.getAttributes().values()) {
+            if (field.getName().equals(attribute.getName())) {
+                attributes.put(attribute.getName(), attribute.convertToArray());
             }
         }
     }
 
+    private void deriveMapOfSimpleStruct(ImmutableSDField field) {
+        deriveMapKeyField(field.getStructField("key"));
+        deriveMapValueField(field.getStructField("value"));
+    }
+
+    private void deriveMapKeyField(ImmutableSDField keyField) {
+        deriveAttributesAsArrayType(keyField);
+    }
+
+    private void deriveMapValueField(ImmutableSDField valueField) {
+        for (ImmutableSDField structField : valueField.getStructFields()) {
+            deriveAttributesAsArrayType(structField);
+        }
+    }
+
     /** Returns a read only attribute iterator */
     public Iterator attributeIterator() {
         return attributes().iterator();
diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
new file mode 100644
index 00000000000..604fc1f6ea7
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
@@ -0,0 +1,60 @@
+attribute[0].name "elem_map.key"
+attribute[0].datatype STRING
+attribute[0].collectiontype ARRAY
+attribute[0].removeifzero false
+attribute[0].createifnonexistent false
+attribute[0].fastsearch false
+attribute[0].huge false
+attribute[0].sortascending true
+attribute[0].sortfunction UCA
+attribute[0].sortstrength PRIMARY
+attribute[0].sortlocale ""
+attribute[0].enablebitvectors false
+attribute[0].enableonlybitvector false
+attribute[0].fastaccess false
+attribute[0].arity 8
+attribute[0].lowerbound -9223372036854775808
+attribute[0].upperbound 9223372036854775807
+attribute[0].densepostinglistthreshold 0.4
+attribute[0].tensortype ""
+attribute[0].imported false
+attribute[1].name "elem_map.value.name"
+attribute[1].datatype STRING
+attribute[1].collectiontype ARRAY
+attribute[1].removeifzero false
+attribute[1].createifnonexistent false
+attribute[1].fastsearch false
+attribute[1].huge false
+attribute[1].sortascending true
+attribute[1].sortfunction UCA
+attribute[1].sortstrength PRIMARY
+attribute[1].sortlocale ""
+attribute[1].enablebitvectors false
+attribute[1].enableonlybitvector false
+attribute[1].fastaccess false
+attribute[1].arity 8
+attribute[1].lowerbound -9223372036854775808
+attribute[1].upperbound 9223372036854775807
+attribute[1].densepostinglistthreshold 0.4
+attribute[1].tensortype ""
+attribute[1].imported false
+attribute[2].name "elem_map.value.weight"
+attribute[2].datatype INT32
+attribute[2].collectiontype ARRAY
+attribute[2].removeifzero false
+attribute[2].createifnonexistent false
+attribute[2].fastsearch false
+attribute[2].huge false
+attribute[2].sortascending true
+attribute[2].sortfunction UCA
+attribute[2].sortstrength PRIMARY
+attribute[2].sortlocale ""
+attribute[2].enablebitvectors false
+attribute[2].enableonlybitvector false
+attribute[2].fastaccess false
+attribute[2].arity 8
+attribute[2].lowerbound -9223372036854775808
+attribute[2].upperbound 9223372036854775807
+attribute[2].densepostinglistthreshold 0.4
+attribute[2].tensortype ""
+attribute[2].imported false
\ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
new file mode 100644
index 00000000000..7af49d95d09
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
@@ -0,0 +1,11 @@
+defaultsummaryid 653486243
+classes[0].id 653486243
+classes[0].name "default"
+classes[0].fields[0].name "elem_map"
+classes[0].fields[0].type "jsonstring"
+classes[0].fields[1].name "rankfeatures"
+classes[0].fields[1].type "featuredata"
+classes[0].fields[2].name "summaryfeatures"
+classes[0].fields[2].type "featuredata"
+classes[0].fields[3].name "documentid"
+classes[0].fields[3].type "longstring"
\ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg
new file mode 100644
index 00000000000..42b6e811ee6
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg
@@ -0,0 +1,7 @@
+defaultoutputclass -1
+override[0].field "rankfeatures"
+override[0].command "rankfeatures"
+override[0].arguments ""
+override[1].field "summaryfeatures"
+override[1].command "summaryfeatures"
+override[1].arguments ""
\ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd
new file mode 100644
index 00000000000..cb2eac4ed78
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd
@@ -0,0 +1,20 @@
+search test {
+  document test {
+    struct elem {
+      field name type string {}
+      field weight type int {}
+    }
+    field elem_map type map<string,elem> {
+      indexing: summary
+      struct-field key {
+        indexing: attribute
+      }
+      struct-field value.name {
+        indexing: attribute
+      }
+      struct-field value.weight {
+        indexing: attribute
+      }
+    }
+  }
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index 990ebe7f993..c3cfcae66e6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -79,6 +79,17 @@ public class AttributeListTestCase extends SearchDefinitionTestCase {
         assertTrue(!attributes.hasNext());
     }
 
+    @Test
+    public void map_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException {
+        Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd");
+        Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator();
+
+        assertAttribute("elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+        assertAttribute("elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+        assertAttribute("elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next());
+        assertTrue(!attributes.hasNext());
+    }
+
     private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) {
         assertEquals(name, attr.getName());
         assertEquals(type, attr.getType());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
index 9e73edf9b35..72c7aab4a39 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
@@ -18,4 +18,14 @@ public class AttributesTestCase extends AbstractExportingTestCase {
         assertCorrectDeriving("attributes");
     }
 
+    @Test
+    public void testArrayOfStructAttribute() throws IOException, ParseException {
+        assertCorrectDeriving("array_of_struct_attribute");
+    }
+
+    @Test
+    public void testMapOfStructAttribute() throws IOException, ParseException {
+        assertCorrectDeriving("map_of_struct_attribute");
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index dc2d3b7cea1..4600f6ae4c6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -139,9 +139,4 @@ public class ExportingTestCase extends AbstractExportingTestCase {
         assertCorrectDeriving("tensor");
     }
 
-    @Test
-    public void testArrayOfStructAttribute() throws IOException, ParseException {
-        assertCorrectDeriving("array_of_struct_attribute");
-    }
-
 }