Support that map of primitive types is derived into array attributes.

9 files changed, 192 insertions, 15 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 6b4a9564e1e..0f1918f99d6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct;
+import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct;
 
 /**
@@ -45,7 +46,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
     @Override
     protected void derive(ImmutableSDField field, Search search) {
         if (unsupportedFieldType(field)) {
-            return; // Ignore majority of struct fields for indexed search (only implemented for streaming search)
+            return; // Ignore complex struct and map fields for indexed search (only supported for streaming search)
         }
         if (field.isImportedField()) {
             deriveImportedAttributes(field);
@@ -53,6 +54,8 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
             deriveArrayOfSimpleStruct(field);
         } else if (isMapOfSimpleStruct(field)) {
             deriveMapOfSimpleStruct(field);
+        } else if (isMapOfPrimitiveType(field)) {
+            deriveMapOfPrimitiveType(field);
         } else {
             deriveAttributes(field);
         }
@@ -62,6 +65,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         return (field.usesStructOrMap() &&
                 !isArrayOfSimpleStruct(field) &&
                 !isMapOfSimpleStruct(field) &&
+                !isMapOfPrimitiveType(field) &&
                 !field.getDataType().equals(PositionDataType.INSTANCE) &&
                 !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)));
     }
@@ -113,33 +117,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
 
     private void deriveArrayOfSimpleStruct(ImmutableSDField field) {
         for (ImmutableSDField structField : field.getStructFields()) {
-            deriveAttributesAsArrayType(structField);
+            deriveAttributeAsArrayType(structField);
         }
     }
 
-    private void deriveAttributesAsArrayType(ImmutableSDField field) {
-        for (Attribute attribute : field.getAttributes().values()) {
-            if (field.getName().equals(attribute.getName())) {
-                attributes.put(attribute.getName(), attribute.convertToArray());
-            }
+    private void deriveAttributeAsArrayType(ImmutableSDField field) {
+        Attribute attribute = field.getAttributes().get(field.getName());
+        if (attribute != null) {
+            attributes.put(attribute.getName(), attribute.convertToArray());
         }
     }
 
     private void deriveMapOfSimpleStruct(ImmutableSDField field) {
-        deriveMapKeyField(field.getStructField("key"));
+        deriveAttributeAsArrayType(field.getStructField("key"));
         deriveMapValueField(field.getStructField("value"));
     }
 
-    private void deriveMapKeyField(ImmutableSDField keyField) {
-        deriveAttributesAsArrayType(keyField);
-    }
-
     private void deriveMapValueField(ImmutableSDField valueField) {
         for (ImmutableSDField structField : valueField.getStructFields()) {
-            deriveAttributesAsArrayType(structField);
+            deriveAttributeAsArrayType(structField);
         }
     }
 
+    private void deriveMapOfPrimitiveType(ImmutableSDField field) {
+        deriveAttributeAsArrayType(field.getStructField("key"));
+        deriveAttributeAsArrayType(field.getStructField("value"));
+    }
+
     /** Returns a read only attribute iterator */
     public Iterator attributeIterator() {
         return attributes().iterator();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
index a46e94398a1..89d72756512 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
@@ -39,6 +39,17 @@ public class ComplexAttributeFieldUtils {
         }
     }
 
+    public static boolean isMapOfPrimitiveType(ImmutableSDField field) {
+        DataType fieldType = field.getDataType();
+        if (fieldType instanceof MapDataType) {
+            MapDataType mapType = (MapDataType)fieldType;
+            return isPrimitiveType(mapType.getKeyType()) &&
+                    isPrimitiveType(mapType.getValueType());
+        } else {
+            return false;
+        }
+    }
+
     private static boolean isSimpleStruct(DataType type) {
         if (type instanceof StructDataType &&
                 !(type.equals(PositionDataType.INSTANCE))) {
@@ -69,6 +80,9 @@ public class ComplexAttributeFieldUtils {
         } else if (isMapOfSimpleStruct(field)) {
             return hasSingleAttribute(field.getStructField("key")) &&
                     hasOnlyStructFieldAttributes(field.getStructField("value"));
+        } else if (isMapOfPrimitiveType(field)) {
+            return hasSingleAttribute(field.getStructField("key")) &&
+                    hasSingleAttribute(field.getStructField("value"));
         }
         return false;
     }
diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg
new file mode 100644
index 00000000000..cdd929a1818
--- /dev/null
+++ b/config-model/src/test/derived/map_attribute/attributes.cfg
@@ -0,0 +1,60 @@
+attribute[].name "str_map.key"
+attribute[].datatype STRING
+attribute[].collectiontype ARRAY
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].name "str_map.value"
+attribute[].datatype STRING
+attribute[].collectiontype ARRAY
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].name "int_map.key"
+attribute[].datatype INT32
+attribute[].collectiontype ARRAY
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg
new file mode 100644
index 00000000000..24d6cab7697
--- /dev/null
+++ b/config-model/src/test/derived/map_attribute/summary.cfg
@@ -0,0 +1,13 @@
+defaultsummaryid 1376056200
+classes[].id 1376056200
+classes[].name "default"
+classes[].fields[].name "str_map"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "int_map"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
diff --git a/config-model/src/test/derived/map_attribute/summarymap.cfg b/config-model/src/test/derived/map_attribute/summarymap.cfg
new file mode 100644
index 00000000000..8936b26ca64
--- /dev/null
+++ b/config-model/src/test/derived/map_attribute/summarymap.cfg
@@ -0,0 +1,10 @@
+defaultoutputclass -1
+override[].field "str_map"
+override[].command "attributecombiner"
+override[].arguments ""
+override[].field "rankfeatures"
+override[].command "rankfeatures"
+override[].arguments ""
+override[].field "summaryfeatures"
+override[].command "summaryfeatures"
+override[].arguments ""
diff --git a/config-model/src/test/derived/map_attribute/test.sd b/config-model/src/test/derived/map_attribute/test.sd
new file mode 100644
index 00000000000..da31f575396
--- /dev/null
+++ b/config-model/src/test/derived/map_attribute/test.sd
@@ -0,0 +1,19 @@
+search test {
+  document test {
+    field str_map type map<string, string> {
+      indexing: summary 
+      struct-field key {
+        indexing: attribute
+      }
+      struct-field value {
+        indexing: attribute
+      }
+    }  
+    field int_map type map<int, int> {
+      indexing: summary 
+      struct-field key {
+        indexing: attribute
+      }
+    }  
+  }
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index d36a165ceb5..3704c9b9166 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -114,4 +114,15 @@ public class AttributeListTestCase extends SearchDefinitionTestCase {
         assertTrue(!attributes.hasNext());
     }
 
+    @Test
+    public void map_of_primitive_fields_are_derived_into_array_attributes() throws IOException, ParseException {
+        Search search = SearchBuilder.buildFromFile("src/test/derived/map_attribute/test.sd");
+        Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator();
+
+        assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+        assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+        assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next());
+        assertTrue(!attributes.hasNext());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
index 72c7aab4a39..2483d249125 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
@@ -28,4 +28,9 @@ public class AttributesTestCase extends AbstractExportingTestCase {
         assertCorrectDeriving("map_of_struct_attribute");
     }
 
+    @Test
+    public void testMapOfPrimitiveAttribute() throws IOException, ParseException {
+        assertCorrectDeriving("map_attribute");
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
index 047ba73e784..4cdc48b330e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
@@ -8,6 +8,7 @@ import org.junit.Test;
 import static com.yahoo.config.model.test.TestUtil.joinLines;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes;
+import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -61,6 +62,7 @@ public class ComplexAttributeFieldUtilsTestCase {
                         "  struct-field value.weight { indexing: attribute }",
                         "}"));
         assertTrue(isMapOfSimpleStruct(field));
+        assertFalse(isMapOfPrimitiveType(field));
         assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field));
     }
 
@@ -68,22 +70,61 @@ public class ComplexAttributeFieldUtilsTestCase {
     public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException {
         {
             ImmutableSDField field = createField("elem_map",
-                    joinLines("field elem_map type map<string, elem> {",
+                    joinLines("field elem_map type map<int, elem> {",
                             "  indexing: summary",
                             "  struct-field value.name { indexing: attribute }",
                             "  struct-field value.weight { indexing: attribute }",
                             "}"));
             assertTrue(isMapOfSimpleStruct(field));
+            assertFalse(isMapOfPrimitiveType(field));
             assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
         }
         {
             ImmutableSDField field = createField("elem_map",
-                    joinLines("field elem_map type map<string, elem> {",
+                    joinLines("field elem_map type map<int, elem> {",
                             "  indexing: summary",
                             "  struct-field key { indexing: attribute }",
                             "  struct-field value.weight { indexing: attribute }",
                             "}"));
             assertTrue(isMapOfSimpleStruct(field));
+            assertFalse(isMapOfPrimitiveType(field));
+            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+        }
+    }
+
+    @Test
+    public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException {
+         ImmutableSDField field = createField("str_map",
+                 joinLines("field str_map type map<string, string> {",
+                         "  indexing: summary",
+                         "  struct-field key { indexing: attribute }",
+                         "  struct-field value { indexing: attribute }",
+                         "}"));
+         assertTrue(isMapOfPrimitiveType(field));
+         assertFalse(isMapOfSimpleStruct(field));
+         assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field));
+    }
+
+    @Test
+    public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException {
+        {
+            ImmutableSDField field = createField("int_map",
+                    joinLines("field int_map type map<int, int> {",
+                            "  indexing: summary",
+                            "  struct-field key { indexing: attribute }",
+                            "}"));
+            assertTrue(isMapOfPrimitiveType(field));
+            assertFalse(isMapOfSimpleStruct(field));
+            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+        }
+        {
+            ImmutableSDField field = createField("int_map",
+                    joinLines("field int_map type map<int, int> {",
+                            "  indexing: summary",
+                            "  struct-field value { indexing: attribute }",
+                            "}"));
+            assertTrue(isMapOfPrimitiveType(field));
+            assertFalse(isMapOfSimpleStruct(field));
             assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
         }
     }