Merge pull request #5944 from vespa-engine/geirst/array-of-struct-field-is-derived-into-array-attributes

Support that array of struct field is derived into array attributes.

9 files changed, 198 insertions, 0 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 72ba6de7022..44655b5997e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -2,8 +2,11 @@
 package com.yahoo.searchdefinition.derived;
 
 import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
 import com.yahoo.document.PositionDataType;
+import com.yahoo.document.StructDataType;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.ImmutableSDField;
@@ -41,18 +44,52 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
     /** Derives everything from a field */
     @Override
     protected void derive(ImmutableSDField field, Search search) {
+        boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field);
         if (field.usesStructOrMap() &&
+            !fieldIsArrayOfSimpleStruct &&
             !field.getDataType().equals(PositionDataType.INSTANCE) &&
             !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
             return; // Ignore struct fields for indexed search (only implemented for streaming search)
         }
         if (field.isImportedField()) {
             deriveImportedAttributes(field);
+        } else if (fieldIsArrayOfSimpleStruct) {
+            deriveArrayOfSimpleStruct(field);
         } else {
             deriveAttributes(field);
         }
     }
 
+    private static boolean isArrayOfSimpleStruct(ImmutableSDField field) {
+        DataType fieldType = field.getDataType();
+        if (fieldType instanceof ArrayDataType) {
+            ArrayDataType arrayType = (ArrayDataType)fieldType;
+            DataType nestedType = arrayType.getNestedType();
+            if (nestedType instanceof StructDataType) {
+                StructDataType structType = (StructDataType)nestedType;
+                for (Field innerField : structType.getFields()) {
+                    if (!isPrimitiveType(innerField.getDataType())) {
+                        return false;
+                    }
+                }
+                return true;
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    }
+
+    private static boolean isPrimitiveType(DataType dataType) {
+        return dataType.equals(DataType.BYTE) ||
+                dataType.equals(DataType.INT) ||
+                dataType.equals(DataType.LONG) ||
+                dataType.equals(DataType.FLOAT) ||
+                dataType.equals(DataType.DOUBLE) ||
+                dataType.equals(DataType.STRING);
+    }
+
     /** Returns an attribute by name, or null if it doesn't exist */
     public Attribute getAttribute(String attributeName) {
         return attributes.get(attributeName);
@@ -98,6 +135,16 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         }
     }
 
+    private void deriveArrayOfSimpleStruct(ImmutableSDField field) {
+        for (ImmutableSDField structField : field.getStructFields()) {
+            for (Attribute attribute : structField.getAttributes().values()) {
+                if (structField.getName().equals(attribute.getName())) {
+                    attributes.put(attribute.getName(), attribute.convertToArray());
+                }
+            }
+        }
+    }
+
     /** Returns a read only attribute iterator */
     public Iterator attributeIterator() {
         return attributes().iterator();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index f932265cb93..81e44850e71 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -147,6 +147,12 @@ public final class Attribute implements Cloneable, Serializable {
         this.referenceDocumentType = referenceDocumentType;
     }
 
+    public Attribute convertToArray() {
+        Attribute result = clone();
+        result.collectionType = CollectionType.ARRAY;
+        return result;
+    }
+
     /**
      * <p>Returns whether this attribute should be included in the "attributeprefetch" summary
      * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc.
@@ -181,6 +187,7 @@ public final class Attribute implements Cloneable, Serializable {
     public long upperBound() { return upperBound; }
     public double densePostingListThreshold() { return densePostingListThreshold; }
     public Optional<TensorType> tensorType() { return tensorType; }
+    public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; }
 
     public Sorting getSorting() { return sorting; }
 
diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
new file mode 100644
index 00000000000..9e6b5cea55e
--- /dev/null
+++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
@@ -0,0 +1,40 @@
+attribute[].name "elem_array.name"
+attribute[].datatype STRING
+attribute[].collectiontype ARRAY
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].name "elem_array.weight"
+attribute[].datatype INT32
+attribute[].collectiontype ARRAY
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
new file mode 100644
index 00000000000..c1679c57d1a
--- /dev/null
+++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
@@ -0,0 +1,11 @@
+defaultsummaryid 252850086
+classes[].id 252850086
+classes[].name "default"
+classes[].fields[].name "elem_array"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg
new file mode 100644
index 00000000000..8956a146b74
--- /dev/null
+++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg
@@ -0,0 +1,7 @@
+defaultoutputclass -1
+override[].field "rankfeatures"
+override[].command "rankfeatures"
+override[].arguments ""
+override[].field "summaryfeatures"
+override[].command "summaryfeatures"
+override[].arguments ""
diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd
new file mode 100644
index 00000000000..5b2d50cbdba
--- /dev/null
+++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd
@@ -0,0 +1,17 @@
+search test {
+  document test {
+    struct elem {
+      field name type string {}
+      field weight type int {}
+    }
+    field elem_array type array<elem> {
+      indexing: summary
+      struct-field name {
+        indexing: attribute
+      }
+      struct-field weight {
+        indexing: attribute
+      }
+    }
+  }
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index 95d5832b70d..4ee33abfc08 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -1,12 +1,16 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
+import com.yahoo.document.StructDataType;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
 import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.tensor.TensorType;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Optional;
 
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.*;
@@ -88,4 +92,48 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
         assertTrue(attr.isFastAccess());
     }
 
+    @Test
+    public void attribute_convert_to_array_copies_internal_state() {
+        StructDataType refType = new StructDataType("my_struct");
+        Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE,
+                Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType));
+        single.setRemoveIfZero(true);
+        single.setCreateIfNonExistent(true);
+        single.setPrefetch(Boolean.TRUE);
+        single.setEnableBitVectors(true);
+        single.setEnableOnlyBitVector(true);
+        single.setFastSearch(true);
+        single.setHuge(true);
+        single.setFastAccess(true);
+        single.setPosition(true);
+        single.setArity(5);
+        single.setLowerBound(7);
+        single.setUpperBound(11);
+        single.setDensePostingListThreshold(13.3);
+        single.getSorting().setAscending();
+        single.getAliases().add("foo");
+
+        Attribute array = single.convertToArray();
+        assertEquals("foo", array.getName());
+        assertEquals(Attribute.Type.STRING, array.getType());
+        assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType());
+        assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType());
+        assertSame(single.referenceDocumentType(), array.referenceDocumentType());
+        assertTrue(array.isRemoveIfZero());
+        assertTrue(array.isCreateIfNonExistent());
+        assertTrue(array.isPrefetch());
+        assertTrue(array.isEnabledBitVectors());
+        assertTrue(array.isEnabledOnlyBitVector());
+        assertTrue(array.isFastSearch());
+        assertTrue(array.isHuge());
+        assertTrue(array.isFastAccess());
+        assertTrue(array.isPosition());
+        assertEquals(5, array.arity());
+        assertEquals(7, array.lowerBound());
+        assertEquals(11, array.upperBound());
+        assertEquals(13.3, array.densePostingListThreshold(), 0.00001);
+        assertSame(single.getSorting(), array.getSorting());
+        assertSame(single.getAliases(), array.getAliases());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index 86f30ba3c11..6f48d8a7e86 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -68,4 +68,20 @@ public class AttributeListTestCase extends SearchDefinitionTestCase {
         assertTrue(!attributes.hasNext());
     }
 
+    @Test
+    public void array_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException {
+        Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd");
+        Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator();
+
+        assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+        assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next());
+        assertTrue(!attributes.hasNext());
+    }
+
+    private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) {
+        assertEquals(name, attr.getName());
+        assertEquals(type, attr.getType());
+        assertEquals(collection, attr.getCollectionType());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index 4600f6ae4c6..dc2d3b7cea1 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -139,4 +139,9 @@ public class ExportingTestCase extends AbstractExportingTestCase {
         assertCorrectDeriving("tensor");
     }
 
+    @Test
+    public void testArrayOfStructAttribute() throws IOException, ParseException {
+        assertCorrectDeriving("array_of_struct_attribute");
+    }
+
 }