Use document type to resolve struct data types when checking whether a complex field supports being represented as struct field attributes.

4 files changed, 196 insertions, 70 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 6070aeb0ee1..55f3a94bb70 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.searchdefinition.document.Ranking;
+import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.Sorting;
 import com.yahoo.vespa.config.search.AttributesConfig;
 import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
@@ -23,6 +24,7 @@ import java.util.stream.Collectors;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType;
 import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct;
+import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField;
 
 /**
  * The set of all attribute fields defined by a search definition
@@ -46,14 +48,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
     /** Derives everything from a field */
     @Override
     protected void derive(ImmutableSDField field, Search search) {
-        if (unsupportedFieldType(field)) {
+        if (unsupportedFieldType(field, search.getDocument())) {
             return; // Ignore complex struct and map fields for indexed search (only supported for streaming search)
         }
         if (field.isImportedField()) {
             deriveImportedAttributes(field);
-        } else if (isArrayOfSimpleStruct(field)) {
+        } else if (isArrayOfSimpleStruct(field, search.getDocument())) {
             deriveArrayOfSimpleStruct(field);
-        } else if (isMapOfSimpleStruct(field)) {
+        } else if (isMapOfSimpleStruct(field, search.getDocument())) {
             deriveMapOfSimpleStruct(field);
         } else if (isMapOfPrimitiveType(field)) {
             deriveMapOfPrimitiveType(field);
@@ -62,11 +64,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         }
     }
 
-    private static boolean unsupportedFieldType(ImmutableSDField field) {
+    private static boolean unsupportedFieldType(ImmutableSDField field, SDDocumentType docType) {
         return (field.usesStructOrMap() &&
-                !isArrayOfSimpleStruct(field) &&
-                !isMapOfSimpleStruct(field) &&
-                !isMapOfPrimitiveType(field) &&
+                !isSupportedComplexField(field, docType) &&
                 !field.getDataType().equals(PositionDataType.INSTANCE) &&
                 !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)));
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
index a56364a6975..857959d0678 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java
@@ -7,6 +7,11 @@ import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.PositionDataType;
 import com.yahoo.document.StructDataType;
+import com.yahoo.document.TemporaryStructuredDataType;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
 
 /**
  * Utils used to check whether a complex field supports being represented as struct field attributes.
@@ -14,33 +19,48 @@ import com.yahoo.document.StructDataType;
  * Currently we support:
  *   - array of simple struct
  *   - map of primitive type to simple struct
+ *   - map of primitive type to primitive type
  *
  * @author geirst
  */
 public class ComplexAttributeFieldUtils {
 
-    public static boolean isArrayOfSimpleStruct(ImmutableSDField field) {
-        return isArrayOfSimpleStruct(field.getDataType());
+    public static boolean isSupportedComplexField(ImmutableSDField field, SDDocumentType docType) {
+        return (isArrayOfSimpleStruct(field, docType) ||
+                isMapOfSimpleStruct(field, docType) ||
+                isMapOfPrimitiveType(field));
+    }
+
+    public static boolean isArrayOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) {
+        return isArrayOfSimpleStruct(field.getDataType(), Optional.of(docType));
     }
 
     public static boolean isArrayOfSimpleStruct(DataType fieldType) {
+        return isArrayOfSimpleStruct(fieldType, Optional.empty());
+    }
+
+    private static boolean isArrayOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) {
         if (fieldType instanceof ArrayDataType) {
             ArrayDataType arrayType = (ArrayDataType)fieldType;
-            return isSimpleStruct(arrayType.getNestedType());
+            return isSimpleStruct(arrayType.getNestedType(), docType);
         } else {
             return false;
         }
     }
 
-    public static boolean isMapOfSimpleStruct(ImmutableSDField field) {
-        return isMapOfSimpleStruct(field.getDataType());
+    public static boolean isMapOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) {
+        return isMapOfSimpleStruct(field.getDataType(), Optional.of(docType));
     }
 
     public static boolean isMapOfSimpleStruct(DataType fieldType) {
+        return isMapOfSimpleStruct(fieldType, Optional.empty());
+    }
+
+    private static boolean isMapOfSimpleStruct(DataType fieldType, Optional<SDDocumentType> docType) {
         if (fieldType instanceof MapDataType) {
             MapDataType mapType = (MapDataType)fieldType;
             return isPrimitiveType(mapType.getKeyType()) &&
-                    isSimpleStruct(mapType.getValueType());
+                    isSimpleStruct(mapType.getValueType(), docType);
         } else {
             return false;
         }
@@ -60,11 +80,15 @@ public class ComplexAttributeFieldUtils {
         }
     }
 
-    private static boolean isSimpleStruct(DataType type) {
+    private static boolean isSimpleStruct(DataType type, Optional<SDDocumentType> docType) {
         if (type instanceof StructDataType &&
                 !(type.equals(PositionDataType.INSTANCE))) {
             StructDataType structType = (StructDataType) type;
-            for (Field innerField : structType.getFields()) {
+            Collection<Field> structFields = getStructFields(structType, docType);
+            if (structFields.isEmpty()) {
+                return false;
+            }
+            for (Field innerField : structFields) {
                 if (!isPrimitiveType(innerField.getDataType())) {
                     return false;
                 }
@@ -75,6 +99,19 @@ public class ComplexAttributeFieldUtils {
         }
     }
 
+    private static Collection<Field> getStructFields(StructDataType structType, Optional<SDDocumentType> docType) {
+        // The struct data type might be unresolved at this point. If so we use the document type to resolve it.
+        if (docType.isPresent() && (structType instanceof TemporaryStructuredDataType)) {
+            SDDocumentType realStructType = docType.get().getOwnedType(structType.getName());
+            if (structType != null) {
+                return realStructType.getDocumentType().getFields();
+            }
+            return Collections.emptyList();
+        } else {
+            return structType.getFields();
+        }
+    }
+
     private static boolean isPrimitiveType(DataType dataType) {
         return dataType.equals(DataType.BYTE) ||
                 dataType.equals(DataType.INT) ||
@@ -84,10 +121,10 @@ public class ComplexAttributeFieldUtils {
                 dataType.equals(DataType.STRING);
     }
 
-    public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) {
-        if (isArrayOfSimpleStruct(field)) {
+    public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field, SDDocumentType docType) {
+        if (isArrayOfSimpleStruct(field, docType)) {
             return hasOnlyStructFieldAttributes(field);
-        } else if (isMapOfSimpleStruct(field)) {
+        } else if (isMapOfSimpleStruct(field, docType)) {
             return hasSingleAttribute(field.getStructField("key")) &&
                     hasOnlyStructFieldAttributes(field.getStructField("value"));
         } else if (isMapOfPrimitiveType(field)) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
index b51524b7e62..2d2fc10d9c5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
@@ -83,7 +83,7 @@ public class ImplicitSummaries extends Processor {
             }
         }
 
-        if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) {
+        if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field, search.getDocument())) {
             addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER);
         }
 
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
index 4cdc48b330e..91a89c204c9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java
@@ -1,3 +1,4 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.document;
 
 import com.yahoo.searchdefinition.Search;
@@ -6,126 +7,214 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
 import static com.yahoo.config.model.test.TestUtil.joinLines;
-import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct;
-import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes;
-import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType;
-import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class ComplexAttributeFieldUtilsTestCase {
 
-    private static ImmutableSDField createField(String fieldName, String sdFieldContent) throws ParseException {
-        String sdContent = joinLines("search test {",
-                "  document test {",
-                "    struct elem {",
-                "      field name type string {}",
-                "      field weight type string {}",
-                "    }",
-                sdFieldContent,
-                "  }",
-                "}");
-        Search search = SearchBuilder.createFromString(sdContent).getSearch();
-        return search.getConcreteField(fieldName);
+    private static class FixtureBase {
+        private final Search search;
+        private final ImmutableSDField field;
+
+        public FixtureBase(String fieldName, String sdContent) throws ParseException {
+            search = SearchBuilder.createFromString(sdContent).getSearch();
+            field = search.getConcreteField(fieldName);
+        }
+
+        public ImmutableSDField field() {
+            return field;
+        }
+
+        public SDDocumentType docType() {
+            return search.getDocument();
+        }
+
+        public boolean isSupportedComplexField() {
+            return ComplexAttributeFieldUtils.isSupportedComplexField(field(), docType());
+        }
+
+        public boolean isArrayOfSimpleStruct() {
+            return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field(), docType());
+        }
+
+        public boolean isMapOfSimpleStruct() {
+            return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field(), docType());
+        }
+
+        public boolean isMapOfPrimitiveType() {
+            return ComplexAttributeFieldUtils.isMapOfPrimitiveType(field());
+        }
+
+        public boolean isComplexFieldWithOnlyStructFieldAttributes() {
+            return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field(), docType());
+        }
+    }
+
+    private static class Fixture extends FixtureBase {
+
+        public Fixture(String fieldName, String sdFieldContent) throws ParseException {
+            super(fieldName, joinLines("search test {",
+                    "  document test {",
+                    "    struct elem {",
+                    "      field name type string {}",
+                    "      field weight type string {}",
+                    "    }",
+                    sdFieldContent,
+                    "  }",
+                    "}"));
+        }
+    }
+
+    private static class ComplexFixture extends FixtureBase {
+
+        public ComplexFixture(String fieldName, String sdFieldContent) throws ParseException {
+            super(fieldName, joinLines("search test {",
+                    "  document test {",
+                    "    struct elem {",
+                    "      field name type string {}",
+                    "      field weight type array<string> {}",
+                    "    }",
+                    sdFieldContent,
+                    "  }",
+                    "}"));
+        }
     }
 
     @Test
     public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException {
-        ImmutableSDField field = createField("elem_array",
+        Fixture f = new Fixture("elem_array",
                 joinLines("field elem_array type array<elem> {",
                         "  indexing: summary",
                         "  struct-field name { indexing: attribute }",
                         "  struct-field weight { indexing: attribute }",
                         "}"));
-        assertTrue(isArrayOfSimpleStruct(field));
-        assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field));
+        assertTrue(f.isSupportedComplexField());
+        assertTrue(f.isArrayOfSimpleStruct());
+        assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes());
     }
 
     @Test
     public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException {
-        ImmutableSDField field = createField("elem_array",
+        Fixture f = new Fixture("elem_array",
                 joinLines("field elem_array type array<elem> {",
                         "  indexing: summary",
                         "  struct-field weight { indexing: attribute }",
                         "}"));
-        assertTrue(isArrayOfSimpleStruct(field));
-        assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+        assertTrue(f.isSupportedComplexField());
+        assertTrue(f.isArrayOfSimpleStruct());
+        assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
     }
 
     @Test
     public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException {
-        ImmutableSDField field = createField("elem_map",
+        Fixture f = new Fixture("elem_map",
                 joinLines("field elem_map type map<string, elem> {",
                         "  indexing: summary",
                         "  struct-field key { indexing: attribute }",
                         "  struct-field value.name { indexing: attribute }",
                         "  struct-field value.weight { indexing: attribute }",
                         "}"));
-        assertTrue(isMapOfSimpleStruct(field));
-        assertFalse(isMapOfPrimitiveType(field));
-        assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field));
+        assertTrue(f.isSupportedComplexField());
+        assertTrue(f.isMapOfSimpleStruct());
+        assertFalse(f.isMapOfPrimitiveType());
+        assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes());
     }
 
     @Test
     public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException {
         {
-            ImmutableSDField field = createField("elem_map",
+            Fixture f = new Fixture("elem_map",
                     joinLines("field elem_map type map<int, elem> {",
                             "  indexing: summary",
                             "  struct-field value.name { indexing: attribute }",
                             "  struct-field value.weight { indexing: attribute }",
                             "}"));
-            assertTrue(isMapOfSimpleStruct(field));
-            assertFalse(isMapOfPrimitiveType(field));
-            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+            assertTrue(f.isSupportedComplexField());
+            assertTrue(f.isMapOfSimpleStruct());
+            assertFalse(f.isMapOfPrimitiveType());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
         }
         {
-            ImmutableSDField field = createField("elem_map",
+            Fixture f = new Fixture("elem_map",
                     joinLines("field elem_map type map<int, elem> {",
                             "  indexing: summary",
                             "  struct-field key { indexing: attribute }",
                             "  struct-field value.weight { indexing: attribute }",
                             "}"));
-            assertTrue(isMapOfSimpleStruct(field));
-            assertFalse(isMapOfPrimitiveType(field));
-            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+            assertTrue(f.isSupportedComplexField());
+            assertTrue(f.isMapOfSimpleStruct());
+            assertFalse(f.isMapOfPrimitiveType());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
         }
     }
 
     @Test
     public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException {
-         ImmutableSDField field = createField("str_map",
-                 joinLines("field str_map type map<string, string> {",
-                         "  indexing: summary",
-                         "  struct-field key { indexing: attribute }",
-                         "  struct-field value { indexing: attribute }",
-                         "}"));
-         assertTrue(isMapOfPrimitiveType(field));
-         assertFalse(isMapOfSimpleStruct(field));
-         assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field));
+        Fixture f = new Fixture("str_map",
+                joinLines("field str_map type map<string, string> {",
+                        "  indexing: summary",
+                        "  struct-field key { indexing: attribute }",
+                        "  struct-field value { indexing: attribute }",
+                        "}"));
+        assertTrue(f.isSupportedComplexField());
+        assertTrue(f.isMapOfPrimitiveType());
+        assertFalse(f.isMapOfSimpleStruct());
+        assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes());
     }
 
     @Test
     public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException {
         {
-            ImmutableSDField field = createField("int_map",
+            Fixture f = new Fixture("int_map",
                     joinLines("field int_map type map<int, int> {",
                             "  indexing: summary",
                             "  struct-field key { indexing: attribute }",
                             "}"));
-            assertTrue(isMapOfPrimitiveType(field));
-            assertFalse(isMapOfSimpleStruct(field));
-            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+            assertTrue(f.isSupportedComplexField());
+            assertTrue(f.isMapOfPrimitiveType());
+            assertFalse(f.isMapOfSimpleStruct());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
         }
         {
-            ImmutableSDField field = createField("int_map",
+            Fixture f = new Fixture("int_map",
                     joinLines("field int_map type map<int, int> {",
                             "  indexing: summary",
                             "  struct-field value { indexing: attribute }",
                             "}"));
-            assertTrue(isMapOfPrimitiveType(field));
-            assertFalse(isMapOfSimpleStruct(field));
-            assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field));
+            assertTrue(f.isSupportedComplexField());
+            assertTrue(f.isMapOfPrimitiveType());
+            assertFalse(f.isMapOfSimpleStruct());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
+        }
+    }
+
+    @Test
+    public void unsupported_complex_field_is_tagged_as_such() throws ParseException {
+        {
+            ComplexFixture f = new ComplexFixture("elem_array",
+                    joinLines("field elem_array type array<elem> {",
+                            "  struct-field name { indexing: attribute }",
+                            "  struct-field weight { indexing: attribute }",
+                            "}"));
+            assertFalse(f.isSupportedComplexField());
+            assertFalse(f.isArrayOfSimpleStruct());
+            assertFalse(f.isMapOfSimpleStruct());
+            assertFalse(f.isMapOfPrimitiveType());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
+        }
+        {
+            ComplexFixture f = new ComplexFixture("elem_map",
+                    joinLines("field elem_map type map<int, elem> {",
+                            "  indexing: summary",
+                            "  struct-field key { indexing: attribute }",
+                            "  struct-field value.weight { indexing: attribute }",
+                            "}"));
+            assertFalse(f.isSupportedComplexField());
+            assertFalse(f.isArrayOfSimpleStruct());
+            assertFalse(f.isMapOfSimpleStruct());
+            assertFalse(f.isMapOfPrimitiveType());
+            assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes());
         }
     }
+
 }