From 27b64a81a6a133976bd17385eb150e34added12d Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 14 Jun 2018 14:14:37 +0200 Subject: Use document type to resolve struct data types when checking whether a complex field supports being represented as struct field attributes. --- .../searchdefinition/derived/AttributeFields.java | 14 +- .../document/ComplexAttributeFieldUtils.java | 59 +++++-- .../processing/ImplicitSummaries.java | 2 +- .../ComplexAttributeFieldUtilsTestCase.java | 191 +++++++++++++++------ 4 files changed, 196 insertions(+), 70 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 6070aeb0ee1..55f3a94bb70 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; +import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; @@ -23,6 +24,7 @@ import java.util.stream.Collectors; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; /** * The set of all attribute fields defined by a search definition @@ -46,14 +48,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - if (unsupportedFieldType(field)) { + if (unsupportedFieldType(field, search.getDocument())) { return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); - } else if (isArrayOfSimpleStruct(field)) { + } else if (isArrayOfSimpleStruct(field, search.getDocument())) { deriveArrayOfSimpleStruct(field); - } else if (isMapOfSimpleStruct(field)) { + } else if (isMapOfSimpleStruct(field, search.getDocument())) { deriveMapOfSimpleStruct(field); } else if (isMapOfPrimitiveType(field)) { deriveMapOfPrimitiveType(field); @@ -62,11 +64,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } - private static boolean unsupportedFieldType(ImmutableSDField field) { + private static boolean unsupportedFieldType(ImmutableSDField field, SDDocumentType docType) { return (field.usesStructOrMap() && - !isArrayOfSimpleStruct(field) && - !isMapOfSimpleStruct(field) && - !isMapOfPrimitiveType(field) && + !isSupportedComplexField(field, docType) && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index a56364a6975..857959d0678 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -7,6 +7,11 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TemporaryStructuredDataType; + +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; /** * Utils used to check whether a complex field supports being represented as struct field attributes. @@ -14,33 +19,48 @@ import com.yahoo.document.StructDataType; * Currently we support: * - array of simple struct * - map of primitive type to simple struct + * - map of primitive type to primitive type * * @author geirst */ public class ComplexAttributeFieldUtils { - public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { - return isArrayOfSimpleStruct(field.getDataType()); + public static boolean isSupportedComplexField(ImmutableSDField field, SDDocumentType docType) { + return (isArrayOfSimpleStruct(field, docType) || + isMapOfSimpleStruct(field, docType) || + isMapOfPrimitiveType(field)); + } + + public static boolean isArrayOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { + return isArrayOfSimpleStruct(field.getDataType(), Optional.of(docType)); } public static boolean isArrayOfSimpleStruct(DataType fieldType) { + return isArrayOfSimpleStruct(fieldType, Optional.empty()); + } + + private static boolean isArrayOfSimpleStruct(DataType fieldType, Optional docType) { if (fieldType instanceof ArrayDataType) { ArrayDataType arrayType = (ArrayDataType)fieldType; - return isSimpleStruct(arrayType.getNestedType()); + return isSimpleStruct(arrayType.getNestedType(), docType); } else { return false; } } - public static boolean isMapOfSimpleStruct(ImmutableSDField field) { - return isMapOfSimpleStruct(field.getDataType()); + public static boolean isMapOfSimpleStruct(ImmutableSDField field, SDDocumentType docType) { + return isMapOfSimpleStruct(field.getDataType(), Optional.of(docType)); } public static boolean isMapOfSimpleStruct(DataType fieldType) { + return isMapOfSimpleStruct(fieldType, Optional.empty()); + } + + private static boolean isMapOfSimpleStruct(DataType fieldType, Optional docType) { if (fieldType instanceof MapDataType) { MapDataType mapType = (MapDataType)fieldType; return isPrimitiveType(mapType.getKeyType()) && - isSimpleStruct(mapType.getValueType()); + isSimpleStruct(mapType.getValueType(), docType); } else { return false; } @@ -60,11 +80,15 @@ public class ComplexAttributeFieldUtils { } } - private static boolean isSimpleStruct(DataType type) { + private static boolean isSimpleStruct(DataType type, Optional docType) { if (type instanceof StructDataType && !(type.equals(PositionDataType.INSTANCE))) { StructDataType structType = (StructDataType) type; - for (Field innerField : structType.getFields()) { + Collection structFields = getStructFields(structType, docType); + if (structFields.isEmpty()) { + return false; + } + for (Field innerField : structFields) { if (!isPrimitiveType(innerField.getDataType())) { return false; } @@ -75,6 +99,19 @@ public class ComplexAttributeFieldUtils { } } + private static Collection getStructFields(StructDataType structType, Optional docType) { + // The struct data type might be unresolved at this point. If so we use the document type to resolve it. + if (docType.isPresent() && (structType instanceof TemporaryStructuredDataType)) { + SDDocumentType realStructType = docType.get().getOwnedType(structType.getName()); + if (structType != null) { + return realStructType.getDocumentType().getFields(); + } + return Collections.emptyList(); + } else { + return structType.getFields(); + } + } + private static boolean isPrimitiveType(DataType dataType) { return dataType.equals(DataType.BYTE) || dataType.equals(DataType.INT) || @@ -84,10 +121,10 @@ public class ComplexAttributeFieldUtils { dataType.equals(DataType.STRING); } - public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { - if (isArrayOfSimpleStruct(field)) { + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field, SDDocumentType docType) { + if (isArrayOfSimpleStruct(field, docType)) { return hasOnlyStructFieldAttributes(field); - } else if (isMapOfSimpleStruct(field)) { + } else if (isMapOfSimpleStruct(field, docType)) { return hasSingleAttribute(field.getStructField("key")) && hasOnlyStructFieldAttributes(field.getStructField("value")); } else if (isMapOfPrimitiveType(field)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..2d2fc10d9c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -83,7 +83,7 @@ public class ImplicitSummaries extends Processor { } } - if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { + if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field, search.getDocument())) { addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 4cdc48b330e..91a89c204c9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -1,3 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; import com.yahoo.searchdefinition.Search; @@ -6,126 +7,214 @@ import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ComplexAttributeFieldUtilsTestCase { - private static ImmutableSDField createField(String fieldName, String sdFieldContent) throws ParseException { - String sdContent = joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type string {}", - " }", - sdFieldContent, - " }", - "}"); - Search search = SearchBuilder.createFromString(sdContent).getSearch(); - return search.getConcreteField(fieldName); + private static class FixtureBase { + private final Search search; + private final ImmutableSDField field; + + public FixtureBase(String fieldName, String sdContent) throws ParseException { + search = SearchBuilder.createFromString(sdContent).getSearch(); + field = search.getConcreteField(fieldName); + } + + public ImmutableSDField field() { + return field; + } + + public SDDocumentType docType() { + return search.getDocument(); + } + + public boolean isSupportedComplexField() { + return ComplexAttributeFieldUtils.isSupportedComplexField(field(), docType()); + } + + public boolean isArrayOfSimpleStruct() { + return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field(), docType()); + } + + public boolean isMapOfSimpleStruct() { + return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field(), docType()); + } + + public boolean isMapOfPrimitiveType() { + return ComplexAttributeFieldUtils.isMapOfPrimitiveType(field()); + } + + public boolean isComplexFieldWithOnlyStructFieldAttributes() { + return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field(), docType()); + } + } + + private static class Fixture extends FixtureBase { + + public Fixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type string {}", + " }", + sdFieldContent, + " }", + "}")); + } + } + + private static class ComplexFixture extends FixtureBase { + + public ComplexFixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type array {}", + " }", + sdFieldContent, + " }", + "}")); + } } @Test public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_array", + Fixture f = new Fixture("elem_array", joinLines("field elem_array type array {", " indexing: summary", " struct-field name { indexing: attribute }", " struct-field weight { indexing: attribute }", "}")); - assertTrue(isArrayOfSimpleStruct(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_array", + Fixture f = new Fixture("elem_array", joinLines("field elem_array type array {", " indexing: summary", " struct-field weight { indexing: attribute }", "}")); - assertTrue(isArrayOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map {", " indexing: summary", " struct-field key { indexing: attribute }", " struct-field value.name { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map {", " indexing: summary", " struct-field value.name { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } { - ImmutableSDField field = createField("elem_map", + Fixture f = new Fixture("elem_map", joinLines("field elem_map type map {", " indexing: summary", " struct-field key { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); - assertTrue(isMapOfSimpleStruct(field)); - assertFalse(isMapOfPrimitiveType(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } } @Test public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - ImmutableSDField field = createField("str_map", - joinLines("field str_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + Fixture f = new Fixture("str_map", + joinLines("field str_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); } @Test public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { { - ImmutableSDField field = createField("int_map", + Fixture f = new Fixture("int_map", joinLines("field int_map type map {", " indexing: summary", " struct-field key { indexing: attribute }", "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } { - ImmutableSDField field = createField("int_map", + Fixture f = new Fixture("int_map", joinLines("field int_map type map {", " indexing: summary", " struct-field value { indexing: attribute }", "}")); - assertTrue(isMapOfPrimitiveType(field)); - assertFalse(isMapOfSimpleStruct(field)); - assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + + @Test + public void unsupported_complex_field_is_tagged_as_such() throws ParseException { + { + ComplexFixture f = new ComplexFixture("elem_array", + joinLines("field elem_array type array {", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + ComplexFixture f = new ComplexFixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); } } + } -- cgit v1.2.3