diff options
8 files changed, 220 insertions, 56 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index fc75f7a19fc..6b4a9564e1e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -2,12 +2,8 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; -import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -23,6 +19,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + /** * The set of all attribute fields defined by a search definition * @@ -67,51 +66,6 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } - private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { - DataType fieldType = field.getDataType(); - if (fieldType instanceof ArrayDataType) { - ArrayDataType arrayType = (ArrayDataType)fieldType; - return isSimpleStruct(arrayType.getNestedType()); - } else { - return false; - } - } - - private static boolean isMapOfSimpleStruct(ImmutableSDField field) { - DataType fieldType = field.getDataType(); - if (fieldType instanceof MapDataType) { - MapDataType mapType = (MapDataType)fieldType; - return isPrimitiveType(mapType.getKeyType()) && - isSimpleStruct(mapType.getValueType()); - } else { - return false; - } - } - - private static boolean isSimpleStruct(DataType type) { - if (type instanceof StructDataType && - !(type.equals(PositionDataType.INSTANCE))) { - StructDataType structType = (StructDataType) type; - for (Field innerField : structType.getFields()) { - if (!isPrimitiveType(innerField.getDataType())) { - return false; - } - } - return true; - } else { - return false; - } - } - - private static boolean isPrimitiveType(DataType dataType) { - return dataType.equals(DataType.BYTE) || - dataType.equals(DataType.INT) || - dataType.equals(DataType.LONG) || - dataType.equals(DataType.FLOAT) || - dataType.equals(DataType.DOUBLE) || - dataType.equals(DataType.STRING); - } - /** Returns an attribute by name, or null if it doesn't exist */ public Attribute getAttribute(String attributeName) { return attributes.get(attributeName); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java new file mode 100644 index 00000000000..a46e94398a1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -0,0 +1,92 @@ +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; + +/** + * Utils used to check whether a complex field supports being represented as struct field attributes. + * + * Currently we support: + * - array of simple struct + * - map of primitive type to simple struct + * + * @author geirst + */ +public class ComplexAttributeFieldUtils { + + public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)fieldType; + return isSimpleStruct(arrayType.getNestedType()); + } else { + return false; + } + } + + public static boolean isMapOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof MapDataType) { + MapDataType mapType = (MapDataType)fieldType; + return isPrimitiveType(mapType.getKeyType()) && + isSimpleStruct(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isSimpleStruct(DataType type) { + if (type instanceof StructDataType && + !(type.equals(PositionDataType.INSTANCE))) { + StructDataType structType = (StructDataType) type; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; + } + } + return true; + } else { + return false; + } + } + + private static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { + if (isArrayOfSimpleStruct(field)) { + return hasOnlyStructFieldAttributes(field); + } else if (isMapOfSimpleStruct(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasOnlyStructFieldAttributes(field.getStructField("value")); + } + return false; + } + + private static boolean hasOnlyStructFieldAttributes(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + if (!hasSingleAttribute(structField)) { + return false; + } + } + return true; + } + + private static boolean hasSingleAttribute(ImmutableSDField field) { + if (field.getAttributes().size() != 1) { + return false; + } + return (field.getAttributes().get(field.getName()) != null); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 9f4a63b8a9a..b51524b7e62 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + /** * Makes implicitly defined summaries into explicit summaries * @@ -81,6 +83,10 @@ public class ImplicitSummaries extends Processor { } } + if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { + addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + // Position attributes if (field.doesSummarying()) { for (Attribute attribute : field.getAttributes().values()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 4ab0539dfb4..082265c7694 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -19,7 +19,8 @@ public enum SummaryTransform { RANKFEATURES("rankfeatures"), SUMMARYFEATURES("summaryfeatures"), TEXTEXTRACTOR("textextractor"), - GEOPOS("geopos"); + GEOPOS("geopos"), + ATTRIBUTECOMBINER("attributecombiner"); private String name; @@ -86,6 +87,7 @@ public enum SummaryTransform { case GEOPOS: case RANKFEATURES: case SUMMARYFEATURES: + case ATTRIBUTECOMBINER: return true; default: diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg index 8956a146b74..bb99d6ced39 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -1,4 +1,7 @@ defaultoutputclass -1 +override[].field "elem_array" +override[].command "attributecombiner" +override[].arguments "" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg index 42b6e811ee6..ccfba719a96 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -1,7 +1,10 @@ defaultoutputclass -1 -override[0].field "rankfeatures" -override[0].command "rankfeatures" -override[0].arguments "" -override[1].field "summaryfeatures" -override[1].command "summaryfeatures" -override[1].arguments ""
\ No newline at end of file +override[].field "elem_map" +override[].command "attributecombiner" +override[].arguments "" +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java new file mode 100644 index 00000000000..047ba73e784 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -0,0 +1,90 @@ +package com.yahoo.searchdefinition.document; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ComplexAttributeFieldUtilsTestCase { + + private static ImmutableSDField createField(String fieldName, String sdFieldContent) throws ParseException { + String sdContent = joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type string {}", + " }", + sdFieldContent, + " }", + "}"); + Search search = SearchBuilder.createFromString(sdContent).getSearch(); + return search.getConcreteField(fieldName); + } + + @Test + public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + ImmutableSDField field = createField("elem_array", + joinLines("field elem_array type array<elem> {", + " indexing: summary", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}")); + assertTrue(isArrayOfSimpleStruct(field)); + assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + + @Test + public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + ImmutableSDField field = createField("elem_array", + joinLines("field elem_array type array<elem> {", + " indexing: summary", + " struct-field weight { indexing: attribute }", + "}")); + assertTrue(isArrayOfSimpleStruct(field)); + assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + + @Test + public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + ImmutableSDField field = createField("elem_map", + joinLines("field elem_map type map<string, elem> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(isMapOfSimpleStruct(field)); + assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + + @Test + public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + { + ImmutableSDField field = createField("elem_map", + joinLines("field elem_map type map<string, elem> {", + " indexing: summary", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(isMapOfSimpleStruct(field)); + assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + { + ImmutableSDField field = createField("elem_map", + joinLines("field elem_map type map<string, elem> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(isMapOfSimpleStruct(field)); + assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java index c7127cf3ede..9212617a870 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; import java.io.IOException; @@ -14,6 +15,7 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -55,4 +57,16 @@ public class ImplicitSummariesTestCase { } } + + @Test + public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, search.getSummaryField("elem_array").getTransform()); + } + + @Test + public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, search.getSummaryField("elem_map").getTransform()); + } } |