diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-05-31 21:07:04 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-31 21:07:04 +0200 |
commit | d77d00c6fe573fe07c60b1545f4b5ab99687f658 (patch) | |
tree | 1b95f566455f1df0c27bb71eb6a39fa7921f0f6c /config-model | |
parent | b17c853d5f593585eb50a794a3824951b4032278 (diff) | |
parent | a94c73bc75e89f041056c8fc347fe23d652db505 (diff) |
Merge pull request #6041 from vespa-engine/geirst/derive-map-of-struct-field-into-array-attributes
Support that map of simple struct field is derived into array attribu…
Diffstat (limited to 'config-model')
8 files changed, 181 insertions, 27 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index c7ca1a33ff2..fc75f7a19fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -5,6 +5,7 @@ import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; @@ -44,39 +45,59 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field); - if (field.usesStructOrMap() && - !fieldIsArrayOfSimpleStruct && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { - return; // Ignore struct fields for indexed search (only implemented for streaming search) + if (unsupportedFieldType(field)) { + return; // Ignore majority of struct fields for indexed search (only implemented for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); - } else if (fieldIsArrayOfSimpleStruct) { + } else if (isArrayOfSimpleStruct(field)) { deriveArrayOfSimpleStruct(field); + } else if (isMapOfSimpleStruct(field)) { + deriveMapOfSimpleStruct(field); } else { deriveAttributes(field); } } + private static boolean unsupportedFieldType(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isArrayOfSimpleStruct(field) && + !isMapOfSimpleStruct(field) && + !field.getDataType().equals(PositionDataType.INSTANCE) && + !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + } + private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { DataType fieldType = field.getDataType(); if (fieldType instanceof ArrayDataType) { ArrayDataType arrayType = (ArrayDataType)fieldType; - DataType nestedType = arrayType.getNestedType(); - if (nestedType instanceof StructDataType && - !(nestedType.equals(PositionDataType.INSTANCE))) { - StructDataType structType = (StructDataType)nestedType; - for (Field innerField : structType.getFields()) { - if (!isPrimitiveType(innerField.getDataType())) { - return false; - } + return isSimpleStruct(arrayType.getNestedType()); + } else { + return false; + } + } + + private static boolean isMapOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof MapDataType) { + MapDataType mapType = (MapDataType)fieldType; + return isPrimitiveType(mapType.getKeyType()) && + isSimpleStruct(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isSimpleStruct(DataType type) { + if (type instanceof StructDataType && + !(type.equals(PositionDataType.INSTANCE))) { + StructDataType structType = (StructDataType) type; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; } - return true; - } else { - return false; } + return true; } else { return false; } @@ -138,14 +159,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce private void deriveArrayOfSimpleStruct(ImmutableSDField field) { for (ImmutableSDField structField : field.getStructFields()) { - for (Attribute attribute : structField.getAttributes().values()) { - if (structField.getName().equals(attribute.getName())) { - attributes.put(attribute.getName(), attribute.convertToArray()); - } + deriveAttributesAsArrayType(structField); + } + } + + private void deriveAttributesAsArrayType(ImmutableSDField field) { + for (Attribute attribute : field.getAttributes().values()) { + if (field.getName().equals(attribute.getName())) { + attributes.put(attribute.getName(), attribute.convertToArray()); } } } + private void deriveMapOfSimpleStruct(ImmutableSDField field) { + deriveMapKeyField(field.getStructField("key")); + deriveMapValueField(field.getStructField("value")); + } + + private void deriveMapKeyField(ImmutableSDField keyField) { + deriveAttributesAsArrayType(keyField); + } + + private void deriveMapValueField(ImmutableSDField valueField) { + for (ImmutableSDField structField : valueField.getStructFields()) { + deriveAttributesAsArrayType(structField); + } + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..604fc1f6ea7 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -0,0 +1,60 @@ +attribute[0].name "elem_map.key" +attribute[0].datatype STRING +attribute[0].collectiontype ARRAY +attribute[0].removeifzero false +attribute[0].createifnonexistent false +attribute[0].fastsearch false +attribute[0].huge false +attribute[0].sortascending true +attribute[0].sortfunction UCA +attribute[0].sortstrength PRIMARY +attribute[0].sortlocale "" +attribute[0].enablebitvectors false +attribute[0].enableonlybitvector false +attribute[0].fastaccess false +attribute[0].arity 8 +attribute[0].lowerbound -9223372036854775808 +attribute[0].upperbound 9223372036854775807 +attribute[0].densepostinglistthreshold 0.4 +attribute[0].tensortype "" +attribute[0].imported false +attribute[1].name "elem_map.value.name" +attribute[1].datatype STRING +attribute[1].collectiontype ARRAY +attribute[1].removeifzero false +attribute[1].createifnonexistent false +attribute[1].fastsearch false +attribute[1].huge false +attribute[1].sortascending true +attribute[1].sortfunction UCA +attribute[1].sortstrength PRIMARY +attribute[1].sortlocale "" +attribute[1].enablebitvectors false +attribute[1].enableonlybitvector false +attribute[1].fastaccess false +attribute[1].arity 8 +attribute[1].lowerbound -9223372036854775808 +attribute[1].upperbound 9223372036854775807 +attribute[1].densepostinglistthreshold 0.4 +attribute[1].tensortype "" +attribute[1].imported false +attribute[2].name "elem_map.value.weight" +attribute[2].datatype INT32 +attribute[2].collectiontype ARRAY +attribute[2].removeifzero false +attribute[2].createifnonexistent false +attribute[2].fastsearch false +attribute[2].huge false +attribute[2].sortascending true +attribute[2].sortfunction UCA +attribute[2].sortstrength PRIMARY +attribute[2].sortlocale "" +attribute[2].enablebitvectors false +attribute[2].enableonlybitvector false +attribute[2].fastaccess false +attribute[2].arity 8 +attribute[2].lowerbound -9223372036854775808 +attribute[2].upperbound 9223372036854775807 +attribute[2].densepostinglistthreshold 0.4 +attribute[2].tensortype "" +attribute[2].imported false
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..7af49d95d09 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 653486243 +classes[0].id 653486243 +classes[0].name "default" +classes[0].fields[0].name "elem_map" +classes[0].fields[0].type "jsonstring" +classes[0].fields[1].name "rankfeatures" +classes[0].fields[1].type "featuredata" +classes[0].fields[2].name "summaryfeatures" +classes[0].fields[2].type "featuredata" +classes[0].fields[3].name "documentid" +classes[0].fields[3].type "longstring"
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..42b6e811ee6 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[0].field "rankfeatures" +override[0].command "rankfeatures" +override[0].arguments "" +override[1].field "summaryfeatures" +override[1].command "summaryfeatures" +override[1].arguments ""
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd new file mode 100644 index 00000000000..cb2eac4ed78 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -0,0 +1,20 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_map type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + } + struct-field value.name { + indexing: attribute + } + struct-field value.weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 990ebe7f993..c3cfcae66e6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -79,6 +79,17 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void map_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { assertEquals(name, attr.getName()); assertEquals(type, attr.getType()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java index 9e73edf9b35..72c7aab4a39 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java @@ -18,4 +18,14 @@ public class AttributesTestCase extends AbstractExportingTestCase { assertCorrectDeriving("attributes"); } + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + + @Test + public void testMapOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_of_struct_attribute"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index dc2d3b7cea1..4600f6ae4c6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -139,9 +139,4 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } - @Test - public void testArrayOfStructAttribute() throws IOException, ParseException { - assertCorrectDeriving("array_of_struct_attribute"); - } - } |