diff options
author | Geir Storli <geirst@oath.com> | 2018-06-07 11:18:57 +0000 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-06-07 11:18:57 +0000 |
commit | f923ea8a4b4c3a94a5beabef70f77e9f49bde407 (patch) | |
tree | 84a219408c88217418c5f8efe0ae1cac20b15ba3 /config-model | |
parent | c6c88798f7e02b88dcf57bed11357128d3ddda78 (diff) |
Support that map of primitive types is derived into array attributes.
Diffstat (limited to 'config-model')
9 files changed, 192 insertions, 15 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 6b4a9564e1e..0f1918f99d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; /** @@ -45,7 +46,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce @Override protected void derive(ImmutableSDField field, Search search) { if (unsupportedFieldType(field)) { - return; // Ignore majority of struct fields for indexed search (only implemented for streaming search) + return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); @@ -53,6 +54,8 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce deriveArrayOfSimpleStruct(field); } else if (isMapOfSimpleStruct(field)) { deriveMapOfSimpleStruct(field); + } else if (isMapOfPrimitiveType(field)) { + deriveMapOfPrimitiveType(field); } else { deriveAttributes(field); } @@ -62,6 +65,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return (field.usesStructOrMap() && !isArrayOfSimpleStruct(field) && !isMapOfSimpleStruct(field) && + !isMapOfPrimitiveType(field) && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); } @@ -113,33 +117,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce private void deriveArrayOfSimpleStruct(ImmutableSDField field) { for (ImmutableSDField structField : field.getStructFields()) { - deriveAttributesAsArrayType(structField); + deriveAttributeAsArrayType(structField); } } - private void deriveAttributesAsArrayType(ImmutableSDField field) { - for (Attribute attribute : field.getAttributes().values()) { - if (field.getName().equals(attribute.getName())) { - attributes.put(attribute.getName(), attribute.convertToArray()); - } + private void deriveAttributeAsArrayType(ImmutableSDField field) { + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null) { + attributes.put(attribute.getName(), attribute.convertToArray()); } } private void deriveMapOfSimpleStruct(ImmutableSDField field) { - deriveMapKeyField(field.getStructField("key")); + deriveAttributeAsArrayType(field.getStructField("key")); deriveMapValueField(field.getStructField("value")); } - private void deriveMapKeyField(ImmutableSDField keyField) { - deriveAttributesAsArrayType(keyField); - } - private void deriveMapValueField(ImmutableSDField valueField) { for (ImmutableSDField structField : valueField.getStructFields()) { - deriveAttributesAsArrayType(structField); + deriveAttributeAsArrayType(structField); } } + private void deriveMapOfPrimitiveType(ImmutableSDField field) { + deriveAttributeAsArrayType(field.getStructField("key")); + deriveAttributeAsArrayType(field.getStructField("value")); + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index a46e94398a1..89d72756512 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -39,6 +39,17 @@ public class ComplexAttributeFieldUtils { } } + public static boolean isMapOfPrimitiveType(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof MapDataType) { + MapDataType mapType = (MapDataType)fieldType; + return isPrimitiveType(mapType.getKeyType()) && + isPrimitiveType(mapType.getValueType()); + } else { + return false; + } + } + private static boolean isSimpleStruct(DataType type) { if (type instanceof StructDataType && !(type.equals(PositionDataType.INSTANCE))) { @@ -69,6 +80,9 @@ public class ComplexAttributeFieldUtils { } else if (isMapOfSimpleStruct(field)) { return hasSingleAttribute(field.getStructField("key")) && hasOnlyStructFieldAttributes(field.getStructField("value")); + } else if (isMapOfPrimitiveType(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasSingleAttribute(field.getStructField("value")); } return false; } diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg new file mode 100644 index 00000000000..cdd929a1818 --- /dev/null +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -0,0 +1,60 @@ +attribute[].name "str_map.key" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "str_map.value" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "int_map.key" +attribute[].datatype INT32 +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg new file mode 100644 index 00000000000..24d6cab7697 --- /dev/null +++ b/config-model/src/test/derived/map_attribute/summary.cfg @@ -0,0 +1,13 @@ +defaultsummaryid 1376056200 +classes[].id 1376056200 +classes[].name "default" +classes[].fields[].name "str_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "int_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" diff --git a/config-model/src/test/derived/map_attribute/summarymap.cfg b/config-model/src/test/derived/map_attribute/summarymap.cfg new file mode 100644 index 00000000000..8936b26ca64 --- /dev/null +++ b/config-model/src/test/derived/map_attribute/summarymap.cfg @@ -0,0 +1,10 @@ +defaultoutputclass -1 +override[].field "str_map" +override[].command "attributecombiner" +override[].arguments "" +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" diff --git a/config-model/src/test/derived/map_attribute/test.sd b/config-model/src/test/derived/map_attribute/test.sd new file mode 100644 index 00000000000..da31f575396 --- /dev/null +++ b/config-model/src/test/derived/map_attribute/test.sd @@ -0,0 +1,19 @@ +search test { + document test { + field str_map type map<string, string> { + indexing: summary + struct-field key { + indexing: attribute + } + struct-field value { + indexing: attribute + } + } + field int_map type map<int, int> { + indexing: summary + struct-field key { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index d36a165ceb5..3704c9b9166 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -114,4 +114,15 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void map_of_primitive_fields_are_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java index 72c7aab4a39..2483d249125 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java @@ -28,4 +28,9 @@ public class AttributesTestCase extends AbstractExportingTestCase { assertCorrectDeriving("map_of_struct_attribute"); } + @Test + public void testMapOfPrimitiveAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_attribute"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 047ba73e784..4cdc48b330e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -8,6 +8,7 @@ import org.junit.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -61,6 +62,7 @@ public class ComplexAttributeFieldUtilsTestCase { " struct-field value.weight { indexing: attribute }", "}")); assertTrue(isMapOfSimpleStruct(field)); + assertFalse(isMapOfPrimitiveType(field)); assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); } @@ -68,22 +70,61 @@ public class ComplexAttributeFieldUtilsTestCase { public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { { ImmutableSDField field = createField("elem_map", - joinLines("field elem_map type map<string, elem> {", + joinLines("field elem_map type map<int, elem> {", " indexing: summary", " struct-field value.name { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); assertTrue(isMapOfSimpleStruct(field)); + assertFalse(isMapOfPrimitiveType(field)); assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); } { ImmutableSDField field = createField("elem_map", - joinLines("field elem_map type map<string, elem> {", + joinLines("field elem_map type map<int, elem> {", " indexing: summary", " struct-field key { indexing: attribute }", " struct-field value.weight { indexing: attribute }", "}")); assertTrue(isMapOfSimpleStruct(field)); + assertFalse(isMapOfPrimitiveType(field)); + assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + } + + @Test + public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + ImmutableSDField field = createField("str_map", + joinLines("field str_map type map<string, string> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(isMapOfPrimitiveType(field)); + assertFalse(isMapOfSimpleStruct(field)); + assertTrue(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + + @Test + public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + { + ImmutableSDField field = createField("int_map", + joinLines("field int_map type map<int, int> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + "}")); + assertTrue(isMapOfPrimitiveType(field)); + assertFalse(isMapOfSimpleStruct(field)); + assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); + } + { + ImmutableSDField field = createField("int_map", + joinLines("field int_map type map<int, int> {", + " indexing: summary", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(isMapOfPrimitiveType(field)); + assertFalse(isMapOfSimpleStruct(field)); assertFalse(isComplexFieldWithOnlyStructFieldAttributes(field)); } } |