From 582d8cdde12cf7d7ef1c717dff111b3c7366a44a Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 25 May 2018 14:30:20 +0200 Subject: Support that array of struct field is derived into array attributes. --- .../searchdefinition/derived/AttributeFields.java | 47 +++++++++++++++++++++ .../yahoo/searchdefinition/document/Attribute.java | 7 ++++ .../array_of_struct_attribute/attributes.cfg | 40 ++++++++++++++++++ .../derived/array_of_struct_attribute/summary.cfg | 11 +++++ .../array_of_struct_attribute/summarymap.cfg | 7 ++++ .../test/derived/array_of_struct_attribute/test.sd | 17 ++++++++ .../AttributeSettingsTestCase.java | 48 ++++++++++++++++++++++ .../derived/AttributeListTestCase.java | 16 ++++++++ .../derived/ExportingTestCase.java | 5 +++ 9 files changed, 198 insertions(+) create mode 100644 config-model/src/test/derived/array_of_struct_attribute/attributes.cfg create mode 100644 config-model/src/test/derived/array_of_struct_attribute/summary.cfg create mode 100644 config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg create mode 100644 config-model/src/test/derived/array_of_struct_attribute/test.sd (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 72ba6de7022..44655b5997e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -2,8 +2,11 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; +import com.yahoo.document.Field; import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -41,18 +44,52 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { + boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field); if (field.usesStructOrMap() && + !fieldIsArrayOfSimpleStruct && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { return; // Ignore struct fields for indexed search (only implemented for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); + } else if (fieldIsArrayOfSimpleStruct) { + deriveArrayOfSimpleStruct(field); } else { deriveAttributes(field); } } + private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)fieldType; + DataType nestedType = arrayType.getNestedType(); + if (nestedType instanceof StructDataType) { + StructDataType structType = (StructDataType)nestedType; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; + } + } + return true; + } else { + return false; + } + } else { + return false; + } + } + + private static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + /** Returns an attribute by name, or null if it doesn't exist */ public Attribute getAttribute(String attributeName) { return attributes.get(attributeName); @@ -98,6 +135,16 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } + private void deriveArrayOfSimpleStruct(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + for (Attribute attribute : structField.getAttributes().values()) { + if (structField.getName().equals(attribute.getName())) { + attributes.put(attribute.getName(), attribute.convertToArray()); + } + } + } + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index f932265cb93..81e44850e71 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -147,6 +147,12 @@ public final class Attribute implements Cloneable, Serializable { this.referenceDocumentType = referenceDocumentType; } + public Attribute convertToArray() { + Attribute result = clone(); + result.collectionType = CollectionType.ARRAY; + return result; + } + /** *

Returns whether this attribute should be included in the "attributeprefetch" summary * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. @@ -181,6 +187,7 @@ public final class Attribute implements Cloneable, Serializable { public long upperBound() { return upperBound; } public double densePostingListThreshold() { return densePostingListThreshold; } public Optional tensorType() { return tensorType; } + public Optional referenceDocumentType() { return referenceDocumentType; } public Sorting getSorting() { return sorting; } diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..9e6b5cea55e --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -0,0 +1,40 @@ +attribute[].name "elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..c1679c57d1a --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 252850086 +classes[].id 252850086 +classes[].name "default" +classes[].fields[].name "elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..8956a146b74 --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd new file mode 100644 index 00000000000..5b2d50cbdba --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -0,0 +1,17 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_array type array { + indexing: summary + struct-field name { + indexing: attribute + } + struct-field weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 95d5832b70d..4ee33abfc08 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -1,12 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.tensor.TensorType; import org.junit.Test; import java.io.IOException; +import java.util.Optional; import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -88,4 +92,48 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { assertTrue(attr.isFastAccess()); } + @Test + public void attribute_convert_to_array_copies_internal_state() { + StructDataType refType = new StructDataType("my_struct"); + Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, + Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType)); + single.setRemoveIfZero(true); + single.setCreateIfNonExistent(true); + single.setPrefetch(Boolean.TRUE); + single.setEnableBitVectors(true); + single.setEnableOnlyBitVector(true); + single.setFastSearch(true); + single.setHuge(true); + single.setFastAccess(true); + single.setPosition(true); + single.setArity(5); + single.setLowerBound(7); + single.setUpperBound(11); + single.setDensePostingListThreshold(13.3); + single.getSorting().setAscending(); + single.getAliases().add("foo"); + + Attribute array = single.convertToArray(); + assertEquals("foo", array.getName()); + assertEquals(Attribute.Type.STRING, array.getType()); + assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType()); + assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType()); + assertSame(single.referenceDocumentType(), array.referenceDocumentType()); + assertTrue(array.isRemoveIfZero()); + assertTrue(array.isCreateIfNonExistent()); + assertTrue(array.isPrefetch()); + assertTrue(array.isEnabledBitVectors()); + assertTrue(array.isEnabledOnlyBitVector()); + assertTrue(array.isFastSearch()); + assertTrue(array.isHuge()); + assertTrue(array.isFastAccess()); + assertTrue(array.isPosition()); + assertEquals(5, array.arity()); + assertEquals(7, array.lowerBound()); + assertEquals(11, array.upperBound()); + assertEquals(13.3, array.densePostingListThreshold(), 0.00001); + assertSame(single.getSorting(), array.getSorting()); + assertSame(single.getAliases(), array.getAliases()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 86f30ba3c11..6f48d8a7e86 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -68,4 +68,20 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void array_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Iterator attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { + assertEquals(name, attr.getName()); + assertEquals(type, attr.getType()); + assertEquals(collection, attr.getCollectionType()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 4600f6ae4c6..dc2d3b7cea1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -139,4 +139,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + } -- cgit v1.2.3