From b729036aa5fb9bb8db2d24db817661c7aa1ff1d1 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 23 Feb 2024 13:50:21 +0100 Subject: Derive indexes for nested attribute fields in streaming search. --- .../java/com/yahoo/schema/derived/VsmFields.java | 34 ++++++---- .../yahoo/schema/derived/VsmFieldsTestCase.java | 74 ++++++++++++++++++++++ 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index a590a3a74bf..1ba9e9b6797 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -17,6 +17,7 @@ import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.Case; import com.yahoo.schema.document.FieldSet; import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; @@ -52,34 +53,34 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { doctypes.put(document.getName(), docType); } for (Object o : document.fieldSet()) { - derive(docType, (SDField) o); + derive(docType, (SDField) o, false, false); } } - private void derive(StreamingDocumentType document, SDField field) { + private void derive(StreamingDocumentType document, SDField field, boolean isStructField, boolean ignoreAttributAspect) { if (field.usesStructOrMap()) { if (GeoPos.isAnyPos(field)) { - StreamingField streamingField = new StreamingField(field); + StreamingField streamingField = new StreamingField(field, isStructField, true); addField(streamingField.getName(), streamingField); addFieldToIndices(document, field.getName(), streamingField); } for (SDField structField : field.getStructFields()) { - derive(document, structField); // Recursion + derive(document, structField, true, ignoreAttributAspect || GeoPos.isAnyPos(field)); // Recursion } } else { - if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) + if (! (field.doesIndexing() || field.doesSummarying() || isAttributeField(field, isStructField, ignoreAttributAspect)) ) return; - StreamingField streamingField = new StreamingField(field); + StreamingField streamingField = new StreamingField(field, isStructField, ignoreAttributAspect); addField(streamingField.getName(),streamingField); - deriveIndices(document, field, streamingField); + deriveIndices(document, field, streamingField, isStructField, ignoreAttributAspect); } } - private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) { + private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField, boolean isStructField, boolean ignoreAttributAspect) { if (field.doesIndexing()) { addFieldToIndices(document, field.getName(), streamingField); - } else if (field.doesAttributing()) { + } else if (isAttributeField(field, isStructField, ignoreAttributAspect)) { for (String indexName : field.getAttributes().keySet()) { addFieldToIndices(document, indexName, streamingField); } @@ -115,6 +116,17 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } } + private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributAspect) { + if (field.doesAttributing()) { + return true; + } + if (!isStructField || ignoreAttributAspect) { + return false; + } + var attribute = field.getAttributes().get(field.getName()); + return attribute != null; + } + private static class StreamingField { private final String name; @@ -170,8 +182,8 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } - public StreamingField(SDField field) { - this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing(), getDistanceMetric(field)); + public StreamingField(SDField field, boolean isStructField, boolean ignoreAttributAspect) { + this(field.getName(), field.getDataType(), field.getMatching(), isAttributeField(field, isStructField, ignoreAttributAspect), getDistanceMetric(field)); } private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute, Attribute.DistanceMetric distanceMetric) { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java index 61d636d911f..852f567ccfa 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Case; import com.yahoo.schema.document.MatchType; @@ -13,9 +14,14 @@ import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.TemporarySDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import org.junit.jupiter.api.Test; +import java.util.HashSet; +import java.util.Set; + +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -77,4 +83,72 @@ public class VsmFieldsTestCase { testIndexMatching(new Matching(MatchType.WORD).setCase(Case.CASED), VsmfieldsConfig.Fieldspec.Normalize.NONE, "word"); } + + private static Set getIndexes(VsmfieldsConfig config, String field) { + var indexes = new HashSet(); + var doctype = config.documenttype(0); + for (var index : doctype.index()) { + for (var indexField : index.field()) { + if (field.equals(indexField.name())) { + indexes.add(index.name()); + break; + } + } + } + return indexes; + } + + @Test + void deriveIndexFromNestedAttributes() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field map_field type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + " }", + " }", + "}"); + var schema = ApplicationBuilder.createFromString(sd).getSchema(); + var config = vsmfieldsConfig(schema); + assertEquals(Set.of("map_field", "map_field.key"), getIndexes(config, "map_field.key")); + assertEquals(Set.of("map_field", "map_field.value"), getIndexes(config, "map_field.value")); + } + + @Test + void deriveIndexFromIndexStatement() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field map_field type map {", + " indexing: summary | index", + " }", + " }", + "}"); + var schema = ApplicationBuilder.createFromString(sd).getSchema(); + var config = vsmfieldsConfig(schema); + assertEquals(Set.of("map_field", "map_field.key"), getIndexes(config, "map_field.key")); + assertEquals(Set.of("map_field", "map_field.value"), getIndexes(config, "map_field.value")); + } + + @Test + void positionFieldTypeBlocksderivingOfIndexFromNestedAttributes() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field pos type position {", + " indexing: attribute | summary", + " struct-field x { indexing: attribute }", + " struct-field y { indexing: attribute }", + " }", + " }", + "}"); + var schema = ApplicationBuilder.createFromString(sd).getSchema(); + var config = vsmfieldsConfig(schema); + assertEquals(Set.of("pos"), getIndexes(config, "pos")); + assertEquals(Set.of(), getIndexes(config, "pos.x")); + assertEquals(Set.of(), getIndexes(config, "pos.y")); + } + } -- cgit v1.2.3 From 498eda0c984c9953161eafb4d018ef8d221c263c Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 23 Feb 2024 15:42:16 +0100 Subject: Fix typo. --- .../java/com/yahoo/schema/derived/VsmFields.java | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index 1ba9e9b6797..c58a49e152e 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -57,7 +57,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } } - private void derive(StreamingDocumentType document, SDField field, boolean isStructField, boolean ignoreAttributAspect) { + private void derive(StreamingDocumentType document, SDField field, boolean isStructField, boolean ignoreAttributeAspect) { if (field.usesStructOrMap()) { if (GeoPos.isAnyPos(field)) { StreamingField streamingField = new StreamingField(field, isStructField, true); @@ -65,22 +65,22 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { addFieldToIndices(document, field.getName(), streamingField); } for (SDField structField : field.getStructFields()) { - derive(document, structField, true, ignoreAttributAspect || GeoPos.isAnyPos(field)); // Recursion + derive(document, structField, true, ignoreAttributeAspect || GeoPos.isAnyPos(field)); // Recursion } } else { - if (! (field.doesIndexing() || field.doesSummarying() || isAttributeField(field, isStructField, ignoreAttributAspect)) ) + if (! (field.doesIndexing() || field.doesSummarying() || isAttributeField(field, isStructField, ignoreAttributeAspect)) ) return; - StreamingField streamingField = new StreamingField(field, isStructField, ignoreAttributAspect); + StreamingField streamingField = new StreamingField(field, isStructField, ignoreAttributeAspect); addField(streamingField.getName(),streamingField); - deriveIndices(document, field, streamingField, isStructField, ignoreAttributAspect); + deriveIndices(document, field, streamingField, isStructField, ignoreAttributeAspect); } } - private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField, boolean isStructField, boolean ignoreAttributAspect) { + private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField, boolean isStructField, boolean ignoreAttributeAspect) { if (field.doesIndexing()) { addFieldToIndices(document, field.getName(), streamingField); - } else if (isAttributeField(field, isStructField, ignoreAttributAspect)) { + } else if (isAttributeField(field, isStructField, ignoreAttributeAspect)) { for (String indexName : field.getAttributes().keySet()) { addFieldToIndices(document, indexName, streamingField); } @@ -116,11 +116,11 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } } - private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributAspect) { + private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributeAspect) { if (field.doesAttributing()) { return true; } - if (!isStructField || ignoreAttributAspect) { + if (!isStructField || ignoreAttributeAspect) { return false; } var attribute = field.getAttributes().get(field.getName()); @@ -182,8 +182,8 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } - public StreamingField(SDField field, boolean isStructField, boolean ignoreAttributAspect) { - this(field.getName(), field.getDataType(), field.getMatching(), isAttributeField(field, isStructField, ignoreAttributAspect), getDistanceMetric(field)); + public StreamingField(SDField field, boolean isStructField, boolean ignoreAttributeAspect) { + this(field.getName(), field.getDataType(), field.getMatching(), isAttributeField(field, isStructField, ignoreAttributeAspect), getDistanceMetric(field)); } private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute, Attribute.DistanceMetric distanceMetric) { -- cgit v1.2.3 From a10c7d02471f2ccc6357827b75dd62fedc2c7f51 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 23 Feb 2024 15:43:54 +0100 Subject: Use var. --- config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index c58a49e152e..f2ad6a3ba2f 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -60,7 +60,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { private void derive(StreamingDocumentType document, SDField field, boolean isStructField, boolean ignoreAttributeAspect) { if (field.usesStructOrMap()) { if (GeoPos.isAnyPos(field)) { - StreamingField streamingField = new StreamingField(field, isStructField, true); + var streamingField = new StreamingField(field, isStructField, true); addField(streamingField.getName(), streamingField); addFieldToIndices(document, field.getName(), streamingField); } @@ -71,7 +71,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { if (! (field.doesIndexing() || field.doesSummarying() || isAttributeField(field, isStructField, ignoreAttributeAspect)) ) return; - StreamingField streamingField = new StreamingField(field, isStructField, ignoreAttributeAspect); + var streamingField = new StreamingField(field, isStructField, ignoreAttributeAspect); addField(streamingField.getName(),streamingField); deriveIndices(document, field, streamingField, isStructField, ignoreAttributeAspect); } -- cgit v1.2.3