diff options
| author | Jon Bratseth <bratseth@vespa.ai> | 2023-06-12 15:14:11 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@vespa.ai> | 2023-06-12 15:14:11 +0200 |
| commit | 2092f374fc4f3de42da75f4660849af0f00f7a81 (patch) | |
| tree | 1047036d19f30481925ab38b0601a85000121ad3 | |
| parent | 537d80f3aad6351322ded0f3e300722cbcdba5d7 (diff) | |
Move to SchemaInfo
Add the missing constructs to SchemaInfo to be able to use
it in place of IndexFacts for validation, and rewrite QueryValidator
to use it. The new validation (for prefix search on indexes) is disabled
until this is verified, so this should be a no-op.
37 files changed, 1255 insertions, 193 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/Index.java b/config-model/src/main/java/com/yahoo/schema/Index.java index 190081cf80f..aa1d226d12e 100644 --- a/config-model/src/main/java/com/yahoo/schema/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/Index.java @@ -15,7 +15,7 @@ import java.util.Optional; import java.util.Set; /** - * An index definition in a search definition. + * An index definition in a schema. * Two indices are equal if they have the same name and the same settings, except * alias settings (which are excluded). * diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Index.java b/config-model/src/main/java/com/yahoo/schema/derived/Index.java index 3b5e617d3dc..2f5b674abee 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Index.java @@ -16,11 +16,11 @@ public class Index { /** The index type enumeration */ public static class Type { - public static final Type TEXT=new Type("text"); - public static final Type INT64=new Type("long"); - public static final Type BOOLEANTREE=new Type("booleantree"); + public static final Type TEXT = new Type("text"); + public static final Type INT64 = new Type("long"); + public static final Type BOOLEANTREE = new Type("booleantree"); - private String name; + private final String name; private Type(String name) { this.name=name; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index 517a43fe129..291c67cae02 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -1,7 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.derived; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.schema.Index; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; @@ -9,6 +23,7 @@ import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -52,11 +67,79 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public void getConfig(SchemaInfoConfig.Builder builder) { var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); schemaBuilder.name(schema.getName()); + addFieldsConfig(schemaBuilder); + addFieldSetConfig(schemaBuilder); addSummaryConfig(schemaBuilder); addRankProfilesConfig(schemaBuilder); builder.schema(schemaBuilder); } + private void addFieldsConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var field : schema.allFieldsList()) { + addFieldConfig(field, schemaBuilder); + for (var index : field.getIndices().values()) { + if ( ! index.getName().equals(field.getName())) // additional index + addFieldConfig(index, field.getDataType(), schemaBuilder); + } + for (var attribute : field.getAttributes().values()) { + if ( ! attribute.getName().equals(field.getName())) // additional attribute + addFieldConfig(attribute, field.getDataType(), schemaBuilder); + } + } + } + + private void addFieldConfig(ImmutableSDField field, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(field.getName()); + fieldBuilder.type(toTypeSpec(field.getDataType())); + for (var alias : field.getAliasToName().entrySet()) { + if (alias.getValue().equals(field.getName())) + fieldBuilder.alias(alias.getKey()); + } + fieldBuilder.attribute(field.doesAttributing()); + fieldBuilder.index(field.doesIndexing()); + schemaBuilder.field(fieldBuilder); + } + + // TODO: Make fields and indexes 1-1 so that this can be removed + private void addFieldConfig(Index index, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(index.getName()); + fieldBuilder.type(toTypeSpec(type)); + for (Iterator<String> i = index.aliasIterator(); i.hasNext(); ) + fieldBuilder.alias(i.next()); + fieldBuilder.attribute(false); + fieldBuilder.index(true); + schemaBuilder.field(fieldBuilder); + } + + // TODO: Make fields and attributes 1-1 so that this can be removed + private void addFieldConfig(Attribute attribute, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(attribute.getName()); + fieldBuilder.type(toTypeSpec(type)); + for (var alias : attribute.getAliases()) + fieldBuilder.alias(alias); + fieldBuilder.attribute(true); + fieldBuilder.index(false); + schemaBuilder.field(fieldBuilder); + } + + private void addFieldSetConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var fieldSet : schema.fieldSets().builtInFieldSets().values()) + addFieldSetConfig(fieldSet, schemaBuilder); + for (var fieldSet : schema.fieldSets().userFieldSets().values()) + addFieldSetConfig(fieldSet, schemaBuilder); + } + + private void addFieldSetConfig(FieldSet fieldSet, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldSetBuilder = new SchemaInfoConfig.Schema.Fieldset.Builder(); + fieldSetBuilder.name(fieldSet.getName()); + for (String fieldName : fieldSet.getFieldNames()) + fieldSetBuilder.field(fieldName); + schemaBuilder.fieldset(fieldSetBuilder); + } + private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (var summary : summaries.asList()) { var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); @@ -88,6 +171,29 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc } } + /** Returns this type as a spec on the form following "field [name] type " in schemas. */ + private String toTypeSpec(DataType dataType) { + if (dataType instanceof PrimitiveDataType) + return dataType.getName(); + if (dataType instanceof AnnotationReferenceDataType annotationType) + return "annotationreference<" + annotationType.getAnnotationType().getName() + ">"; + if (dataType instanceof ArrayDataType arrayType) + return "array<" + toTypeSpec(arrayType.getNestedType()) + ">"; + if (dataType instanceof MapDataType mapType) + return "map<" + toTypeSpec(mapType.getKeyType()) + "," + toTypeSpec(mapType.getValueType()) + ">"; + if (dataType instanceof ReferenceDataType referenceType) + return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">"; + if (dataType instanceof NewDocumentReferenceDataType referenceType) + return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">"; + if (dataType instanceof StructuredDataType structType) + return structType.getName(); + if (dataType instanceof TensorDataType tensorType) + return tensorType.getTensorType().toString(); + if (dataType instanceof WeightedSetDataType weightedSetDataType) + return "weightedset<" + toTypeSpec(weightedSetDataType.getNestedType()) + ">"; + throw new IllegalArgumentException("Unknown data type " + dataType + " class " + dataType.getClass()); + } + /** A store of a *small* (in memory) amount of rank profile info. */ public static final class RankProfileInfo { diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index 919a6023151..5ffe4ec533c 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -216,7 +216,7 @@ public class SDDocumentType implements Cloneable { return f; } - public void addField(Field field) { + public Field addField(Field field) { verifyInheritance(field); for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { if (field.getName().equalsIgnoreCase((i.next()).getName())) { @@ -224,6 +224,7 @@ public class SDDocumentType implements Cloneable { } } docType.addField(field); + return field; } /** Parse-time inheritance check. */ diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 4eaa3e05047..7821c101880 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -556,8 +556,9 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } /** Adds an explicit index defined in this field */ - public void addIndex(Index index) { - indices.put(index.getName(),index); + public Index addIndex(Index index) { + indices.put(index.getName(), index); + return index; } /** @@ -624,13 +625,14 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { return attributes.get(getName()); } - public void addAttribute(Attribute attribute) { + public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); if (name == null || "".equals(name)) { name = getName(); attribute.setName(name); } attributes.put(attribute.getName(),attribute); + return attribute; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 080a2ca43dc..960850def1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -43,7 +43,7 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private IndexingDocproc indexingDocproc; + private final IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; diff --git a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg index e4bd88e6ac9..e04a8e6c1a7 100644 --- a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg @@ -1,4 +1,20 @@ schema[].name "child" +schema[].field[].name "parent_ref" +schema[].field[].type "reference<parent>" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "my_pos_zcurve" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "pos_zcurve" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "my_pos" +schema[].field[].type "position" +schema[].field[].attribute false +schema[].field[].index false schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "parent_ref" schema[].summaryclass[].fields[].type "longstring" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index 82bba81f0d5..8e4c1920e60 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -1,4 +1,72 @@ schema[].name "neuralnet" +schema[].field[].name "uniqueRCount" +schema[].field[].type "double" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "pinned" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "createdAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "updatedAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "uvCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "dvCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "aVoteCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "rCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "uniqueRACount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "rTo" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "markedAsAAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "normalizedTextScore" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "t" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "relevance" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "normalizedCS" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "laAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "hsScore" +schema[].field[].type "double" +schema[].field[].attribute true +schema[].field[].index false schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "rankfeatures" schema[].summaryclass[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg index 86cbfa0562d..8af39cab572 100644 --- a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg +++ b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg @@ -1,4 +1,8 @@ schema[].name "test" +schema[].field[].name "title" +schema[].field[].type "string" +schema[].field[].attribute false +schema[].field[].index true schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "rankfeatures" schema[].summaryclass[].fields[].type "featuredata" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 622fcfdf4fd..3b3fe196cad 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search.test; +import com.yahoo.document.DataType; +import com.yahoo.schema.Index; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.model.VespaModel; @@ -13,10 +17,116 @@ import static org.junit.jupiter.api.Assertions.fail; public class SchemaInfoTestCase { + @Test + void testFieldWithAliasExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + field.getAliasToName().put("alias1", "f1"); + field.getAliasToName().put("alias2", "f1"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].alias[0] "alias1" + schema[0].field[0].alias[1] "alias2" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldsetExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + schema.getDocument().addField(new SDField("f1", DataType.STRING)); + schema.getDocument().addField(new SDField("f2", DataType.STRING)); + schema.fieldSets().addUserFieldSetItem("fs1", "f1"); + schema.fieldSets().addUserFieldSetItem("fs1", "f2"); + schema.fieldSets().addUserFieldSetItem("fs2", "f1"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f2" + schema[0].field[1].type "string" + schema[0].field[1].attribute false + schema[0].field[1].index false + schema[0].fieldset[0].name "fs1" + schema[0].fieldset[0].field[0] "f1" + schema[0].fieldset[0].field[1] "f2" + schema[0].fieldset[1].name "fs2" + schema[0].fieldset[1].field[0] "f1" + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldWithIndexExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var index = field.addIndex(new Index("f1Index")); + index.addAlias("a1"); + index.addAlias("a2"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f1Index" + schema[0].field[1].type "string" + schema[0].field[1].alias[0] "a1" + schema[0].field[1].alias[1] "a2" + schema[0].field[1].attribute false + schema[0].field[1].index true + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldWithAttributeExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var attribute = field.addAttribute(new Attribute("f1Attribute", field.getDataType())); + attribute.getAliases().add("a1"); + attribute.getAliases().add("a2"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f1Attribute" + schema[0].field[1].type "string" + schema[0].field[1].alias[0] "a1" + schema[0].field[1].alias[1] "a2" + schema[0].field[1].attribute true + schema[0].field[1].index false + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + /** Schema-info should contain all schemas, independent of clusters. */ @Test void requireThatSchemaInfoIsAvailable() { - List.of(1.0, 2.0, 3.0).toArray(new Double[3]); String inputs = " rank-profile inputs {" + " inputs {" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java new file mode 100644 index 00000000000..b8bb83540d4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.derived.Summaries; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; + +/** + * @author bratseth + */ +public class SchemaInfoTester { + + public Schema createSchema(String name) { + var schema = new Schema(name, null); + var document = new SDDocumentType(name); + schema.addDocument(document); + schema.addSummary(new DocumentSummary("default", schema)); + return schema; + } + + public String schemaInfoConfig(Schema schema) { + var schemaInfo = new SchemaInfo(schema, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties())); + var schemaInfoConfigBuilder = new SchemaInfoConfig.Builder(); + schemaInfo.getConfig(schemaInfoConfigBuilder); + var schemaInfoConfig = schemaInfoConfigBuilder.build(); + return schemaInfoConfig.toString(); + } + +} diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 84411b31274..e439f7905cc 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1697,7 +1697,7 @@ "methods" : [ "public void <init>()", "public abstract boolean visit(com.yahoo.prelude.query.Item)", - "public abstract void onExit()" + "public void onExit()" ], "fields" : [ ] }, @@ -8086,6 +8086,36 @@ ], "fields" : [ ] }, + "com.yahoo.search.schema.Cluster$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.Cluster$Builder setStreaming(boolean)", + "public com.yahoo.search.schema.Cluster$Builder addSchema(java.lang.String)", + "public com.yahoo.search.schema.Cluster build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Cluster" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.lang.String name()", + "public boolean isStreaming()", + "public java.util.Set schemas()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, "com.yahoo.search.schema.DocumentSummary$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -8169,6 +8199,148 @@ ], "fields" : [ ] }, + "com.yahoo.search.schema.Field$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String)", + "public com.yahoo.search.schema.Field$Builder addAlias(java.lang.String)", + "public com.yahoo.search.schema.Field$Builder setAttribute(boolean)", + "public com.yahoo.search.schema.Field$Builder setIndex(boolean)", + "public com.yahoo.search.schema.Field build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field$TensorFieldType" : { + "superClass" : "com.yahoo.search.schema.Field$Type", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorType tensorType()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field$Type$Kind" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.search.schema.Field$Type$Kind[] values()", + "public static com.yahoo.search.schema.Field$Type$Kind valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.search.schema.Field$Type$Kind ANNOTATIONREFERENCE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind ARRAY", + "public static final enum com.yahoo.search.schema.Field$Type$Kind BOOL", + "public static final enum com.yahoo.search.schema.Field$Type$Kind BYTE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind DOUBLE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind FLOAT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind INT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind LONG", + "public static final enum com.yahoo.search.schema.Field$Type$Kind MAP", + "public static final enum com.yahoo.search.schema.Field$Type$Kind POSITION", + "public static final enum com.yahoo.search.schema.Field$Type$Kind PREDICATE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind RAW", + "public static final enum com.yahoo.search.schema.Field$Type$Kind REFERENCE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind STRING", + "public static final enum com.yahoo.search.schema.Field$Type$Kind STRUCT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind TENSOR", + "public static final enum com.yahoo.search.schema.Field$Type$Kind URL", + "public static final enum com.yahoo.search.schema.Field$Type$Kind WEIGHTEDSET" + ] + }, + "com.yahoo.search.schema.Field$Type" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public com.yahoo.search.schema.Field$Type$Kind kind()", + "public static com.yahoo.search.schema.Field$Type from(java.lang.String)" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.search.schema.FieldInfo" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.search.schema.Field$Builder)", + "public java.lang.String name()", + "public com.yahoo.search.schema.Field$Type type()", + "public java.util.Set aliases()", + "public boolean isAttribute()", + "public boolean isIndex()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.lang.String name()", + "public abstract com.yahoo.search.schema.Field$Type type()", + "public abstract boolean isAttribute()", + "public abstract boolean isIndex()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldSet$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.FieldSet$Builder addField(java.lang.String)", + "public com.yahoo.search.schema.FieldSet build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldSet" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.search.schema.FieldInfo" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.lang.String name()", + "public com.yahoo.search.schema.Field$Type type()", + "public boolean isAttribute()", + "public boolean isIndex()", + "public java.util.Set fieldNames()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, "com.yahoo.search.schema.RankProfile$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -8210,6 +8382,8 @@ ], "methods" : [ "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.Field)", + "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.FieldSet)", "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.RankProfile)", "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.DocumentSummary)", "public com.yahoo.search.schema.Schema build()" @@ -8224,8 +8398,10 @@ ], "methods" : [ "public java.lang.String name()", + "public java.util.Map fields()", "public java.util.Map rankProfiles()", "public java.util.Map documentSummaries()", + "public java.util.Optional fieldInfo(java.lang.String)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -8239,6 +8415,8 @@ "public" ], "methods" : [ + "public boolean isStreaming()", + "public java.util.Optional fieldInfo(java.lang.String)", "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" ], "fields" : [ ] @@ -8250,9 +8428,10 @@ "public" ], "methods" : [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", - "public void <init>(java.util.List, java.util.Map)", + "public void <init>(com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", + "public void <init>(java.util.List, java.util.List)", "public java.util.Map schemas()", + "public java.util.Map clusters()", "public com.yahoo.search.schema.SchemaInfo$Session newSession(com.yahoo.search.Query)", "public static com.yahoo.search.schema.SchemaInfo empty()", "public boolean equals(java.lang.Object)", diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java index 7f37b77919b..e278ad38487 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java @@ -27,8 +27,9 @@ public final class ToolBox { /** * Invoked when all sub-items have been visited, or immediately after * visit() if there are no sub-items or visit() returned false. + * This default implementation does nothing. */ - public abstract void onExit(); + public void onExit() {} } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 0c65f4bf8dc..8a9ea281b0c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -168,7 +168,7 @@ public class SelectParser implements Parser { Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + - " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } try { @@ -186,32 +186,15 @@ public class SelectParser implements Parser { inspector.traverse((ObjectTraverser) (key, value) -> { String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; switch (type) { - case AND: - item[0] = buildAnd(key, value); - break; - case AND_NOT: - item[0] = buildNotAnd(key, value); - break; - case OR: - item[0] = buildOr(key, value); - break; - case EQ: - item[0] = buildEquals(key, value); - break; - case RANGE: - item[0] = buildRange(key, value); - break; - case CONTAINS: - item[0] = buildTermSearch(key, value); - break; - case MATCHES: - item[0] = buildRegExpSearch(key, value); - break; - case CALL: - item[0] = buildFunctionCall(key, value); - break; - default: - throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); + case AND -> item[0] = buildAnd(key, value); + case AND_NOT -> item[0] = buildNotAnd(key, value); + case OR -> item[0] = buildOr(key, value); + case EQ -> item[0] = buildEquals(key, value); + case RANGE -> item[0] = buildRange(key, value); + case CONTAINS -> item[0] = buildTermSearch(key, value); + case MATCHES -> item[0] = buildRegExpSearch(key, value); + case CALL -> item[0] = buildFunctionCall(key, value); + default -> throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); } }); return item[0]; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Cluster.java b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java new file mode 100644 index 00000000000..f5ea4fdffc7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Information about the search aspects of a content cluster. + * + * @author bratseth + */ +@Beta +public class Cluster { + + private final String name; + private final boolean isStreaming; + private final Set<String> schemas; + + private Cluster(Builder builder) { + this.name = builder.name; + this.isStreaming = builder.isStreaming; + this.schemas = Set.copyOf(builder.schemas); + } + + public String name() { return name; } + + /** Returns true if this cluster uses streaming search. */ + public boolean isStreaming() { return isStreaming; } + + /** Returns the names of the subset of all schemas that are present in this cluster. */ + public Set<String> schemas() { return schemas; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Cluster other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( this.isStreaming != other.isStreaming()) return false; + if ( ! this.schemas.equals(other.schemas)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, isStreaming, schemas); + } + + @Override + public String toString() { return "cluster '" + name + "'"; } + + public static class Builder { + + private final String name; + private boolean isStreaming = false; + private final Set<String> schemas = new HashSet<>(); + + public Builder(String name) { + this.name = name; + } + + public Builder setStreaming(boolean isStreaming) { + this.isStreaming = isStreaming; + return this; + } + + public Builder addSchema(String schema) { + schemas.add(schema); + return this; + } + + public Cluster build() { + return new Cluster(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java index 0aec6b0a4f6..a4f208710a0 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java +++ b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; -import java.util.ArrayList; +import com.yahoo.api.annotations.Beta; + import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -15,6 +15,7 @@ import java.util.Objects; * * @author bratseth */ +@Beta public class DocumentSummary { private final String name; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Field.java b/container-search/src/main/java/com/yahoo/search/schema/Field.java new file mode 100644 index 00000000000..ad949d5bad9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/Field.java @@ -0,0 +1,177 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.tensor.TensorType; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * A field in a schema. + * + * @author bratseth + */ +@Beta +public class Field implements FieldInfo { + + private final String name; + private final Type type; + private final boolean isAttribute; + private final boolean isIndex; + private final Set<String> aliases; + + public Field(Builder builder) { + this.name = builder.name; + this.type = builder.type; + this.isAttribute = builder.isAttribute; + this.isIndex = builder.isIndex; + this.aliases = Set.copyOf(builder.aliases); + } + + @Override + public String name() { return name; } + + @Override + public Type type() { return type; } + + public Set<String> aliases() { return aliases; } + + /** Returns whether this field is an attribute, i.e. does indexing: attribute. */ + @Override + public boolean isAttribute() { return isAttribute; } + + /** Returns whether this field is an index, i.e. does indexing: index. */ + @Override + public boolean isIndex() { return isIndex; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Field other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( this.isAttribute != other.isAttribute) return false; + if ( this.isIndex != other.isIndex) return false; + if ( ! this.aliases.equals(other.aliases)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, isAttribute, isIndex, aliases); + } + + @Override + public String toString() { return "field '" + name + "'"; } + + public static class Type { + + private final Kind kind; + + /** The kind of type this is. */ + public enum Kind { + ANNOTATIONREFERENCE, ARRAY, BOOL, BYTE, DOUBLE, FLOAT, INT, LONG, MAP, POSITION, PREDICATE, RAW, REFERENCE, STRING, STRUCT, TENSOR, URL, WEIGHTEDSET; + } + + private Type(Kind kind) { + this.kind = kind; + } + + /** + * Returns the kind of type this is. + * Structured types have additional information in the subclass specific to that kind of type. + */ + public Kind kind() { return kind; } + + /** Creates this from a type string on the syntax following "field [name] type " in a schema definition. */ + public static Type from(String typeString) { + if (typeString.startsWith("annotationreference<")) + return new Type(Kind.ANNOTATIONREFERENCE); // TODO: Model as subclass + if (typeString.startsWith("array<")) + return new Type(Kind.ARRAY); // TODO: Model as subclass + if (typeString.equals("bool")) + return new Type(Kind.BOOL); + if (typeString.equals("byte")) + return new Type(Kind.BYTE); + if (typeString.equals("double")) + return new Type(Kind.DOUBLE); + if (typeString.equals("float")) + return new Type(Kind.FLOAT); + if (typeString.equals("int")) + return new Type(Kind.INT); + if (typeString.equals("long")) + return new Type(Kind.LONG); + if (typeString.startsWith("map<")) + return new Type(Kind.MAP); // TODO: Model as subclass + if (typeString.equals("position")) + return new Type(Kind.POSITION); + if (typeString.equals("predicate")) + return new Type(Kind.PREDICATE); + if (typeString.equals("raw")) + return new Type(Kind.RAW); + if (typeString.startsWith("reference<")) + return new Type(Kind.REFERENCE); // TODO: Model as subclass + if (typeString.equals("string")) + return new Type(Kind.STRING); + if (typeString.startsWith("tensor<") || typeString.startsWith("tensor(")) + return new TensorFieldType(TensorType.fromSpec(typeString)); + if (typeString.equals("url")) + return new Type(Kind.URL); + if (typeString.startsWith("weightedset<")) + return new Type(Kind.WEIGHTEDSET); // TODO: Model as subclass + else + return new Type(Kind.STRUCT); // TODO: Model as a subclass + } + + } + + public static class TensorFieldType extends Type { + + private final TensorType tensorType; + + public TensorFieldType(TensorType tensorType) { + super(Kind.TENSOR); + this.tensorType = tensorType; + } + + public TensorType tensorType() { return tensorType; } + + } + + public static class Builder { + + private final String name; + private final Type type; + private final Set<String> aliases = new HashSet<>(); + private boolean isAttribute; + private boolean isIndex; + + public Builder(String name, String typeString) { + this.name = name; + this.type = Type.from(typeString); + } + + public Builder addAlias(String alias) { + aliases.add(alias); + return this; + } + + public Builder setAttribute(boolean isAttribute) { + this.isAttribute = isAttribute; + return this; + } + + public Builder setIndex(boolean isIndex) { + this.isIndex = isIndex; + return this; + } + + public Field build() { + return new Field(this); + } + + } + + + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java new file mode 100644 index 00000000000..c3f6f22f1b9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.Set; + +/** + * Information about a field or field set. + * + * @author bratseth + */ +@Beta +public interface FieldInfo { + + /** Returns the name of this field or field set. */ + String name(); + + Field.Type type(); + + /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */ + boolean isAttribute(); + + /** Returns whether this field is index(es), i.e. does indexing: index. */ + boolean isIndex(); + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java new file mode 100644 index 00000000000..33c57dd1238 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * A set of fields which can be queried as one. + * + * @author bratseth + */ +@Beta +public class FieldSet implements FieldInfo { + + private final String name; + private final Set<String> fieldNames; + + // Assigned when this is added to a schema + private Schema schema = null; + + private FieldSet(Builder builder) { + this.name = builder.name; + this.fieldNames = Set.copyOf(builder.fieldNames); + } + + @Override + public String name() { return name; } + + @Override + public Field.Type type() { + if (schema == null || fieldNames.isEmpty()) return null; + return randomFieldInThis().type(); + } + + /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */ + @Override + public boolean isAttribute() { + if (schema == null || fieldNames.isEmpty()) return false; + return randomFieldInThis().isAttribute(); + } + + /** Returns whether this field is index(es), i.e. does indexing: index. */ + @Override + public boolean isIndex() { + if (schema == null || fieldNames.isEmpty()) return false; + return randomFieldInThis().isIndex(); + } + + void setSchema(Schema schema) { + if ( this.schema != null) + throw new IllegalStateException("Cannot add field set '" + name + "' to schema '" + schema.name() + + "' as it is already added to schema '" + this.schema.name() + "'"); + this.schema = schema; + } + + /** Use a random field in this to determine its properties. Any inconsistency will have been warned about on deploy. */ + private Field randomFieldInThis() { + return schema.fields().get(fieldNames.iterator().next()); + } + + public Set<String> fieldNames() { return fieldNames; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof FieldSet other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( ! this.fieldNames.equals(other.fieldNames)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, fieldNames); + } + + @Override + public String toString() { return "field set '" + name + "'"; } + + public static class Builder { + + private final String name; + private final Set<String> fieldNames = new HashSet<>(); + + public Builder(String name) { + this.name = name; + } + + public Builder addField(String fieldName) { + fieldNames.add(fieldName); + return this; + } + + public FieldSet build() { + return new FieldSet(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index 85bb3915975..5eba6c220bb 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; +import com.yahoo.api.annotations.Beta; import com.yahoo.tensor.TensorType; import java.util.Collections; @@ -14,6 +15,7 @@ import java.util.Objects; * * @author bratseth */ +@Beta public class RankProfile { private final String name; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java index c20aa1e81bd..20a776dc53e 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/Schema.java +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -8,6 +8,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * Information about a schema which is part of the application running this. @@ -20,25 +21,57 @@ import java.util.Objects; public class Schema { private final String name; + private final Map<String, Field> fields; + private final Map<String, FieldSet> fieldSets; private final Map<String, RankProfile> rankProfiles; private final Map<String, DocumentSummary> documentSummaries; + /** Fields indexed by both name and aliases. */ + private final Map<String, Field> fieldsByAliases; + private Schema(Builder builder) { this.name = builder.name; + this.fields = Collections.unmodifiableMap(builder.fields); + this.fieldSets = Collections.unmodifiableMap(builder.fieldSets); this.rankProfiles = Collections.unmodifiableMap(builder.rankProfiles); this.documentSummaries = Collections.unmodifiableMap(builder.documentSummaries); + + fieldSets.values().forEach(fieldSet -> fieldSet.setSchema(this)); rankProfiles.values().forEach(rankProfile -> rankProfile.setSchema(this)); + + fieldsByAliases = new HashMap<>(); + for (Field field : fields.values()) { + fieldsByAliases.put(field.name(), field); + field.aliases().forEach(alias -> fieldsByAliases.put(alias, field)); + } } public String name() { return name; } + public Map<String, Field> fields() { return fields; } public Map<String, RankProfile> rankProfiles() { return rankProfiles; } public Map<String, DocumentSummary> documentSummaries() { return documentSummaries; } + /** + * Looks up a field or field set by the given name or alias in this schema. + * + * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up + * @return information about the field or field set with the given name, or empty if no item with this name exists + */ + public Optional<FieldInfo> fieldInfo(String fieldName) { + if (fieldName.isEmpty()) + fieldName = "default"; + Field field = fieldsByAliases.get(fieldName); + if (field != null) return Optional.of(field); + return Optional.ofNullable(fieldSets.get(fieldName)); + } + @Override public boolean equals(Object o) { if (o == this) return true; if ( ! (o instanceof Schema other)) return false; if ( ! other.name.equals(this.name)) return false; + if ( ! other.fields.equals(this.fields)) return false; + if ( ! other.fieldSets.equals(this.fieldSets)) return false; if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; if ( ! other.documentSummaries.equals(this.documentSummaries)) return false; return true; @@ -57,6 +90,8 @@ public class Schema { public static class Builder { private final String name; + private final Map<String, Field> fields = new LinkedHashMap<>(); + private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>(); private final Map<String, RankProfile> rankProfiles = new LinkedHashMap<>(); private final Map<String, DocumentSummary> documentSummaries = new LinkedHashMap<>(); @@ -64,6 +99,16 @@ public class Schema { this.name = Objects.requireNonNull(name); } + public Builder add(Field field) { + fields.put(field.name(), field); + return this; + } + + public Builder add(FieldSet fieldSet) { + fieldSets.put(fieldSet.name(), fieldSet); + return this; + } + public Builder add(RankProfile profile) { rankProfiles.put(profile.name(), profile); return this; diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index d29964ea9c5..71bb00b39c5 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -17,6 +17,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -35,37 +36,38 @@ import java.util.stream.Collectors; */ // NOTES: // This should replace IndexFacts, and probably DocumentDatabase. -// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information. -// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation -// given a set of possible schemas: The session mechanism is present here to make that efficient when added -// (resolving schema subsets for every field lookup is too expensive). +// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain complete field information. @Beta public class SchemaInfo { - private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); + private static final SchemaInfo empty = new SchemaInfo(List.of(), List.of()); private final Map<String, Schema> schemas; - /** The schemas contained in each content cluster indexed by cluster name */ - private final Map<String, List<String>> clusters; + private final Map<String, Cluster> clusters; @Inject - public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future - SchemaInfoConfig schemaInfoConfig, + public SchemaInfo(SchemaInfoConfig schemaInfoConfig, QrSearchersConfig qrSearchersConfig) { this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); } - public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) { + public SchemaInfo(List<Schema> schemas, List<Cluster> clusters) { Map<String, Schema> schemaMap = new LinkedHashMap<>(); schemas.forEach(schema -> schemaMap.put(schema.name(), schema)); this.schemas = Collections.unmodifiableMap(schemaMap); - this.clusters = Collections.unmodifiableMap(clusters); + + Map<String, Cluster> clusterMap = new LinkedHashMap<>(); + clusters.forEach(cluster -> clusterMap.put(cluster.name(), cluster)); + this.clusters = Collections.unmodifiableMap(clusterMap); } /** Returns all schemas configured in this application, indexed by schema name. */ public Map<String, Schema> schemas() { return schemas; } + /** Returns information about all clusters available for searching in this applications, indexed by cluyster name. */ + public Map<String, Cluster> clusters() { return clusters; } + public Session newSession(Query query) { return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); } @@ -75,8 +77,7 @@ public class SchemaInfo { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof SchemaInfo)) return false; - SchemaInfo other = (SchemaInfo)o; + if ( ! (o instanceof SchemaInfo other)) return false; if ( ! other.schemas.equals(this.schemas)) return false; if ( ! other.clusters.equals(this.clusters)) return false; return true; @@ -88,15 +89,61 @@ public class SchemaInfo { /** The schema information resolved to be relevant to this session. */ public static class Session { + private final boolean isStreaming; private final Collection<Schema> schemas; private Session(Set<String> sources, Set<String> restrict, - Map<String, List<String>> clusters, + Map<String, Cluster> clusters, Map<String, Schema> candidates) { + this.isStreaming = resolveStreaming(sources, clusters); this.schemas = resolveSchemas(sources, restrict, clusters, candidates.values()); } + /** Returns true if this only searches streaming clusters. */ + public boolean isStreaming() { return isStreaming; } + + /** + * Looks up a field or field set by the given name or alias + * in the schemas resolved for this query. + * + * If there are several fields or field sets by this name or alias across the schemas of this session, + * one is chosen by random. + * + * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up. + * @return the appropriate field or empty if no field or field set has this name or alias + */ + public Optional<FieldInfo> fieldInfo(String fieldName) { + for (var schema : schemas) { + Optional<FieldInfo> field = schema.fieldInfo(fieldName); + if (field.isPresent()) + return field; + } + return Optional.empty(); + } + + private static boolean resolveStreaming(Set<String> sources, Map<String, Cluster> clusters) { + if (sources.isEmpty()) return clusters.values().stream().allMatch(Cluster::isStreaming); + + var matchedClusters = sources.stream().map(source -> clusterOfSource(source, clusters)).filter(Objects::nonNull).toList(); + if (matchedClusters.isEmpty()) return false; + return matchedClusters.stream().allMatch(Cluster::isStreaming); + } + + /** + * A source name is either a cluster or a schema. + * Returns the cluster which either is or contains this name, if any. + */ + private static Cluster clusterOfSource(String source, Map<String, Cluster> clusters) { + var cluster = clusters.get(source); + if (cluster != null) return cluster; + for (var c : clusters.values()) { + if (c.schemas().contains(source)) + return c; + } + return null; + } + /** * Given a search list which is a mixture of schemas and cluster * names, and a restrict list which is a list of schemas, return a @@ -106,7 +153,7 @@ public class SchemaInfo { */ private static Collection<Schema> resolveSchemas(Set<String> sources, Set<String> restrict, - Map<String, List<String>> clusters, + Map<String, Cluster> clusters, Collection<Schema> candidates) { if (sources.isEmpty()) return restrict.isEmpty() ? candidates : keep(restrict, candidates); @@ -114,7 +161,7 @@ public class SchemaInfo { Set<String> schemaNames = new HashSet<>(); for (String source : sources) { if (clusters.containsKey(source)) // source is a cluster - schemaNames.addAll(clusters.get(source)); + schemaNames.addAll(clusters.get(source).schemas()); else // source is a schema schemaNames.add(source); } @@ -126,13 +173,6 @@ public class SchemaInfo { return schemas.stream().filter(schema -> names.contains(schema.name())).toList(); } - private List<RankProfile> profilesNamed(String name) { - return schemas.stream() - .filter(schema -> schema.rankProfiles().containsKey(name)) - .map(schema -> schema.rankProfiles().get(name)) - .toList(); - } - /** * Returns the type of the given rank feature name in the given profile, * if it can be uniquely determined. @@ -165,6 +205,13 @@ public class SchemaInfo { return foundType; } + private List<RankProfile> profilesNamed(String name) { + return schemas.stream() + .filter(schema -> schema.rankProfiles().containsKey(name)) + .map(schema -> schema.rankProfiles().get(name)) + .toList(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 6947a93a833..1b9ba397105 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -6,10 +6,7 @@ import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; /** * Translation between schema info configuration and schema objects. @@ -47,14 +44,15 @@ class SchemaInfoConfigurer { return builder.build(); } - static Map<String, List<String>> toClusters(QrSearchersConfig config) { - Map<String, List<String>> clusters = new HashMap<>(); + static List<Cluster> toClusters(QrSearchersConfig config) { + List<Cluster> clusters = new ArrayList<>(); for (int i = 0; i < config.searchcluster().size(); ++i) { - List<String> schemas = new ArrayList<>(); String clusterName = config.searchcluster(i).name(); - for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j) - schemas.add(config.searchcluster(i).searchdef(j)); - clusters.put(clusterName, schemas); + var clusterInfo = new Cluster.Builder(clusterName); + clusterInfo.setStreaming(config.searchcluster(i).indexingmode() == QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING); + for (var schemaDef : config.searchcluster(i).searchdef()) + clusterInfo.addSchema(schemaDef); + clusters.add(clusterInfo.build()); } return clusters; } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index bfc4219eabc..87880ce2445 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -63,7 +63,7 @@ public class ExecutionFactory extends AbstractComponent { Executor executor) { this(chainsConfig, indexInfo, - new SchemaInfo(indexInfo, schemaInfo, clusters), + new SchemaInfo(schemaInfo, clusters), clusters, searchers, specialTokens, diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java index 341ab342468..0b435c2e32d 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java @@ -3,8 +3,6 @@ package com.yahoo.search.searchers; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.HasIndexItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PrefixItem; @@ -12,6 +10,9 @@ import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.FieldInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; @@ -28,36 +29,36 @@ public class QueryValidator extends Searcher { @Override public Result search(Query query, Execution execution) { - IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); + var session = execution.context().schemaInfo().newSession(query); ToolBox.visit(new TermSearchValidator(session), query.getModel().getQueryTree().getRoot()); - ToolBox.visit(new PrefixSearchValidator(session), query.getModel().getQueryTree().getRoot()); + // ToolBox.visit(new PrefixSearchValidator(session), query.getModel().getQueryTree().getRoot()); TODO: Enable check and QueryValidatorPrefixTest return execution.search(query); } private abstract static class TermValidator extends ToolBox.QueryVisitor { - final IndexFacts.Session session; + final SchemaInfo.Session schema; - public TermValidator(IndexFacts.Session session) { - this.session = session; + public TermValidator(SchemaInfo.Session schema) { + this.schema = schema; } - @Override - public void onExit() { } - } private static class TermSearchValidator extends TermValidator { - public TermSearchValidator(IndexFacts.Session session) { - super(session); + public TermSearchValidator(SchemaInfo.Session schema) { + super(schema); } @Override public boolean visit(Item item) { if (item instanceof HasIndexItem indexItem) { - if (session.getIndex(indexItem.getIndexName()).isTensor()) - throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() + "': It is a tensor field"); + var field = schema.fieldInfo(indexItem.getIndexName()); + if (! field.isPresent()) return true; + if (field.get().type().kind() == Field.Type.Kind.TENSOR) + throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() + + "': It is a tensor field"); } return true; } @@ -66,17 +67,19 @@ public class QueryValidator extends Searcher { private static class PrefixSearchValidator extends TermValidator { - public PrefixSearchValidator(IndexFacts.Session session) { - super(session); + public PrefixSearchValidator(SchemaInfo.Session schema) { + super(schema); } @Override public boolean visit(Item item) { + if (schema.isStreaming()) return true; // prefix is always supported if (item instanceof PrefixItem prefixItem) { - Index index = session.getIndex(prefixItem.getIndexName()); - if ( ! index.isAttribute()) + var field = schema.fieldInfo(prefixItem.getIndexName()); + if (! field.isPresent()) return true; + if ( ! field.get().isAttribute()) throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is not an attribute field: Prefix matching is not supported"); - if (index.isIndex()) // index overrides attribute + if (field.get().isIndex()) // index overrides attribute throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is an index field: Prefix matching is not supported even when it is also an attribute"); } return true; diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 2da6d621973..9018d41d5d9 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -4,7 +4,18 @@ namespace=search.config ## The name of this schema schema[].name string -## The name of the summary class +## Information about a schema field (currently incomplete) +schema[].field[].name string +schema[].field[].type string +schema[].field[].alias[] string +schema[].field[].attribute bool +schema[].field[].index bool + +## Field sets +schema[].fieldset[].name string +schema[].fieldset[].field[] string + +## Information about a summary class schema[].summaryclass[].name string ## The name of a field in the summary class schema[].summaryclass[].fields[].name string diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 06ae9923dae..c164fd3eb1c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -277,7 +277,7 @@ public class ClusterSearcherTestCase { schemaBuilder.add(new RankProfile.Builder("testprofile").build()); schemas.add(schemaBuilder.build()); } - return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, Map.of()))); + return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, List.of()))); } finally { cluster.deconstruct(); } @@ -462,7 +462,7 @@ public class ClusterSearcherTestCase { qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), - new SchemaInfo(List.of(schema.build()), Map.of()), + new SchemaInfo(List.of(schema.build()), List.of()), dispatchers, null, vipStatus, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 73975ecaa96..7a63eb07641 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -129,7 +129,7 @@ public class FastSearcherTestCase { new SummaryParameters(null), new ClusterParams("testhittype"), documentDb, - new SchemaInfo(List.of(schema.build()), Map.of())); + new SchemaInfo(List.of(schema.build()), List.of())); Query q = new Query("?query=foo"); Result result = doSearch(backend, q, 0, 10); assertFalse(backend.summaryNeedsQuery(q)); @@ -210,7 +210,7 @@ public class FastSearcherTestCase { private SchemaInfo schemaInfo(String schemaName) { var schema = new Schema.Builder(schemaName); schema.add(new RankProfile.Builder("default").build()); - return new SchemaInfo(List.of(schema.build()), Map.of()); + return new SchemaInfo(List.of(schema.build()), List.of()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index d70b42aa36b..07394676e09 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -261,14 +261,14 @@ public class QueryTestCase { @Test void testNoneHitsNegativeOffsetValue() { assertQueryError( - "?query=test&hits=(none)&offset=-10", + "?query=test&hits=(none)", "Could not set 'hits' to '(none)': '(none)' is not a valid integer"); } @Test void testFeedbackIsTransferredToResult() { assertQueryError( - "?query=test&hits=(none)&offset=-10", + "?query=test&hits=(none)", "Could not set 'hits' to '(none)': '(none)' is not a valid integer"); } diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index cbe4ddcbc63..03b53970550 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.search.Query; +import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; @@ -259,9 +260,9 @@ public class RankProfileInputTest { .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) .build()); - Map<String, List<String>> clusters = new HashMap<>(); - clusters.put("ab", List.of("a", "b")); - clusters.put("a", List.of("a")); + List<Cluster> clusters = new ArrayList<>(); + clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build()); + clusters.add(new Cluster.Builder("a").addSchema("a").build()); return new SchemaInfo(schemas, clusters); } diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index a46f3480d50..4aced1b5e25 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -83,17 +83,14 @@ public class SchemaInfoTester { .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) .build()); - Map<String, List<String>> clusters = new HashMap<>(); - clusters.put("ab", List.of("a", "b")); - clusters.put("a", List.of("a")); + List<Cluster> clusters = new ArrayList<>(); + clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build()); + clusters.add(new Cluster.Builder("a").addSchema("a").setStreaming(true).build()); return new SchemaInfo(schemas, clusters); } /** Creates the same schema info as createSchemaInfo from config objects. */ static SchemaInfo createSchemaInfoFromConfig() { - - var indexInfoConfig = new IndexInfoConfig.Builder(); - var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileCommon.name("commonProfile"); rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); @@ -141,7 +138,7 @@ public class SchemaInfoTester { schemaInfoInfoConfig.schema(schemaB); - // ----- Info about which schemas are in which clusters + // ----- Info about clusters var qrSearchersConfig = new QrSearchersConfig.Builder(); var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); clusterAB.name("ab"); @@ -149,10 +146,11 @@ public class SchemaInfoTester { qrSearchersConfig.searchcluster(clusterAB); var clusterA = new QrSearchersConfig.Searchcluster.Builder(); clusterA.name("a"); + clusterA.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING); clusterA.searchdef("a"); qrSearchersConfig.searchcluster(clusterA); - return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build()); + return new SchemaInfo(schemaInfoInfoConfig.build(), qrSearchersConfig.build()); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java new file mode 100644 index 00000000000..9367d9f335a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java @@ -0,0 +1,57 @@ +package com.yahoo.search.searchers.test; + +import com.yahoo.search.Query; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.FieldSet; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.QueryValidator; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author bratseth + */ +public class QueryValidatorFieldTypeTest { + + @Test + void testTensorsCannotBeSearchedForTerms() { + var test = new Schema.Builder("test") + .add(new Field.Builder("mytensor1", "tensor(x[100])").build()) + .add(new Field.Builder("mytensor2", "tensor<float>(x[100])").build()) + .add(new Field.Builder("mystring", "string").addAlias("fieldAlias").build()) + .add(new FieldSet.Builder("myFieldSet").addField("mystring").build()) + .build(); + var schemaInfo = new SchemaInfo(List.of(test), List.of()); + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + + assertSucceeds("?query=mystring:foo", execution); + assertSucceeds("?query=fieldAlias:foo", execution); + assertSucceeds("?query=myFieldSet:foo", execution); + assertSucceeds("?query=none:foo", execution); + assertFails("Cannot search for terms in 'mytensor1': It is a tensor field", + "?query=mytensor1:foo", execution); + assertFails("Cannot search for terms in 'mytensor2': It is a tensor field", + "?query=mytensor2:foo", execution); + } + private void assertSucceeds(String query, Execution execution) { + new QueryValidator().search(new Query(query), execution); + } + + private void assertFails(String expectedError, String query, Execution execution) { + try { + new QueryValidator().search(new Query(query), execution); + fail("Expected validation error from " + query); + } + catch (IllegalArgumentException e) { + // success + assertEquals(expectedError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java new file mode 100644 index 00000000000..b653e4d97aa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.search.Query; +import com.yahoo.search.schema.Cluster; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.QueryValidator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author bratseth + */ +public class QueryValidatorPrefixTest { + + @Disabled + @Test + void testPrefixRequiresAttribute() { + var indexing = new Cluster.Builder("indexing").addSchema("test1").build(); + var streaming = new Cluster.Builder("streaming").addSchema("test1").addSchema("test2").setStreaming(true).build(); + var schemaInfo = new SchemaInfo(List.of(schema("test1"), schema("test2")), List.of(indexing, streaming)); + + assertIndexingValidation("", schemaInfo); + assertIndexingValidation("sources=indexing", schemaInfo); + assertIndexingValidation("sources=indexing,streaming", schemaInfo); + assertIndexingValidation("sources=indexing,streaming,ignored", schemaInfo); + assertStreamingValidation("sources=streaming", schemaInfo); + assertStreamingValidation("sources=streaming,ignored", schemaInfo); + assertIndexingValidation("sources=test1", schemaInfo); + assertIndexingValidation("sources=test1,streaming", schemaInfo); + assertStreamingValidation("sources=test2,streaming", schemaInfo); + assertIndexingValidation("sources=test1,test2", schemaInfo); + assertStreamingValidation("sources=test2", schemaInfo); + } + + private Schema schema(String name) { + return new Schema.Builder(name) + .add(new Field.Builder("attributeOnly", "string").setAttribute(true).build()) + .add(new Field.Builder("indexOnly", "string").setIndex(true).build()) + .add(new Field.Builder("attributeAndIndex", "string").setAttribute(true).setIndex(true).build()) + .build(); + } + + private void assertIndexingValidation(String sourcesParameter, SchemaInfo schemaInfo) { + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution); + assertFails("'indexOnly' is not an attribute field: Prefix matching is not supported", + "?query=indexOnly:foo*&" + sourcesParameter, execution); + assertFails("'attributeAndIndex' is an index field: Prefix matching is not supported even when it is also an attribute", + "?query=attributeAndIndex:foo*&" + sourcesParameter, execution); + } + + private void assertStreamingValidation(String sourcesParameter, SchemaInfo schemaInfo) { + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution); + assertSucceeds("?query=indexOnly:foo*&" + sourcesParameter, execution); + assertSucceeds("?query=attributeAndIndex:foo*&" + sourcesParameter, execution); + } + + private void assertSucceeds(String query, Execution execution) { + new QueryValidator().search(new Query(query), execution); + } + + private void assertFails(String expectedError, String query, Execution execution) { + try { + new QueryValidator().search(new Query(query), execution); + fail("Expected validation error from " + query); + } + catch (IllegalArgumentException e) { + // success + assertEquals(expectedError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java deleted file mode 100644 index 8c525b2975a..00000000000 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchers.test; - -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; -import com.yahoo.prelude.SearchDefinition; -import com.yahoo.search.Query; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchers.QueryValidator; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author bratseth - */ -public class QueryValidatorTestCase { - - @Test - void testTensorsCannotBeSearchedForTerms() { - SearchDefinition sd = new SearchDefinition("test"); - sd.addCommand("mytensor1", "type tensor(x[100]"); - sd.addCommand("mytensor2", "type tensor<float>(x[100]"); - sd.addCommand("mystring", "type string"); - IndexModel model = new IndexModel(sd); - - IndexFacts indexFacts = new IndexFacts(model); - Execution execution = new Execution(Execution.Context.createContextStub(indexFacts)); - assertSucceeds("?query=mystring:foo", execution); - assertFails("Cannot search for terms in 'mytensor1': It is a tensor field", - "?query=mytensor1:foo", execution); - assertFails("Cannot search for terms in 'mytensor2': It is a tensor field", - "?query=mytensor2:foo", execution); - } - - @Test - void testPrefixRequiresAttribute() { - SearchDefinition sd = new SearchDefinition("test"); - sd.addCommand("attributeOnly", "type string") - .addCommand("attribute"); - sd.addCommand("indexOnly", "type string") - .addCommand("index"); - sd.addCommand("attributeAndIndex", "type string") - .addCommand("attribute") - .addCommand("index"); - IndexModel model = new IndexModel(sd); - - IndexFacts indexFacts = new IndexFacts(model); - Execution execution = new Execution(Execution.Context.createContextStub(indexFacts)); - - assertSucceeds("?query=attributeOnly:foo*", execution); - assertFails("'indexOnly' is not an attribute field: Prefix matching is not supported", - "?query=indexOnly:foo*", execution); - assertFails("'attributeAndIndex' is an index field: Prefix matching is not supported even when it is also an attribute", - "?query=attributeAndIndex:foo*", execution); - } - - private void assertSucceeds(String query, Execution execution) { - new QueryValidator().search(new Query(query), execution); - } - - private void assertFails(String expectedError, String query, Execution execution) { - try { - new QueryValidator().search(new Query(query), execution); - fail("Expected validation error from " + query); - } - catch (IllegalArgumentException e) { - // success - assertEquals(expectedError, e.getMessage()); - } - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 0bb3095fa9d..3a7641e7dc0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -74,7 +74,7 @@ public class YqlFieldAndSourceTestCase { .addField(FIELD2, "string").build()) .add((new DocumentSummary.Builder(SORTABLE_ATTRIBUTES_SUMMARY_CLASS).addField(FIELD2, "string").build())) .add((new DocumentSummary.Builder(THIRD_OPTION).addField(FIELD3, "string").build())); - return new SchemaInfo(List.of(schema.build()), Map.of()); + return new SchemaInfo(List.of(schema.build()), List.of()); } @AfterEach diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 10d3a5aeabe..2eb136056ac 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -76,9 +76,8 @@ public class SelectTestCase { "my.nested.title:madonna"); } - @Test - void testOr() throws Exception { + void testOr() { ObjectNode json_two_or = jsonMapper.createObjectNode(); ObjectNode json_three_or = jsonMapper.createObjectNode(); ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); @@ -100,7 +99,7 @@ public class SelectTestCase { } @Test - void testAnd() throws Exception { + void testAnd() { ObjectNode json_two_and = jsonMapper.createObjectNode(); ObjectNode json_three_and = jsonMapper.createObjectNode(); ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java index 578ccec7f40..19c03faae66 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -240,7 +240,7 @@ public class VdsStreamingSearcherTestCase { new SummaryParameters("default"), new ClusterParams("clusterName"), new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test")).build(), - new SchemaInfo(List.of(schema.build()), Map.of())); + new SchemaInfo(List.of(schema.build()), List.of())); // Magic query values are used to trigger specific behaviors from mock visitor. checkError(searcher, "/?query=noselection", |