diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-06-12 15:14:11 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-06-12 15:14:11 +0200 |
commit | 2092f374fc4f3de42da75f4660849af0f00f7a81 (patch) | |
tree | 1047036d19f30481925ab38b0601a85000121ad3 /config-model | |
parent | 537d80f3aad6351322ded0f3e300722cbcdba5d7 (diff) |
Move to SchemaInfo
Add the missing constructs to SchemaInfo to be able to use
it in place of IndexFacts for validation, and rewrite QueryValidator
to use it. The new validation (for prefix search on indexes) is disabled
until this is verified, so this should be a no-op.
Diffstat (limited to 'config-model')
11 files changed, 354 insertions, 11 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/Index.java b/config-model/src/main/java/com/yahoo/schema/Index.java index 190081cf80f..aa1d226d12e 100644 --- a/config-model/src/main/java/com/yahoo/schema/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/Index.java @@ -15,7 +15,7 @@ import java.util.Optional; import java.util.Set; /** - * An index definition in a search definition. + * An index definition in a schema. * Two indices are equal if they have the same name and the same settings, except * alias settings (which are excluded). * diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Index.java b/config-model/src/main/java/com/yahoo/schema/derived/Index.java index 3b5e617d3dc..2f5b674abee 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Index.java @@ -16,11 +16,11 @@ public class Index { /** The index type enumeration */ public static class Type { - public static final Type TEXT=new Type("text"); - public static final Type INT64=new Type("long"); - public static final Type BOOLEANTREE=new Type("booleantree"); + public static final Type TEXT = new Type("text"); + public static final Type INT64 = new Type("long"); + public static final Type BOOLEANTREE = new Type("booleantree"); - private String name; + private final String name; private Type(String name) { this.name=name; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index 517a43fe129..291c67cae02 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -1,7 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.derived; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.schema.Index; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; @@ -9,6 +23,7 @@ import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -52,11 +67,79 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public void getConfig(SchemaInfoConfig.Builder builder) { var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); schemaBuilder.name(schema.getName()); + addFieldsConfig(schemaBuilder); + addFieldSetConfig(schemaBuilder); addSummaryConfig(schemaBuilder); addRankProfilesConfig(schemaBuilder); builder.schema(schemaBuilder); } + private void addFieldsConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var field : schema.allFieldsList()) { + addFieldConfig(field, schemaBuilder); + for (var index : field.getIndices().values()) { + if ( ! index.getName().equals(field.getName())) // additional index + addFieldConfig(index, field.getDataType(), schemaBuilder); + } + for (var attribute : field.getAttributes().values()) { + if ( ! attribute.getName().equals(field.getName())) // additional attribute + addFieldConfig(attribute, field.getDataType(), schemaBuilder); + } + } + } + + private void addFieldConfig(ImmutableSDField field, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(field.getName()); + fieldBuilder.type(toTypeSpec(field.getDataType())); + for (var alias : field.getAliasToName().entrySet()) { + if (alias.getValue().equals(field.getName())) + fieldBuilder.alias(alias.getKey()); + } + fieldBuilder.attribute(field.doesAttributing()); + fieldBuilder.index(field.doesIndexing()); + schemaBuilder.field(fieldBuilder); + } + + // TODO: Make fields and indexes 1-1 so that this can be removed + private void addFieldConfig(Index index, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(index.getName()); + fieldBuilder.type(toTypeSpec(type)); + for (Iterator<String> i = index.aliasIterator(); i.hasNext(); ) + fieldBuilder.alias(i.next()); + fieldBuilder.attribute(false); + fieldBuilder.index(true); + schemaBuilder.field(fieldBuilder); + } + + // TODO: Make fields and attributes 1-1 so that this can be removed + private void addFieldConfig(Attribute attribute, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder(); + fieldBuilder.name(attribute.getName()); + fieldBuilder.type(toTypeSpec(type)); + for (var alias : attribute.getAliases()) + fieldBuilder.alias(alias); + fieldBuilder.attribute(true); + fieldBuilder.index(false); + schemaBuilder.field(fieldBuilder); + } + + private void addFieldSetConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var fieldSet : schema.fieldSets().builtInFieldSets().values()) + addFieldSetConfig(fieldSet, schemaBuilder); + for (var fieldSet : schema.fieldSets().userFieldSets().values()) + addFieldSetConfig(fieldSet, schemaBuilder); + } + + private void addFieldSetConfig(FieldSet fieldSet, SchemaInfoConfig.Schema.Builder schemaBuilder) { + var fieldSetBuilder = new SchemaInfoConfig.Schema.Fieldset.Builder(); + fieldSetBuilder.name(fieldSet.getName()); + for (String fieldName : fieldSet.getFieldNames()) + fieldSetBuilder.field(fieldName); + schemaBuilder.fieldset(fieldSetBuilder); + } + private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (var summary : summaries.asList()) { var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); @@ -88,6 +171,29 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc } } + /** Returns this type as a spec on the form following "field [name] type " in schemas. */ + private String toTypeSpec(DataType dataType) { + if (dataType instanceof PrimitiveDataType) + return dataType.getName(); + if (dataType instanceof AnnotationReferenceDataType annotationType) + return "annotationreference<" + annotationType.getAnnotationType().getName() + ">"; + if (dataType instanceof ArrayDataType arrayType) + return "array<" + toTypeSpec(arrayType.getNestedType()) + ">"; + if (dataType instanceof MapDataType mapType) + return "map<" + toTypeSpec(mapType.getKeyType()) + "," + toTypeSpec(mapType.getValueType()) + ">"; + if (dataType instanceof ReferenceDataType referenceType) + return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">"; + if (dataType instanceof NewDocumentReferenceDataType referenceType) + return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">"; + if (dataType instanceof StructuredDataType structType) + return structType.getName(); + if (dataType instanceof TensorDataType tensorType) + return tensorType.getTensorType().toString(); + if (dataType instanceof WeightedSetDataType weightedSetDataType) + return "weightedset<" + toTypeSpec(weightedSetDataType.getNestedType()) + ">"; + throw new IllegalArgumentException("Unknown data type " + dataType + " class " + dataType.getClass()); + } + /** A store of a *small* (in memory) amount of rank profile info. */ public static final class RankProfileInfo { diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index 919a6023151..5ffe4ec533c 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -216,7 +216,7 @@ public class SDDocumentType implements Cloneable { return f; } - public void addField(Field field) { + public Field addField(Field field) { verifyInheritance(field); for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { if (field.getName().equalsIgnoreCase((i.next()).getName())) { @@ -224,6 +224,7 @@ public class SDDocumentType implements Cloneable { } } docType.addField(field); + return field; } /** Parse-time inheritance check. */ diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 4eaa3e05047..7821c101880 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -556,8 +556,9 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } /** Adds an explicit index defined in this field */ - public void addIndex(Index index) { - indices.put(index.getName(),index); + public Index addIndex(Index index) { + indices.put(index.getName(), index); + return index; } /** @@ -624,13 +625,14 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { return attributes.get(getName()); } - public void addAttribute(Attribute attribute) { + public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); if (name == null || "".equals(name)) { name = getName(); attribute.setName(name); } attributes.put(attribute.getName(),attribute); + return attribute; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 080a2ca43dc..960850def1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -43,7 +43,7 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private IndexingDocproc indexingDocproc; + private final IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; diff --git a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg index e4bd88e6ac9..e04a8e6c1a7 100644 --- a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg @@ -1,4 +1,20 @@ schema[].name "child" +schema[].field[].name "parent_ref" +schema[].field[].type "reference<parent>" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "my_pos_zcurve" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "pos_zcurve" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "my_pos" +schema[].field[].type "position" +schema[].field[].attribute false +schema[].field[].index false schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "parent_ref" schema[].summaryclass[].fields[].type "longstring" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index 82bba81f0d5..8e4c1920e60 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -1,4 +1,72 @@ schema[].name "neuralnet" +schema[].field[].name "uniqueRCount" +schema[].field[].type "double" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "pinned" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "createdAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "updatedAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "uvCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "dvCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "aVoteCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "rCount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "uniqueRACount" +schema[].field[].type "int" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "rTo" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "markedAsAAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "normalizedTextScore" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "t" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "relevance" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "normalizedCS" +schema[].field[].type "float" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "laAt" +schema[].field[].type "long" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "hsScore" +schema[].field[].type "double" +schema[].field[].attribute true +schema[].field[].index false schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "rankfeatures" schema[].summaryclass[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg index 86cbfa0562d..8af39cab572 100644 --- a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg +++ b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg @@ -1,4 +1,8 @@ schema[].name "test" +schema[].field[].name "title" +schema[].field[].type "string" +schema[].field[].attribute false +schema[].field[].index true schema[].summaryclass[].name "default" schema[].summaryclass[].fields[].name "rankfeatures" schema[].summaryclass[].fields[].type "featuredata" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 622fcfdf4fd..3b3fe196cad 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search.test; +import com.yahoo.document.DataType; +import com.yahoo.schema.Index; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.model.VespaModel; @@ -13,10 +17,116 @@ import static org.junit.jupiter.api.Assertions.fail; public class SchemaInfoTestCase { + @Test + void testFieldWithAliasExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + field.getAliasToName().put("alias1", "f1"); + field.getAliasToName().put("alias2", "f1"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].alias[0] "alias1" + schema[0].field[0].alias[1] "alias2" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldsetExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + schema.getDocument().addField(new SDField("f1", DataType.STRING)); + schema.getDocument().addField(new SDField("f2", DataType.STRING)); + schema.fieldSets().addUserFieldSetItem("fs1", "f1"); + schema.fieldSets().addUserFieldSetItem("fs1", "f2"); + schema.fieldSets().addUserFieldSetItem("fs2", "f1"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f2" + schema[0].field[1].type "string" + schema[0].field[1].attribute false + schema[0].field[1].index false + schema[0].fieldset[0].name "fs1" + schema[0].fieldset[0].field[0] "f1" + schema[0].fieldset[0].field[1] "f2" + schema[0].fieldset[1].name "fs2" + schema[0].fieldset[1].field[0] "f1" + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldWithIndexExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var index = field.addIndex(new Index("f1Index")); + index.addAlias("a1"); + index.addAlias("a2"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f1Index" + schema[0].field[1].type "string" + schema[0].field[1].alias[0] "a1" + schema[0].field[1].alias[1] "a2" + schema[0].field[1].attribute false + schema[0].field[1].index true + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + + @Test + void testFieldWithAttributeExporting() { + var schemaInfoTester = new SchemaInfoTester(); + var schema = schemaInfoTester.createSchema("test"); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var attribute = field.addAttribute(new Attribute("f1Attribute", field.getDataType())); + attribute.getAliases().add("a1"); + attribute.getAliases().add("a2"); + assertEquals(""" + schema[0].name "test" + schema[0].field[0].name "f1" + schema[0].field[0].type "string" + schema[0].field[0].attribute false + schema[0].field[0].index false + schema[0].field[1].name "f1Attribute" + schema[0].field[1].type "string" + schema[0].field[1].alias[0] "a1" + schema[0].field[1].alias[1] "a2" + schema[0].field[1].attribute true + schema[0].field[1].index false + schema[0].summaryclass[0].name "default" + schema[0].summaryclass[0].fields[0].name "documentid" + schema[0].summaryclass[0].fields[0].type "longstring" + schema[0].summaryclass[0].fields[0].dynamic false""", + schemaInfoTester.schemaInfoConfig(schema)); + } + /** Schema-info should contain all schemas, independent of clusters. */ @Test void requireThatSchemaInfoIsAvailable() { - List.of(1.0, 2.0, 3.0).toArray(new Double[3]); String inputs = " rank-profile inputs {" + " inputs {" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java new file mode 100644 index 00000000000..b8bb83540d4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.derived.Summaries; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; + +/** + * @author bratseth + */ +public class SchemaInfoTester { + + public Schema createSchema(String name) { + var schema = new Schema(name, null); + var document = new SDDocumentType(name); + schema.addDocument(document); + schema.addSummary(new DocumentSummary("default", schema)); + return schema; + } + + public String schemaInfoConfig(Schema schema) { + var schemaInfo = new SchemaInfo(schema, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties())); + var schemaInfoConfigBuilder = new SchemaInfoConfig.Builder(); + schemaInfo.getConfig(schemaInfoConfigBuilder); + var schemaInfoConfig = schemaInfoConfigBuilder.build(); + return schemaInfoConfig.toString(); + } + +} |