diff options
author | Arne H Juul <arnej27959@users.noreply.github.com> | 2023-06-13 08:27:55 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-13 08:27:55 +0200 |
commit | 982817bd81908a320338ae1c2b1945392ca69398 (patch) | |
tree | 34f6c7b5ce6dcd0ab91cd9e0290afab27b44dd12 /container-search | |
parent | bb324300a8884035423dafaab1ebb7c72da2ae4c (diff) | |
parent | d876e1b4855c55ab6a8544001009418efec9abac (diff) |
Merge pull request #27382 from vespa-engine/bratseth/validate-prefix-matching-take-2-alternative-ending
Bratseth/validate prefix matching take 2 alternative ending
Diffstat (limited to 'container-search')
28 files changed, 935 insertions, 155 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 84411b31274..e439f7905cc 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1697,7 +1697,7 @@ "methods" : [ "public void <init>()", "public abstract boolean visit(com.yahoo.prelude.query.Item)", - "public abstract void onExit()" + "public void onExit()" ], "fields" : [ ] }, @@ -8086,6 +8086,36 @@ ], "fields" : [ ] }, + "com.yahoo.search.schema.Cluster$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.Cluster$Builder setStreaming(boolean)", + "public com.yahoo.search.schema.Cluster$Builder addSchema(java.lang.String)", + "public com.yahoo.search.schema.Cluster build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Cluster" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.lang.String name()", + "public boolean isStreaming()", + "public java.util.Set schemas()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, "com.yahoo.search.schema.DocumentSummary$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -8169,6 +8199,148 @@ ], "fields" : [ ] }, + "com.yahoo.search.schema.Field$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String)", + "public com.yahoo.search.schema.Field$Builder addAlias(java.lang.String)", + "public com.yahoo.search.schema.Field$Builder setAttribute(boolean)", + "public com.yahoo.search.schema.Field$Builder setIndex(boolean)", + "public com.yahoo.search.schema.Field build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field$TensorFieldType" : { + "superClass" : "com.yahoo.search.schema.Field$Type", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorType tensorType()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field$Type$Kind" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.search.schema.Field$Type$Kind[] values()", + "public static com.yahoo.search.schema.Field$Type$Kind valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.search.schema.Field$Type$Kind ANNOTATIONREFERENCE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind ARRAY", + "public static final enum com.yahoo.search.schema.Field$Type$Kind BOOL", + "public static final enum com.yahoo.search.schema.Field$Type$Kind BYTE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind DOUBLE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind FLOAT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind INT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind LONG", + "public static final enum com.yahoo.search.schema.Field$Type$Kind MAP", + "public static final enum com.yahoo.search.schema.Field$Type$Kind POSITION", + "public static final enum com.yahoo.search.schema.Field$Type$Kind PREDICATE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind RAW", + "public static final enum com.yahoo.search.schema.Field$Type$Kind REFERENCE", + "public static final enum com.yahoo.search.schema.Field$Type$Kind STRING", + "public static final enum com.yahoo.search.schema.Field$Type$Kind STRUCT", + "public static final enum com.yahoo.search.schema.Field$Type$Kind TENSOR", + "public static final enum com.yahoo.search.schema.Field$Type$Kind URL", + "public static final enum com.yahoo.search.schema.Field$Type$Kind WEIGHTEDSET" + ] + }, + "com.yahoo.search.schema.Field$Type" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public com.yahoo.search.schema.Field$Type$Kind kind()", + "public static com.yahoo.search.schema.Field$Type from(java.lang.String)" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.Field" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.search.schema.FieldInfo" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.search.schema.Field$Builder)", + "public java.lang.String name()", + "public com.yahoo.search.schema.Field$Type type()", + "public java.util.Set aliases()", + "public boolean isAttribute()", + "public boolean isIndex()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.lang.String name()", + "public abstract com.yahoo.search.schema.Field$Type type()", + "public abstract boolean isAttribute()", + "public abstract boolean isIndex()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldSet$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.FieldSet$Builder addField(java.lang.String)", + "public com.yahoo.search.schema.FieldSet build()" + ], + "fields" : [ ] + }, + "com.yahoo.search.schema.FieldSet" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.search.schema.FieldInfo" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.lang.String name()", + "public com.yahoo.search.schema.Field$Type type()", + "public boolean isAttribute()", + "public boolean isIndex()", + "public java.util.Set fieldNames()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, "com.yahoo.search.schema.RankProfile$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -8210,6 +8382,8 @@ ], "methods" : [ "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.Field)", + "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.FieldSet)", "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.RankProfile)", "public com.yahoo.search.schema.Schema$Builder add(com.yahoo.search.schema.DocumentSummary)", "public com.yahoo.search.schema.Schema build()" @@ -8224,8 +8398,10 @@ ], "methods" : [ "public java.lang.String name()", + "public java.util.Map fields()", "public java.util.Map rankProfiles()", "public java.util.Map documentSummaries()", + "public java.util.Optional fieldInfo(java.lang.String)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -8239,6 +8415,8 @@ "public" ], "methods" : [ + "public boolean isStreaming()", + "public java.util.Optional fieldInfo(java.lang.String)", "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" ], "fields" : [ ] @@ -8250,9 +8428,10 @@ "public" ], "methods" : [ - "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", - "public void <init>(java.util.List, java.util.Map)", + "public void <init>(com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", + "public void <init>(java.util.List, java.util.List)", "public java.util.Map schemas()", + "public java.util.Map clusters()", "public com.yahoo.search.schema.SchemaInfo$Session newSession(com.yahoo.search.Query)", "public static com.yahoo.search.schema.SchemaInfo empty()", "public boolean equals(java.lang.Object)", diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index e245faec919..af8f63ab9f2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -36,6 +36,7 @@ public class Index { private boolean hostIndex = false; private StemMode stemMode = StemMode.NONE; private boolean isAttribute = false; + private boolean isIndex = false; private boolean isDefaultPosition = false; private boolean dynamicSummary=false; private boolean highlightSummary=false; @@ -157,6 +158,8 @@ public class Index { setNGram(true, Integer.parseInt(command.substring(6))); } else if (command.equals("attribute")) { setAttribute(true); + } else if (command.equals("index")) { + setIndex(true); } else if (command.equals("default-position")) { setDefaultPosition(true); } else if (command.equals("plain-tokens")) { @@ -273,6 +276,12 @@ public class Index { this.isAttribute = isAttribute; } + public boolean isIndex() { return isIndex; } + + public void setIndex(boolean isIndex) { + this.isIndex = isIndex; + } + public boolean hasPlainTokens() { return plainTokens; } public void setPlainTokens(boolean plainTokens) { diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index 1d9e32ec374..a232841f29f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -86,12 +86,13 @@ public class SearchDefinition { return idx; } - public void addCommand(String indexName, String commandString) { + public Index addCommand(String indexName, String commandString) { Index index = getOrCreateIndex(indexName); index.addCommand(commandString); if (index.isDefaultPosition()) { defaultPosition = index.getName(); } + return index; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java index 7f37b77919b..e278ad38487 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java @@ -27,8 +27,9 @@ public final class ToolBox { /** * Invoked when all sub-items have been visited, or immediately after * visit() if there are no sub-items or visit() returned false. + * This default implementation does nothing. */ - public abstract void onExit(); + public void onExit() {} } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 0c65f4bf8dc..8a9ea281b0c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -168,7 +168,7 @@ public class SelectParser implements Parser { Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + - " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } try { @@ -186,32 +186,15 @@ public class SelectParser implements Parser { inspector.traverse((ObjectTraverser) (key, value) -> { String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; switch (type) { - case AND: - item[0] = buildAnd(key, value); - break; - case AND_NOT: - item[0] = buildNotAnd(key, value); - break; - case OR: - item[0] = buildOr(key, value); - break; - case EQ: - item[0] = buildEquals(key, value); - break; - case RANGE: - item[0] = buildRange(key, value); - break; - case CONTAINS: - item[0] = buildTermSearch(key, value); - break; - case MATCHES: - item[0] = buildRegExpSearch(key, value); - break; - case CALL: - item[0] = buildFunctionCall(key, value); - break; - default: - throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); + case AND -> item[0] = buildAnd(key, value); + case AND_NOT -> item[0] = buildNotAnd(key, value); + case OR -> item[0] = buildOr(key, value); + case EQ -> item[0] = buildEquals(key, value); + case RANGE -> item[0] = buildRange(key, value); + case CONTAINS -> item[0] = buildTermSearch(key, value); + case MATCHES -> item[0] = buildRegExpSearch(key, value); + case CALL -> item[0] = buildFunctionCall(key, value); + default -> throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); } }); return item[0]; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Cluster.java b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java new file mode 100644 index 00000000000..f5ea4fdffc7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Information about the search aspects of a content cluster. + * + * @author bratseth + */ +@Beta +public class Cluster { + + private final String name; + private final boolean isStreaming; + private final Set<String> schemas; + + private Cluster(Builder builder) { + this.name = builder.name; + this.isStreaming = builder.isStreaming; + this.schemas = Set.copyOf(builder.schemas); + } + + public String name() { return name; } + + /** Returns true if this cluster uses streaming search. */ + public boolean isStreaming() { return isStreaming; } + + /** Returns the names of the subset of all schemas that are present in this cluster. */ + public Set<String> schemas() { return schemas; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Cluster other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( this.isStreaming != other.isStreaming()) return false; + if ( ! this.schemas.equals(other.schemas)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, isStreaming, schemas); + } + + @Override + public String toString() { return "cluster '" + name + "'"; } + + public static class Builder { + + private final String name; + private boolean isStreaming = false; + private final Set<String> schemas = new HashSet<>(); + + public Builder(String name) { + this.name = name; + } + + public Builder setStreaming(boolean isStreaming) { + this.isStreaming = isStreaming; + return this; + } + + public Builder addSchema(String schema) { + schemas.add(schema); + return this; + } + + public Cluster build() { + return new Cluster(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java index 0aec6b0a4f6..a4f208710a0 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java +++ b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; -import java.util.ArrayList; +import com.yahoo.api.annotations.Beta; + import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -15,6 +15,7 @@ import java.util.Objects; * * @author bratseth */ +@Beta public class DocumentSummary { private final String name; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Field.java b/container-search/src/main/java/com/yahoo/search/schema/Field.java new file mode 100644 index 00000000000..ad949d5bad9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/Field.java @@ -0,0 +1,177 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.tensor.TensorType; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * A field in a schema. + * + * @author bratseth + */ +@Beta +public class Field implements FieldInfo { + + private final String name; + private final Type type; + private final boolean isAttribute; + private final boolean isIndex; + private final Set<String> aliases; + + public Field(Builder builder) { + this.name = builder.name; + this.type = builder.type; + this.isAttribute = builder.isAttribute; + this.isIndex = builder.isIndex; + this.aliases = Set.copyOf(builder.aliases); + } + + @Override + public String name() { return name; } + + @Override + public Type type() { return type; } + + public Set<String> aliases() { return aliases; } + + /** Returns whether this field is an attribute, i.e. does indexing: attribute. */ + @Override + public boolean isAttribute() { return isAttribute; } + + /** Returns whether this field is an index, i.e. does indexing: index. */ + @Override + public boolean isIndex() { return isIndex; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Field other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( this.isAttribute != other.isAttribute) return false; + if ( this.isIndex != other.isIndex) return false; + if ( ! this.aliases.equals(other.aliases)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, isAttribute, isIndex, aliases); + } + + @Override + public String toString() { return "field '" + name + "'"; } + + public static class Type { + + private final Kind kind; + + /** The kind of type this is. */ + public enum Kind { + ANNOTATIONREFERENCE, ARRAY, BOOL, BYTE, DOUBLE, FLOAT, INT, LONG, MAP, POSITION, PREDICATE, RAW, REFERENCE, STRING, STRUCT, TENSOR, URL, WEIGHTEDSET; + } + + private Type(Kind kind) { + this.kind = kind; + } + + /** + * Returns the kind of type this is. + * Structured types have additional information in the subclass specific to that kind of type. + */ + public Kind kind() { return kind; } + + /** Creates this from a type string on the syntax following "field [name] type " in a schema definition. */ + public static Type from(String typeString) { + if (typeString.startsWith("annotationreference<")) + return new Type(Kind.ANNOTATIONREFERENCE); // TODO: Model as subclass + if (typeString.startsWith("array<")) + return new Type(Kind.ARRAY); // TODO: Model as subclass + if (typeString.equals("bool")) + return new Type(Kind.BOOL); + if (typeString.equals("byte")) + return new Type(Kind.BYTE); + if (typeString.equals("double")) + return new Type(Kind.DOUBLE); + if (typeString.equals("float")) + return new Type(Kind.FLOAT); + if (typeString.equals("int")) + return new Type(Kind.INT); + if (typeString.equals("long")) + return new Type(Kind.LONG); + if (typeString.startsWith("map<")) + return new Type(Kind.MAP); // TODO: Model as subclass + if (typeString.equals("position")) + return new Type(Kind.POSITION); + if (typeString.equals("predicate")) + return new Type(Kind.PREDICATE); + if (typeString.equals("raw")) + return new Type(Kind.RAW); + if (typeString.startsWith("reference<")) + return new Type(Kind.REFERENCE); // TODO: Model as subclass + if (typeString.equals("string")) + return new Type(Kind.STRING); + if (typeString.startsWith("tensor<") || typeString.startsWith("tensor(")) + return new TensorFieldType(TensorType.fromSpec(typeString)); + if (typeString.equals("url")) + return new Type(Kind.URL); + if (typeString.startsWith("weightedset<")) + return new Type(Kind.WEIGHTEDSET); // TODO: Model as subclass + else + return new Type(Kind.STRUCT); // TODO: Model as a subclass + } + + } + + public static class TensorFieldType extends Type { + + private final TensorType tensorType; + + public TensorFieldType(TensorType tensorType) { + super(Kind.TENSOR); + this.tensorType = tensorType; + } + + public TensorType tensorType() { return tensorType; } + + } + + public static class Builder { + + private final String name; + private final Type type; + private final Set<String> aliases = new HashSet<>(); + private boolean isAttribute; + private boolean isIndex; + + public Builder(String name, String typeString) { + this.name = name; + this.type = Type.from(typeString); + } + + public Builder addAlias(String alias) { + aliases.add(alias); + return this; + } + + public Builder setAttribute(boolean isAttribute) { + this.isAttribute = isAttribute; + return this; + } + + public Builder setIndex(boolean isIndex) { + this.isIndex = isIndex; + return this; + } + + public Field build() { + return new Field(this); + } + + } + + + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java new file mode 100644 index 00000000000..c3f6f22f1b9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.Set; + +/** + * Information about a field or field set. + * + * @author bratseth + */ +@Beta +public interface FieldInfo { + + /** Returns the name of this field or field set. */ + String name(); + + Field.Type type(); + + /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */ + boolean isAttribute(); + + /** Returns whether this field is index(es), i.e. does indexing: index. */ + boolean isIndex(); + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java new file mode 100644 index 00000000000..33c57dd1238 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * A set of fields which can be queried as one. + * + * @author bratseth + */ +@Beta +public class FieldSet implements FieldInfo { + + private final String name; + private final Set<String> fieldNames; + + // Assigned when this is added to a schema + private Schema schema = null; + + private FieldSet(Builder builder) { + this.name = builder.name; + this.fieldNames = Set.copyOf(builder.fieldNames); + } + + @Override + public String name() { return name; } + + @Override + public Field.Type type() { + if (schema == null || fieldNames.isEmpty()) return null; + return randomFieldInThis().type(); + } + + /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */ + @Override + public boolean isAttribute() { + if (schema == null || fieldNames.isEmpty()) return false; + return randomFieldInThis().isAttribute(); + } + + /** Returns whether this field is index(es), i.e. does indexing: index. */ + @Override + public boolean isIndex() { + if (schema == null || fieldNames.isEmpty()) return false; + return randomFieldInThis().isIndex(); + } + + void setSchema(Schema schema) { + if ( this.schema != null) + throw new IllegalStateException("Cannot add field set '" + name + "' to schema '" + schema.name() + + "' as it is already added to schema '" + this.schema.name() + "'"); + this.schema = schema; + } + + /** Use a random field in this to determine its properties. Any inconsistency will have been warned about on deploy. */ + private Field randomFieldInThis() { + return schema.fields().get(fieldNames.iterator().next()); + } + + public Set<String> fieldNames() { return fieldNames; } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof FieldSet other)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( ! this.fieldNames.equals(other.fieldNames)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, fieldNames); + } + + @Override + public String toString() { return "field set '" + name + "'"; } + + public static class Builder { + + private final String name; + private final Set<String> fieldNames = new HashSet<>(); + + public Builder(String name) { + this.name = name; + } + + public Builder addField(String fieldName) { + fieldNames.add(fieldName); + return this; + } + + public FieldSet build() { + return new FieldSet(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index 85bb3915975..5eba6c220bb 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; +import com.yahoo.api.annotations.Beta; import com.yahoo.tensor.TensorType; import java.util.Collections; @@ -14,6 +15,7 @@ import java.util.Objects; * * @author bratseth */ +@Beta public class RankProfile { private final String name; diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java index c20aa1e81bd..20a776dc53e 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/Schema.java +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -8,6 +8,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * Information about a schema which is part of the application running this. @@ -20,25 +21,57 @@ import java.util.Objects; public class Schema { private final String name; + private final Map<String, Field> fields; + private final Map<String, FieldSet> fieldSets; private final Map<String, RankProfile> rankProfiles; private final Map<String, DocumentSummary> documentSummaries; + /** Fields indexed by both name and aliases. */ + private final Map<String, Field> fieldsByAliases; + private Schema(Builder builder) { this.name = builder.name; + this.fields = Collections.unmodifiableMap(builder.fields); + this.fieldSets = Collections.unmodifiableMap(builder.fieldSets); this.rankProfiles = Collections.unmodifiableMap(builder.rankProfiles); this.documentSummaries = Collections.unmodifiableMap(builder.documentSummaries); + + fieldSets.values().forEach(fieldSet -> fieldSet.setSchema(this)); rankProfiles.values().forEach(rankProfile -> rankProfile.setSchema(this)); + + fieldsByAliases = new HashMap<>(); + for (Field field : fields.values()) { + fieldsByAliases.put(field.name(), field); + field.aliases().forEach(alias -> fieldsByAliases.put(alias, field)); + } } public String name() { return name; } + public Map<String, Field> fields() { return fields; } public Map<String, RankProfile> rankProfiles() { return rankProfiles; } public Map<String, DocumentSummary> documentSummaries() { return documentSummaries; } + /** + * Looks up a field or field set by the given name or alias in this schema. + * + * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up + * @return information about the field or field set with the given name, or empty if no item with this name exists + */ + public Optional<FieldInfo> fieldInfo(String fieldName) { + if (fieldName.isEmpty()) + fieldName = "default"; + Field field = fieldsByAliases.get(fieldName); + if (field != null) return Optional.of(field); + return Optional.ofNullable(fieldSets.get(fieldName)); + } + @Override public boolean equals(Object o) { if (o == this) return true; if ( ! (o instanceof Schema other)) return false; if ( ! other.name.equals(this.name)) return false; + if ( ! other.fields.equals(this.fields)) return false; + if ( ! other.fieldSets.equals(this.fieldSets)) return false; if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; if ( ! other.documentSummaries.equals(this.documentSummaries)) return false; return true; @@ -57,6 +90,8 @@ public class Schema { public static class Builder { private final String name; + private final Map<String, Field> fields = new LinkedHashMap<>(); + private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>(); private final Map<String, RankProfile> rankProfiles = new LinkedHashMap<>(); private final Map<String, DocumentSummary> documentSummaries = new LinkedHashMap<>(); @@ -64,6 +99,16 @@ public class Schema { this.name = Objects.requireNonNull(name); } + public Builder add(Field field) { + fields.put(field.name(), field); + return this; + } + + public Builder add(FieldSet fieldSet) { + fieldSets.put(fieldSet.name(), fieldSet); + return this; + } + public Builder add(RankProfile profile) { rankProfiles.put(profile.name(), profile); return this; diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index d29964ea9c5..71bb00b39c5 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -17,6 +17,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -35,37 +36,38 @@ import java.util.stream.Collectors; */ // NOTES: // This should replace IndexFacts, and probably DocumentDatabase. -// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information. -// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation -// given a set of possible schemas: The session mechanism is present here to make that efficient when added -// (resolving schema subsets for every field lookup is too expensive). +// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain complete field information. @Beta public class SchemaInfo { - private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); + private static final SchemaInfo empty = new SchemaInfo(List.of(), List.of()); private final Map<String, Schema> schemas; - /** The schemas contained in each content cluster indexed by cluster name */ - private final Map<String, List<String>> clusters; + private final Map<String, Cluster> clusters; @Inject - public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future - SchemaInfoConfig schemaInfoConfig, + public SchemaInfo(SchemaInfoConfig schemaInfoConfig, QrSearchersConfig qrSearchersConfig) { this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); } - public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) { + public SchemaInfo(List<Schema> schemas, List<Cluster> clusters) { Map<String, Schema> schemaMap = new LinkedHashMap<>(); schemas.forEach(schema -> schemaMap.put(schema.name(), schema)); this.schemas = Collections.unmodifiableMap(schemaMap); - this.clusters = Collections.unmodifiableMap(clusters); + + Map<String, Cluster> clusterMap = new LinkedHashMap<>(); + clusters.forEach(cluster -> clusterMap.put(cluster.name(), cluster)); + this.clusters = Collections.unmodifiableMap(clusterMap); } /** Returns all schemas configured in this application, indexed by schema name. */ public Map<String, Schema> schemas() { return schemas; } + /** Returns information about all clusters available for searching in this applications, indexed by cluyster name. */ + public Map<String, Cluster> clusters() { return clusters; } + public Session newSession(Query query) { return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); } @@ -75,8 +77,7 @@ public class SchemaInfo { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof SchemaInfo)) return false; - SchemaInfo other = (SchemaInfo)o; + if ( ! (o instanceof SchemaInfo other)) return false; if ( ! other.schemas.equals(this.schemas)) return false; if ( ! other.clusters.equals(this.clusters)) return false; return true; @@ -88,15 +89,61 @@ public class SchemaInfo { /** The schema information resolved to be relevant to this session. */ public static class Session { + private final boolean isStreaming; private final Collection<Schema> schemas; private Session(Set<String> sources, Set<String> restrict, - Map<String, List<String>> clusters, + Map<String, Cluster> clusters, Map<String, Schema> candidates) { + this.isStreaming = resolveStreaming(sources, clusters); this.schemas = resolveSchemas(sources, restrict, clusters, candidates.values()); } + /** Returns true if this only searches streaming clusters. */ + public boolean isStreaming() { return isStreaming; } + + /** + * Looks up a field or field set by the given name or alias + * in the schemas resolved for this query. + * + * If there are several fields or field sets by this name or alias across the schemas of this session, + * one is chosen by random. + * + * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up. + * @return the appropriate field or empty if no field or field set has this name or alias + */ + public Optional<FieldInfo> fieldInfo(String fieldName) { + for (var schema : schemas) { + Optional<FieldInfo> field = schema.fieldInfo(fieldName); + if (field.isPresent()) + return field; + } + return Optional.empty(); + } + + private static boolean resolveStreaming(Set<String> sources, Map<String, Cluster> clusters) { + if (sources.isEmpty()) return clusters.values().stream().allMatch(Cluster::isStreaming); + + var matchedClusters = sources.stream().map(source -> clusterOfSource(source, clusters)).filter(Objects::nonNull).toList(); + if (matchedClusters.isEmpty()) return false; + return matchedClusters.stream().allMatch(Cluster::isStreaming); + } + + /** + * A source name is either a cluster or a schema. + * Returns the cluster which either is or contains this name, if any. + */ + private static Cluster clusterOfSource(String source, Map<String, Cluster> clusters) { + var cluster = clusters.get(source); + if (cluster != null) return cluster; + for (var c : clusters.values()) { + if (c.schemas().contains(source)) + return c; + } + return null; + } + /** * Given a search list which is a mixture of schemas and cluster * names, and a restrict list which is a list of schemas, return a @@ -106,7 +153,7 @@ public class SchemaInfo { */ private static Collection<Schema> resolveSchemas(Set<String> sources, Set<String> restrict, - Map<String, List<String>> clusters, + Map<String, Cluster> clusters, Collection<Schema> candidates) { if (sources.isEmpty()) return restrict.isEmpty() ? candidates : keep(restrict, candidates); @@ -114,7 +161,7 @@ public class SchemaInfo { Set<String> schemaNames = new HashSet<>(); for (String source : sources) { if (clusters.containsKey(source)) // source is a cluster - schemaNames.addAll(clusters.get(source)); + schemaNames.addAll(clusters.get(source).schemas()); else // source is a schema schemaNames.add(source); } @@ -126,13 +173,6 @@ public class SchemaInfo { return schemas.stream().filter(schema -> names.contains(schema.name())).toList(); } - private List<RankProfile> profilesNamed(String name) { - return schemas.stream() - .filter(schema -> schema.rankProfiles().containsKey(name)) - .map(schema -> schema.rankProfiles().get(name)) - .toList(); - } - /** * Returns the type of the given rank feature name in the given profile, * if it can be uniquely determined. @@ -165,6 +205,13 @@ public class SchemaInfo { return foundType; } + private List<RankProfile> profilesNamed(String name) { + return schemas.stream() + .filter(schema -> schema.rankProfiles().containsKey(name)) + .map(schema -> schema.rankProfiles().get(name)) + .toList(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 6947a93a833..1b9ba397105 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -6,10 +6,7 @@ import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; /** * Translation between schema info configuration and schema objects. @@ -47,14 +44,15 @@ class SchemaInfoConfigurer { return builder.build(); } - static Map<String, List<String>> toClusters(QrSearchersConfig config) { - Map<String, List<String>> clusters = new HashMap<>(); + static List<Cluster> toClusters(QrSearchersConfig config) { + List<Cluster> clusters = new ArrayList<>(); for (int i = 0; i < config.searchcluster().size(); ++i) { - List<String> schemas = new ArrayList<>(); String clusterName = config.searchcluster(i).name(); - for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j) - schemas.add(config.searchcluster(i).searchdef(j)); - clusters.put(clusterName, schemas); + var clusterInfo = new Cluster.Builder(clusterName); + clusterInfo.setStreaming(config.searchcluster(i).indexingmode() == QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING); + for (var schemaDef : config.searchcluster(i).searchdef()) + clusterInfo.addSchema(schemaDef); + clusters.add(clusterInfo.build()); } return clusters; } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index bfc4219eabc..87880ce2445 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -63,7 +63,7 @@ public class ExecutionFactory extends AbstractComponent { Executor executor) { this(chainsConfig, indexInfo, - new SchemaInfo(indexInfo, schemaInfo, clusters), + new SchemaInfo(schemaInfo, clusters), clusters, searchers, specialTokens, diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java index a2e3d038053..0b435c2e32d 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java @@ -3,15 +3,16 @@ package com.yahoo.search.searchers; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.HasIndexItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.FieldInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; @@ -28,31 +29,61 @@ public class QueryValidator extends Searcher { @Override public Result search(Query query, Execution execution) { - IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); - ToolBox.visit(new ItemValidator(session), query.getModel().getQueryTree().getRoot()); + var session = execution.context().schemaInfo().newSession(query); + ToolBox.visit(new TermSearchValidator(session), query.getModel().getQueryTree().getRoot()); + // ToolBox.visit(new PrefixSearchValidator(session), query.getModel().getQueryTree().getRoot()); TODO: Enable check and QueryValidatorPrefixTest return execution.search(query); } - private static class ItemValidator extends ToolBox.QueryVisitor { + private abstract static class TermValidator extends ToolBox.QueryVisitor { - IndexFacts.Session session; + final SchemaInfo.Session schema; - public ItemValidator(IndexFacts.Session session) { - this.session = session; + public TermValidator(SchemaInfo.Session schema) { + this.schema = schema; + } + + } + + private static class TermSearchValidator extends TermValidator { + + public TermSearchValidator(SchemaInfo.Session schema) { + super(schema); } @Override public boolean visit(Item item) { - if (item instanceof HasIndexItem) { - String indexName = ((HasIndexItem)item).getIndexName(); - if (session.getIndex(indexName).isTensor()) - throw new IllegalArgumentException("Cannot search '" + indexName + "': It is a tensor field"); + if (item instanceof HasIndexItem indexItem) { + var field = schema.fieldInfo(indexItem.getIndexName()); + if (! field.isPresent()) return true; + if (field.get().type().kind() == Field.Type.Kind.TENSOR) + throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() + + "': It is a tensor field"); } return true; } + } + + private static class PrefixSearchValidator extends TermValidator { + + public PrefixSearchValidator(SchemaInfo.Session schema) { + super(schema); + } + @Override - public void onExit() { } + public boolean visit(Item item) { + if (schema.isStreaming()) return true; // prefix is always supported + if (item instanceof PrefixItem prefixItem) { + var field = schema.fieldInfo(prefixItem.getIndexName()); + if (! field.isPresent()) return true; + if ( ! field.get().isAttribute()) + throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is not an attribute field: Prefix matching is not supported"); + if (field.get().isIndex()) // index overrides attribute + throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is an index field: Prefix matching is not supported even when it is also an attribute"); + } + return true; + } } diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 2da6d621973..9018d41d5d9 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -4,7 +4,18 @@ namespace=search.config ## The name of this schema schema[].name string -## The name of the summary class +## Information about a schema field (currently incomplete) +schema[].field[].name string +schema[].field[].type string +schema[].field[].alias[] string +schema[].field[].attribute bool +schema[].field[].index bool + +## Field sets +schema[].fieldset[].name string +schema[].fieldset[].field[] string + +## Information about a summary class schema[].summaryclass[].name string ## The name of a field in the summary class schema[].summaryclass[].fields[].name string diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 06ae9923dae..c164fd3eb1c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -277,7 +277,7 @@ public class ClusterSearcherTestCase { schemaBuilder.add(new RankProfile.Builder("testprofile").build()); schemas.add(schemaBuilder.build()); } - return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, Map.of()))); + return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, List.of()))); } finally { cluster.deconstruct(); } @@ -462,7 +462,7 @@ public class ClusterSearcherTestCase { qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), - new SchemaInfo(List.of(schema.build()), Map.of()), + new SchemaInfo(List.of(schema.build()), List.of()), dispatchers, null, vipStatus, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 73975ecaa96..7a63eb07641 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -129,7 +129,7 @@ public class FastSearcherTestCase { new SummaryParameters(null), new ClusterParams("testhittype"), documentDb, - new SchemaInfo(List.of(schema.build()), Map.of())); + new SchemaInfo(List.of(schema.build()), List.of())); Query q = new Query("?query=foo"); Result result = doSearch(backend, q, 0, 10); assertFalse(backend.summaryNeedsQuery(q)); @@ -210,7 +210,7 @@ public class FastSearcherTestCase { private SchemaInfo schemaInfo(String schemaName) { var schema = new Schema.Builder(schemaName); schema.add(new RankProfile.Builder("default").build()); - return new SchemaInfo(List.of(schema.build()), Map.of()); + return new SchemaInfo(List.of(schema.build()), List.of()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index d70b42aa36b..07394676e09 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -261,14 +261,14 @@ public class QueryTestCase { @Test void testNoneHitsNegativeOffsetValue() { assertQueryError( - "?query=test&hits=(none)&offset=-10", + "?query=test&hits=(none)", "Could not set 'hits' to '(none)': '(none)' is not a valid integer"); } @Test void testFeedbackIsTransferredToResult() { assertQueryError( - "?query=test&hits=(none)&offset=-10", + "?query=test&hits=(none)", "Could not set 'hits' to '(none)': '(none)' is not a valid integer"); } diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index cbe4ddcbc63..03b53970550 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.search.Query; +import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; @@ -259,9 +260,9 @@ public class RankProfileInputTest { .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) .build()); - Map<String, List<String>> clusters = new HashMap<>(); - clusters.put("ab", List.of("a", "b")); - clusters.put("a", List.of("a")); + List<Cluster> clusters = new ArrayList<>(); + clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build()); + clusters.add(new Cluster.Builder("a").addSchema("a").build()); return new SchemaInfo(schemas, clusters); } diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index a46f3480d50..4aced1b5e25 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -83,17 +83,14 @@ public class SchemaInfoTester { .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) .build()); - Map<String, List<String>> clusters = new HashMap<>(); - clusters.put("ab", List.of("a", "b")); - clusters.put("a", List.of("a")); + List<Cluster> clusters = new ArrayList<>(); + clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build()); + clusters.add(new Cluster.Builder("a").addSchema("a").setStreaming(true).build()); return new SchemaInfo(schemas, clusters); } /** Creates the same schema info as createSchemaInfo from config objects. */ static SchemaInfo createSchemaInfoFromConfig() { - - var indexInfoConfig = new IndexInfoConfig.Builder(); - var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileCommon.name("commonProfile"); rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); @@ -141,7 +138,7 @@ public class SchemaInfoTester { schemaInfoInfoConfig.schema(schemaB); - // ----- Info about which schemas are in which clusters + // ----- Info about clusters var qrSearchersConfig = new QrSearchersConfig.Builder(); var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); clusterAB.name("ab"); @@ -149,10 +146,11 @@ public class SchemaInfoTester { qrSearchersConfig.searchcluster(clusterAB); var clusterA = new QrSearchersConfig.Searchcluster.Builder(); clusterA.name("a"); + clusterA.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING); clusterA.searchdef("a"); qrSearchersConfig.searchcluster(clusterA); - return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build()); + return new SchemaInfo(schemaInfoInfoConfig.build(), qrSearchersConfig.build()); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java new file mode 100644 index 00000000000..9367d9f335a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java @@ -0,0 +1,57 @@ +package com.yahoo.search.searchers.test; + +import com.yahoo.search.Query; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.FieldSet; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.QueryValidator; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author bratseth + */ +public class QueryValidatorFieldTypeTest { + + @Test + void testTensorsCannotBeSearchedForTerms() { + var test = new Schema.Builder("test") + .add(new Field.Builder("mytensor1", "tensor(x[100])").build()) + .add(new Field.Builder("mytensor2", "tensor<float>(x[100])").build()) + .add(new Field.Builder("mystring", "string").addAlias("fieldAlias").build()) + .add(new FieldSet.Builder("myFieldSet").addField("mystring").build()) + .build(); + var schemaInfo = new SchemaInfo(List.of(test), List.of()); + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + + assertSucceeds("?query=mystring:foo", execution); + assertSucceeds("?query=fieldAlias:foo", execution); + assertSucceeds("?query=myFieldSet:foo", execution); + assertSucceeds("?query=none:foo", execution); + assertFails("Cannot search for terms in 'mytensor1': It is a tensor field", + "?query=mytensor1:foo", execution); + assertFails("Cannot search for terms in 'mytensor2': It is a tensor field", + "?query=mytensor2:foo", execution); + } + private void assertSucceeds(String query, Execution execution) { + new QueryValidator().search(new Query(query), execution); + } + + private void assertFails(String expectedError, String query, Execution execution) { + try { + new QueryValidator().search(new Query(query), execution); + fail("Expected validation error from " + query); + } + catch (IllegalArgumentException e) { + // success + assertEquals(expectedError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java new file mode 100644 index 00000000000..b653e4d97aa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.search.Query; +import com.yahoo.search.schema.Cluster; +import com.yahoo.search.schema.Field; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.QueryValidator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author bratseth + */ +public class QueryValidatorPrefixTest { + + @Disabled + @Test + void testPrefixRequiresAttribute() { + var indexing = new Cluster.Builder("indexing").addSchema("test1").build(); + var streaming = new Cluster.Builder("streaming").addSchema("test1").addSchema("test2").setStreaming(true).build(); + var schemaInfo = new SchemaInfo(List.of(schema("test1"), schema("test2")), List.of(indexing, streaming)); + + assertIndexingValidation("", schemaInfo); + assertIndexingValidation("sources=indexing", schemaInfo); + assertIndexingValidation("sources=indexing,streaming", schemaInfo); + assertIndexingValidation("sources=indexing,streaming,ignored", schemaInfo); + assertStreamingValidation("sources=streaming", schemaInfo); + assertStreamingValidation("sources=streaming,ignored", schemaInfo); + assertIndexingValidation("sources=test1", schemaInfo); + assertIndexingValidation("sources=test1,streaming", schemaInfo); + assertStreamingValidation("sources=test2,streaming", schemaInfo); + assertIndexingValidation("sources=test1,test2", schemaInfo); + assertStreamingValidation("sources=test2", schemaInfo); + } + + private Schema schema(String name) { + return new Schema.Builder(name) + .add(new Field.Builder("attributeOnly", "string").setAttribute(true).build()) + .add(new Field.Builder("indexOnly", "string").setIndex(true).build()) + .add(new Field.Builder("attributeAndIndex", "string").setAttribute(true).setIndex(true).build()) + .build(); + } + + private void assertIndexingValidation(String sourcesParameter, SchemaInfo schemaInfo) { + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution); + assertFails("'indexOnly' is not an attribute field: Prefix matching is not supported", + "?query=indexOnly:foo*&" + sourcesParameter, execution); + assertFails("'attributeAndIndex' is an index field: Prefix matching is not supported even when it is also an attribute", + "?query=attributeAndIndex:foo*&" + sourcesParameter, execution); + } + + private void assertStreamingValidation(String sourcesParameter, SchemaInfo schemaInfo) { + Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo)); + assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution); + assertSucceeds("?query=indexOnly:foo*&" + sourcesParameter, execution); + assertSucceeds("?query=attributeAndIndex:foo*&" + sourcesParameter, execution); + } + + private void assertSucceeds(String query, Execution execution) { + new QueryValidator().search(new Query(query), execution); + } + + private void assertFails(String expectedError, String query, Execution execution) { + try { + new QueryValidator().search(new Query(query), execution); + fail("Expected validation error from " + query); + } + catch (IllegalArgumentException e) { + // success + assertEquals(expectedError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java deleted file mode 100644 index 64fb4354003..00000000000 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchers.test; - -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; -import com.yahoo.prelude.SearchDefinition; -import com.yahoo.search.Query; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchers.QueryValidator; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author bratseth - */ -public class QueryValidatorTestCase { - - @Test - void testValidation() { - SearchDefinition sd = new SearchDefinition("test"); - sd.addCommand("mytensor1", "type tensor(x[100]"); - sd.addCommand("mytensor2", "type tensor<float>(x[100]"); - sd.addCommand("mystring", "type string"); - IndexModel model = new IndexModel(sd); - - IndexFacts indexFacts = new IndexFacts(model); - Execution execution = new Execution(Execution.Context.createContextStub(indexFacts)); - new QueryValidator().search(new Query("?query=mystring:foo"), execution); - - try { - new QueryValidator().search(new Query("?query=mytensor1:foo"), execution); - fail("Expected validation error"); - } - catch (IllegalArgumentException e) { - // success - assertEquals("Cannot search 'mytensor1': It is a tensor field", e.getMessage()); - } - - try { - new QueryValidator().search(new Query("?query=mytensor2:foo"), execution); - fail("Expected validation error"); - } - catch (IllegalArgumentException e) { - // success - assertEquals("Cannot search 'mytensor2': It is a tensor field", e.getMessage()); - } - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 0bb3095fa9d..3a7641e7dc0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -74,7 +74,7 @@ public class YqlFieldAndSourceTestCase { .addField(FIELD2, "string").build()) .add((new DocumentSummary.Builder(SORTABLE_ATTRIBUTES_SUMMARY_CLASS).addField(FIELD2, "string").build())) .add((new DocumentSummary.Builder(THIRD_OPTION).addField(FIELD3, "string").build())); - return new SchemaInfo(List.of(schema.build()), Map.of()); + return new SchemaInfo(List.of(schema.build()), List.of()); } @AfterEach diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 10d3a5aeabe..2eb136056ac 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -76,9 +76,8 @@ public class SelectTestCase { "my.nested.title:madonna"); } - @Test - void testOr() throws Exception { + void testOr() { ObjectNode json_two_or = jsonMapper.createObjectNode(); ObjectNode json_three_or = jsonMapper.createObjectNode(); ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); @@ -100,7 +99,7 @@ public class SelectTestCase { } @Test - void testAnd() throws Exception { + void testAnd() { ObjectNode json_two_and = jsonMapper.createObjectNode(); ObjectNode json_three_and = jsonMapper.createObjectNode(); ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java index 578ccec7f40..19c03faae66 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -240,7 +240,7 @@ public class VdsStreamingSearcherTestCase { new SummaryParameters("default"), new ClusterParams("clusterName"), new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test")).build(), - new SchemaInfo(List.of(schema.build()), Map.of())); + new SchemaInfo(List.of(schema.build()), List.of())); // Magic query values are used to trigger specific behaviors from mock visitor. checkError(searcher, "/?query=noselection", |