aboutsummaryrefslogtreecommitdiffstats
path: root/container-search/src
diff options
context:
space:
mode:
authorArne H Juul <arnej27959@users.noreply.github.com>2023-06-13 08:27:55 +0200
committerGitHub <noreply@github.com>2023-06-13 08:27:55 +0200
commit982817bd81908a320338ae1c2b1945392ca69398 (patch)
tree34f6c7b5ce6dcd0ab91cd9e0290afab27b44dd12 /container-search/src
parentbb324300a8884035423dafaab1ebb7c72da2ae4c (diff)
parentd876e1b4855c55ab6a8544001009418efec9abac (diff)
Merge pull request #27382 from vespa-engine/bratseth/validate-prefix-matching-take-2-alternative-ending
Bratseth/validate prefix matching take 2 alternative ending
Diffstat (limited to 'container-search/src')
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/Index.java9
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java3
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java3
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/SelectParser.java37
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/Cluster.java79
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java5
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/Field.java177
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java27
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/FieldSet.java102
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/RankProfile.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/Schema.java45
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java93
-rw-r--r--container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java16
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java59
-rw-r--r--container-search/src/main/resources/configdefinitions/container.search.schema-info.def13
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java4
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java4
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java7
-rw-r--r--container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java14
-rw-r--r--container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java57
-rw-r--r--container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java83
-rw-r--r--container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java51
-rw-r--r--container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java2
-rw-r--r--container-search/src/test/java/com/yahoo/select/SelectTestCase.java5
-rw-r--r--container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java2
27 files changed, 753 insertions, 152 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java
index e245faec919..af8f63ab9f2 100644
--- a/container-search/src/main/java/com/yahoo/prelude/Index.java
+++ b/container-search/src/main/java/com/yahoo/prelude/Index.java
@@ -36,6 +36,7 @@ public class Index {
private boolean hostIndex = false;
private StemMode stemMode = StemMode.NONE;
private boolean isAttribute = false;
+ private boolean isIndex = false;
private boolean isDefaultPosition = false;
private boolean dynamicSummary=false;
private boolean highlightSummary=false;
@@ -157,6 +158,8 @@ public class Index {
setNGram(true, Integer.parseInt(command.substring(6)));
} else if (command.equals("attribute")) {
setAttribute(true);
+ } else if (command.equals("index")) {
+ setIndex(true);
} else if (command.equals("default-position")) {
setDefaultPosition(true);
} else if (command.equals("plain-tokens")) {
@@ -273,6 +276,12 @@ public class Index {
this.isAttribute = isAttribute;
}
+ public boolean isIndex() { return isIndex; }
+
+ public void setIndex(boolean isIndex) {
+ this.isIndex = isIndex;
+ }
+
public boolean hasPlainTokens() { return plainTokens; }
public void setPlainTokens(boolean plainTokens) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java
index 1d9e32ec374..a232841f29f 100644
--- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java
@@ -86,12 +86,13 @@ public class SearchDefinition {
return idx;
}
- public void addCommand(String indexName, String commandString) {
+ public Index addCommand(String indexName, String commandString) {
Index index = getOrCreateIndex(indexName);
index.addCommand(commandString);
if (index.isDefaultPosition()) {
defaultPosition = index.getName();
}
+ return index;
}
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java
index 7f37b77919b..e278ad38487 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java
@@ -27,8 +27,9 @@ public final class ToolBox {
/**
* Invoked when all sub-items have been visited, or immediately after
* visit() if there are no sub-items or visit() returned false.
+ * This default implementation does nothing.
*/
- public abstract void onExit();
+ public void onExit() {}
}
diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
index 0c65f4bf8dc..8a9ea281b0c 100644
--- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -168,7 +168,7 @@ public class SelectParser implements Parser {
Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get();
if (inspector.field("error_message").valid()) {
throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() +
- " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'");
+ " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'");
}
try {
@@ -186,32 +186,15 @@ public class SelectParser implements Parser {
inspector.traverse((ObjectTraverser) (key, value) -> {
String type = (FUNCTION_CALLS.contains(key)) ? CALL : key;
switch (type) {
- case AND:
- item[0] = buildAnd(key, value);
- break;
- case AND_NOT:
- item[0] = buildNotAnd(key, value);
- break;
- case OR:
- item[0] = buildOr(key, value);
- break;
- case EQ:
- item[0] = buildEquals(key, value);
- break;
- case RANGE:
- item[0] = buildRange(key, value);
- break;
- case CONTAINS:
- item[0] = buildTermSearch(key, value);
- break;
- case MATCHES:
- item[0] = buildRegExpSearch(key, value);
- break;
- case CALL:
- item[0] = buildFunctionCall(key, value);
- break;
- default:
- throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT);
+ case AND -> item[0] = buildAnd(key, value);
+ case AND_NOT -> item[0] = buildNotAnd(key, value);
+ case OR -> item[0] = buildOr(key, value);
+ case EQ -> item[0] = buildEquals(key, value);
+ case RANGE -> item[0] = buildRange(key, value);
+ case CONTAINS -> item[0] = buildTermSearch(key, value);
+ case MATCHES -> item[0] = buildRegExpSearch(key, value);
+ case CALL -> item[0] = buildFunctionCall(key, value);
+ default -> throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT);
}
});
return item[0];
diff --git a/container-search/src/main/java/com/yahoo/search/schema/Cluster.java b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java
new file mode 100644
index 00000000000..f5ea4fdffc7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/schema/Cluster.java
@@ -0,0 +1,79 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.schema;
+
+import com.yahoo.api.annotations.Beta;
+
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Information about the search aspects of a content cluster.
+ *
+ * @author bratseth
+ */
+@Beta
+public class Cluster {
+
+ private final String name;
+ private final boolean isStreaming;
+ private final Set<String> schemas;
+
+ private Cluster(Builder builder) {
+ this.name = builder.name;
+ this.isStreaming = builder.isStreaming;
+ this.schemas = Set.copyOf(builder.schemas);
+ }
+
+ public String name() { return name; }
+
+ /** Returns true if this cluster uses streaming search. */
+ public boolean isStreaming() { return isStreaming; }
+
+ /** Returns the names of the subset of all schemas that are present in this cluster. */
+ public Set<String> schemas() { return schemas; }
+
+ @Override
+ public boolean equals(Object o) {
+ if ( ! (o instanceof Cluster other)) return false;
+ if ( ! this.name.equals(other.name)) return false;
+ if ( this.isStreaming != other.isStreaming()) return false;
+ if ( ! this.schemas.equals(other.schemas)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, isStreaming, schemas);
+ }
+
+ @Override
+ public String toString() { return "cluster '" + name + "'"; }
+
+ public static class Builder {
+
+ private final String name;
+ private boolean isStreaming = false;
+ private final Set<String> schemas = new HashSet<>();
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ public Builder setStreaming(boolean isStreaming) {
+ this.isStreaming = isStreaming;
+ return this;
+ }
+
+ public Builder addSchema(String schema) {
+ schemas.add(schema);
+ return this;
+ }
+
+ public Cluster build() {
+ return new Cluster(this);
+ }
+
+ }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java
index 0aec6b0a4f6..a4f208710a0 100644
--- a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java
+++ b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java
@@ -1,11 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.schema;
-import java.util.ArrayList;
+import com.yahoo.api.annotations.Beta;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -15,6 +15,7 @@ import java.util.Objects;
*
* @author bratseth
*/
+@Beta
public class DocumentSummary {
private final String name;
diff --git a/container-search/src/main/java/com/yahoo/search/schema/Field.java b/container-search/src/main/java/com/yahoo/search/schema/Field.java
new file mode 100644
index 00000000000..ad949d5bad9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/schema/Field.java
@@ -0,0 +1,177 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.schema;
+
+import com.yahoo.api.annotations.Beta;
+import com.yahoo.tensor.TensorType;
+
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A field in a schema.
+ *
+ * @author bratseth
+ */
+@Beta
+public class Field implements FieldInfo {
+
+ private final String name;
+ private final Type type;
+ private final boolean isAttribute;
+ private final boolean isIndex;
+ private final Set<String> aliases;
+
+ public Field(Builder builder) {
+ this.name = builder.name;
+ this.type = builder.type;
+ this.isAttribute = builder.isAttribute;
+ this.isIndex = builder.isIndex;
+ this.aliases = Set.copyOf(builder.aliases);
+ }
+
+ @Override
+ public String name() { return name; }
+
+ @Override
+ public Type type() { return type; }
+
+ public Set<String> aliases() { return aliases; }
+
+ /** Returns whether this field is an attribute, i.e. does indexing: attribute. */
+ @Override
+ public boolean isAttribute() { return isAttribute; }
+
+ /** Returns whether this field is an index, i.e. does indexing: index. */
+ @Override
+ public boolean isIndex() { return isIndex; }
+
+ @Override
+ public boolean equals(Object o) {
+ if ( ! (o instanceof Field other)) return false;
+ if ( ! this.name.equals(other.name)) return false;
+ if ( this.isAttribute != other.isAttribute) return false;
+ if ( this.isIndex != other.isIndex) return false;
+ if ( ! this.aliases.equals(other.aliases)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type, isAttribute, isIndex, aliases);
+ }
+
+ @Override
+ public String toString() { return "field '" + name + "'"; }
+
+ public static class Type {
+
+ private final Kind kind;
+
+ /** The kind of type this is. */
+ public enum Kind {
+ ANNOTATIONREFERENCE, ARRAY, BOOL, BYTE, DOUBLE, FLOAT, INT, LONG, MAP, POSITION, PREDICATE, RAW, REFERENCE, STRING, STRUCT, TENSOR, URL, WEIGHTEDSET;
+ }
+
+ private Type(Kind kind) {
+ this.kind = kind;
+ }
+
+ /**
+ * Returns the kind of type this is.
+ * Structured types have additional information in the subclass specific to that kind of type.
+ */
+ public Kind kind() { return kind; }
+
+ /** Creates this from a type string on the syntax following "field [name] type " in a schema definition. */
+ public static Type from(String typeString) {
+ if (typeString.startsWith("annotationreference<"))
+ return new Type(Kind.ANNOTATIONREFERENCE); // TODO: Model as subclass
+ if (typeString.startsWith("array<"))
+ return new Type(Kind.ARRAY); // TODO: Model as subclass
+ if (typeString.equals("bool"))
+ return new Type(Kind.BOOL);
+ if (typeString.equals("byte"))
+ return new Type(Kind.BYTE);
+ if (typeString.equals("double"))
+ return new Type(Kind.DOUBLE);
+ if (typeString.equals("float"))
+ return new Type(Kind.FLOAT);
+ if (typeString.equals("int"))
+ return new Type(Kind.INT);
+ if (typeString.equals("long"))
+ return new Type(Kind.LONG);
+ if (typeString.startsWith("map<"))
+ return new Type(Kind.MAP); // TODO: Model as subclass
+ if (typeString.equals("position"))
+ return new Type(Kind.POSITION);
+ if (typeString.equals("predicate"))
+ return new Type(Kind.PREDICATE);
+ if (typeString.equals("raw"))
+ return new Type(Kind.RAW);
+ if (typeString.startsWith("reference<"))
+ return new Type(Kind.REFERENCE); // TODO: Model as subclass
+ if (typeString.equals("string"))
+ return new Type(Kind.STRING);
+ if (typeString.startsWith("tensor<") || typeString.startsWith("tensor("))
+ return new TensorFieldType(TensorType.fromSpec(typeString));
+ if (typeString.equals("url"))
+ return new Type(Kind.URL);
+ if (typeString.startsWith("weightedset<"))
+ return new Type(Kind.WEIGHTEDSET); // TODO: Model as subclass
+ else
+ return new Type(Kind.STRUCT); // TODO: Model as a subclass
+ }
+
+ }
+
+ public static class TensorFieldType extends Type {
+
+ private final TensorType tensorType;
+
+ public TensorFieldType(TensorType tensorType) {
+ super(Kind.TENSOR);
+ this.tensorType = tensorType;
+ }
+
+ public TensorType tensorType() { return tensorType; }
+
+ }
+
+ public static class Builder {
+
+ private final String name;
+ private final Type type;
+ private final Set<String> aliases = new HashSet<>();
+ private boolean isAttribute;
+ private boolean isIndex;
+
+ public Builder(String name, String typeString) {
+ this.name = name;
+ this.type = Type.from(typeString);
+ }
+
+ public Builder addAlias(String alias) {
+ aliases.add(alias);
+ return this;
+ }
+
+ public Builder setAttribute(boolean isAttribute) {
+ this.isAttribute = isAttribute;
+ return this;
+ }
+
+ public Builder setIndex(boolean isIndex) {
+ this.isIndex = isIndex;
+ return this;
+ }
+
+ public Field build() {
+ return new Field(this);
+ }
+
+ }
+
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java
new file mode 100644
index 00000000000..c3f6f22f1b9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/schema/FieldInfo.java
@@ -0,0 +1,27 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.schema;
+
+import com.yahoo.api.annotations.Beta;
+
+import java.util.Set;
+
+/**
+ * Information about a field or field set.
+ *
+ * @author bratseth
+ */
+@Beta
+public interface FieldInfo {
+
+ /** Returns the name of this field or field set. */
+ String name();
+
+ Field.Type type();
+
+ /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */
+ boolean isAttribute();
+
+ /** Returns whether this field is index(es), i.e. does indexing: index. */
+ boolean isIndex();
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java
new file mode 100644
index 00000000000..33c57dd1238
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/schema/FieldSet.java
@@ -0,0 +1,102 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.schema;
+
+import com.yahoo.api.annotations.Beta;
+
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A set of fields which can be queried as one.
+ *
+ * @author bratseth
+ */
+@Beta
+public class FieldSet implements FieldInfo {
+
+ private final String name;
+ private final Set<String> fieldNames;
+
+ // Assigned when this is added to a schema
+ private Schema schema = null;
+
+ private FieldSet(Builder builder) {
+ this.name = builder.name;
+ this.fieldNames = Set.copyOf(builder.fieldNames);
+ }
+
+ @Override
+ public String name() { return name; }
+
+ @Override
+ public Field.Type type() {
+ if (schema == null || fieldNames.isEmpty()) return null;
+ return randomFieldInThis().type();
+ }
+
+ /** Returns whether this field or field set is attribute(s), i.e. does indexing: attribute. */
+ @Override
+ public boolean isAttribute() {
+ if (schema == null || fieldNames.isEmpty()) return false;
+ return randomFieldInThis().isAttribute();
+ }
+
+ /** Returns whether this field is index(es), i.e. does indexing: index. */
+ @Override
+ public boolean isIndex() {
+ if (schema == null || fieldNames.isEmpty()) return false;
+ return randomFieldInThis().isIndex();
+ }
+
+ void setSchema(Schema schema) {
+ if ( this.schema != null)
+ throw new IllegalStateException("Cannot add field set '" + name + "' to schema '" + schema.name() +
+ "' as it is already added to schema '" + this.schema.name() + "'");
+ this.schema = schema;
+ }
+
+ /** Use a random field in this to determine its properties. Any inconsistency will have been warned about on deploy. */
+ private Field randomFieldInThis() {
+ return schema.fields().get(fieldNames.iterator().next());
+ }
+
+ public Set<String> fieldNames() { return fieldNames; }
+
+ @Override
+ public boolean equals(Object o) {
+ if ( ! (o instanceof FieldSet other)) return false;
+ if ( ! this.name.equals(other.name)) return false;
+ if ( ! this.fieldNames.equals(other.fieldNames)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, fieldNames);
+ }
+
+ @Override
+ public String toString() { return "field set '" + name + "'"; }
+
+ public static class Builder {
+
+ private final String name;
+ private final Set<String> fieldNames = new HashSet<>();
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ public Builder addField(String fieldName) {
+ fieldNames.add(fieldName);
+ return this;
+ }
+
+ public FieldSet build() {
+ return new FieldSet(this);
+ }
+
+ }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java
index 85bb3915975..5eba6c220bb 100644
--- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java
+++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.schema;
+import com.yahoo.api.annotations.Beta;
import com.yahoo.tensor.TensorType;
import java.util.Collections;
@@ -14,6 +15,7 @@ import java.util.Objects;
*
* @author bratseth
*/
+@Beta
public class RankProfile {
private final String name;
diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java
index c20aa1e81bd..20a776dc53e 100644
--- a/container-search/src/main/java/com/yahoo/search/schema/Schema.java
+++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java
@@ -8,6 +8,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
/**
* Information about a schema which is part of the application running this.
@@ -20,25 +21,57 @@ import java.util.Objects;
public class Schema {
private final String name;
+ private final Map<String, Field> fields;
+ private final Map<String, FieldSet> fieldSets;
private final Map<String, RankProfile> rankProfiles;
private final Map<String, DocumentSummary> documentSummaries;
+ /** Fields indexed by both name and aliases. */
+ private final Map<String, Field> fieldsByAliases;
+
private Schema(Builder builder) {
this.name = builder.name;
+ this.fields = Collections.unmodifiableMap(builder.fields);
+ this.fieldSets = Collections.unmodifiableMap(builder.fieldSets);
this.rankProfiles = Collections.unmodifiableMap(builder.rankProfiles);
this.documentSummaries = Collections.unmodifiableMap(builder.documentSummaries);
+
+ fieldSets.values().forEach(fieldSet -> fieldSet.setSchema(this));
rankProfiles.values().forEach(rankProfile -> rankProfile.setSchema(this));
+
+ fieldsByAliases = new HashMap<>();
+ for (Field field : fields.values()) {
+ fieldsByAliases.put(field.name(), field);
+ field.aliases().forEach(alias -> fieldsByAliases.put(alias, field));
+ }
}
public String name() { return name; }
+ public Map<String, Field> fields() { return fields; }
public Map<String, RankProfile> rankProfiles() { return rankProfiles; }
public Map<String, DocumentSummary> documentSummaries() { return documentSummaries; }
+ /**
+ * Looks up a field or field set by the given name or alias in this schema.
+ *
+ * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up
+ * @return information about the field or field set with the given name, or empty if no item with this name exists
+ */
+ public Optional<FieldInfo> fieldInfo(String fieldName) {
+ if (fieldName.isEmpty())
+ fieldName = "default";
+ Field field = fieldsByAliases.get(fieldName);
+ if (field != null) return Optional.of(field);
+ return Optional.ofNullable(fieldSets.get(fieldName));
+ }
+
@Override
public boolean equals(Object o) {
if (o == this) return true;
if ( ! (o instanceof Schema other)) return false;
if ( ! other.name.equals(this.name)) return false;
+ if ( ! other.fields.equals(this.fields)) return false;
+ if ( ! other.fieldSets.equals(this.fieldSets)) return false;
if ( ! other.rankProfiles.equals(this.rankProfiles)) return false;
if ( ! other.documentSummaries.equals(this.documentSummaries)) return false;
return true;
@@ -57,6 +90,8 @@ public class Schema {
public static class Builder {
private final String name;
+ private final Map<String, Field> fields = new LinkedHashMap<>();
+ private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>();
private final Map<String, RankProfile> rankProfiles = new LinkedHashMap<>();
private final Map<String, DocumentSummary> documentSummaries = new LinkedHashMap<>();
@@ -64,6 +99,16 @@ public class Schema {
this.name = Objects.requireNonNull(name);
}
+ public Builder add(Field field) {
+ fields.put(field.name(), field);
+ return this;
+ }
+
+ public Builder add(FieldSet fieldSet) {
+ fieldSets.put(fieldSet.name(), fieldSet);
+ return this;
+ }
+
public Builder add(RankProfile profile) {
rankProfiles.put(profile.name(), profile);
return this;
diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java
index d29964ea9c5..71bb00b39c5 100644
--- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java
+++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java
@@ -17,6 +17,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -35,37 +36,38 @@ import java.util.stream.Collectors;
*/
// NOTES:
// This should replace IndexFacts, and probably DocumentDatabase.
-// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information.
-// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation
-// given a set of possible schemas: The session mechanism is present here to make that efficient when added
-// (resolving schema subsets for every field lookup is too expensive).
+// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain complete field information.
@Beta
public class SchemaInfo {
- private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of());
+ private static final SchemaInfo empty = new SchemaInfo(List.of(), List.of());
private final Map<String, Schema> schemas;
- /** The schemas contained in each content cluster indexed by cluster name */
- private final Map<String, List<String>> clusters;
+ private final Map<String, Cluster> clusters;
@Inject
- public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future
- SchemaInfoConfig schemaInfoConfig,
+ public SchemaInfo(SchemaInfoConfig schemaInfoConfig,
QrSearchersConfig qrSearchersConfig) {
this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig));
}
- public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) {
+ public SchemaInfo(List<Schema> schemas, List<Cluster> clusters) {
Map<String, Schema> schemaMap = new LinkedHashMap<>();
schemas.forEach(schema -> schemaMap.put(schema.name(), schema));
this.schemas = Collections.unmodifiableMap(schemaMap);
- this.clusters = Collections.unmodifiableMap(clusters);
+
+ Map<String, Cluster> clusterMap = new LinkedHashMap<>();
+ clusters.forEach(cluster -> clusterMap.put(cluster.name(), cluster));
+ this.clusters = Collections.unmodifiableMap(clusterMap);
}
/** Returns all schemas configured in this application, indexed by schema name. */
public Map<String, Schema> schemas() { return schemas; }
+ /** Returns information about all clusters available for searching in this applications, indexed by cluyster name. */
+ public Map<String, Cluster> clusters() { return clusters; }
+
public Session newSession(Query query) {
return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas);
}
@@ -75,8 +77,7 @@ public class SchemaInfo {
@Override
public boolean equals(Object o) {
if (o == this) return true;
- if ( ! (o instanceof SchemaInfo)) return false;
- SchemaInfo other = (SchemaInfo)o;
+ if ( ! (o instanceof SchemaInfo other)) return false;
if ( ! other.schemas.equals(this.schemas)) return false;
if ( ! other.clusters.equals(this.clusters)) return false;
return true;
@@ -88,15 +89,61 @@ public class SchemaInfo {
/** The schema information resolved to be relevant to this session. */
public static class Session {
+ private final boolean isStreaming;
private final Collection<Schema> schemas;
private Session(Set<String> sources,
Set<String> restrict,
- Map<String, List<String>> clusters,
+ Map<String, Cluster> clusters,
Map<String, Schema> candidates) {
+ this.isStreaming = resolveStreaming(sources, clusters);
this.schemas = resolveSchemas(sources, restrict, clusters, candidates.values());
}
+ /** Returns true if this only searches streaming clusters. */
+ public boolean isStreaming() { return isStreaming; }
+
+ /**
+ * Looks up a field or field set by the given name or alias
+ * in the schemas resolved for this query.
+ *
+ * If there are several fields or field sets by this name or alias across the schemas of this session,
+ * one is chosen by random.
+ *
+ * @param fieldName the name or alias of the field or field set. If this is empty, the name "default" is looked up.
+ * @return the appropriate field or empty if no field or field set has this name or alias
+ */
+ public Optional<FieldInfo> fieldInfo(String fieldName) {
+ for (var schema : schemas) {
+ Optional<FieldInfo> field = schema.fieldInfo(fieldName);
+ if (field.isPresent())
+ return field;
+ }
+ return Optional.empty();
+ }
+
+ private static boolean resolveStreaming(Set<String> sources, Map<String, Cluster> clusters) {
+ if (sources.isEmpty()) return clusters.values().stream().allMatch(Cluster::isStreaming);
+
+ var matchedClusters = sources.stream().map(source -> clusterOfSource(source, clusters)).filter(Objects::nonNull).toList();
+ if (matchedClusters.isEmpty()) return false;
+ return matchedClusters.stream().allMatch(Cluster::isStreaming);
+ }
+
+ /**
+ * A source name is either a cluster or a schema.
+ * Returns the cluster which either is or contains this name, if any.
+ */
+ private static Cluster clusterOfSource(String source, Map<String, Cluster> clusters) {
+ var cluster = clusters.get(source);
+ if (cluster != null) return cluster;
+ for (var c : clusters.values()) {
+ if (c.schemas().contains(source))
+ return c;
+ }
+ return null;
+ }
+
/**
* Given a search list which is a mixture of schemas and cluster
* names, and a restrict list which is a list of schemas, return a
@@ -106,7 +153,7 @@ public class SchemaInfo {
*/
private static Collection<Schema> resolveSchemas(Set<String> sources,
Set<String> restrict,
- Map<String, List<String>> clusters,
+ Map<String, Cluster> clusters,
Collection<Schema> candidates) {
if (sources.isEmpty())
return restrict.isEmpty() ? candidates : keep(restrict, candidates);
@@ -114,7 +161,7 @@ public class SchemaInfo {
Set<String> schemaNames = new HashSet<>();
for (String source : sources) {
if (clusters.containsKey(source)) // source is a cluster
- schemaNames.addAll(clusters.get(source));
+ schemaNames.addAll(clusters.get(source).schemas());
else // source is a schema
schemaNames.add(source);
}
@@ -126,13 +173,6 @@ public class SchemaInfo {
return schemas.stream().filter(schema -> names.contains(schema.name())).toList();
}
- private List<RankProfile> profilesNamed(String name) {
- return schemas.stream()
- .filter(schema -> schema.rankProfiles().containsKey(name))
- .map(schema -> schema.rankProfiles().get(name))
- .toList();
- }
-
/**
* Returns the type of the given rank feature name in the given profile,
* if it can be uniquely determined.
@@ -165,6 +205,13 @@ public class SchemaInfo {
return foundType;
}
+ private List<RankProfile> profilesNamed(String name) {
+ return schemas.stream()
+ .filter(schema -> schema.rankProfiles().containsKey(name))
+ .map(schema -> schema.rankProfiles().get(name))
+ .toList();
+ }
+
}
}
diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java
index 6947a93a833..1b9ba397105 100644
--- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java
+++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java
@@ -6,10 +6,7 @@ import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.tensor.TensorType;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
/**
* Translation between schema info configuration and schema objects.
@@ -47,14 +44,15 @@ class SchemaInfoConfigurer {
return builder.build();
}
- static Map<String, List<String>> toClusters(QrSearchersConfig config) {
- Map<String, List<String>> clusters = new HashMap<>();
+ static List<Cluster> toClusters(QrSearchersConfig config) {
+ List<Cluster> clusters = new ArrayList<>();
for (int i = 0; i < config.searchcluster().size(); ++i) {
- List<String> schemas = new ArrayList<>();
String clusterName = config.searchcluster(i).name();
- for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j)
- schemas.add(config.searchcluster(i).searchdef(j));
- clusters.put(clusterName, schemas);
+ var clusterInfo = new Cluster.Builder(clusterName);
+ clusterInfo.setStreaming(config.searchcluster(i).indexingmode() == QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING);
+ for (var schemaDef : config.searchcluster(i).searchdef())
+ clusterInfo.addSchema(schemaDef);
+ clusters.add(clusterInfo.build());
}
return clusters;
}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
index bfc4219eabc..87880ce2445 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
@@ -63,7 +63,7 @@ public class ExecutionFactory extends AbstractComponent {
Executor executor) {
this(chainsConfig,
indexInfo,
- new SchemaInfo(indexInfo, schemaInfo, clusters),
+ new SchemaInfo(schemaInfo, clusters),
clusters,
searchers,
specialTokens,
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
index a2e3d038053..0b435c2e32d 100644
--- a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java
@@ -3,15 +3,16 @@ package com.yahoo.search.searchers;
import com.yahoo.component.chain.dependencies.After;
import com.yahoo.component.chain.dependencies.Before;
-import com.yahoo.prelude.Index;
-import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.query.CompositeItem;
import com.yahoo.prelude.query.HasIndexItem;
import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PrefixItem;
import com.yahoo.prelude.query.ToolBox;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.Searcher;
+import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.FieldInfo;
+import com.yahoo.search.schema.SchemaInfo;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.search.searchchain.PhaseNames;
@@ -28,31 +29,61 @@ public class QueryValidator extends Searcher {
@Override
public Result search(Query query, Execution execution) {
- IndexFacts.Session session = execution.context().getIndexFacts().newSession(query);
- ToolBox.visit(new ItemValidator(session), query.getModel().getQueryTree().getRoot());
+ var session = execution.context().schemaInfo().newSession(query);
+ ToolBox.visit(new TermSearchValidator(session), query.getModel().getQueryTree().getRoot());
+ // ToolBox.visit(new PrefixSearchValidator(session), query.getModel().getQueryTree().getRoot()); TODO: Enable check and QueryValidatorPrefixTest
return execution.search(query);
}
- private static class ItemValidator extends ToolBox.QueryVisitor {
+ private abstract static class TermValidator extends ToolBox.QueryVisitor {
- IndexFacts.Session session;
+ final SchemaInfo.Session schema;
- public ItemValidator(IndexFacts.Session session) {
- this.session = session;
+ public TermValidator(SchemaInfo.Session schema) {
+ this.schema = schema;
+ }
+
+ }
+
+ private static class TermSearchValidator extends TermValidator {
+
+ public TermSearchValidator(SchemaInfo.Session schema) {
+ super(schema);
}
@Override
public boolean visit(Item item) {
- if (item instanceof HasIndexItem) {
- String indexName = ((HasIndexItem)item).getIndexName();
- if (session.getIndex(indexName).isTensor())
- throw new IllegalArgumentException("Cannot search '" + indexName + "': It is a tensor field");
+ if (item instanceof HasIndexItem indexItem) {
+ var field = schema.fieldInfo(indexItem.getIndexName());
+ if (! field.isPresent()) return true;
+ if (field.get().type().kind() == Field.Type.Kind.TENSOR)
+ throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() +
+ "': It is a tensor field");
}
return true;
}
+ }
+
+ private static class PrefixSearchValidator extends TermValidator {
+
+ public PrefixSearchValidator(SchemaInfo.Session schema) {
+ super(schema);
+ }
+
@Override
- public void onExit() { }
+ public boolean visit(Item item) {
+ if (schema.isStreaming()) return true; // prefix is always supported
+ if (item instanceof PrefixItem prefixItem) {
+ var field = schema.fieldInfo(prefixItem.getIndexName());
+ if (! field.isPresent()) return true;
+ if ( ! field.get().isAttribute())
+ throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is not an attribute field: Prefix matching is not supported");
+ if (field.get().isIndex()) // index overrides attribute
+ throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is an index field: Prefix matching is not supported even when it is also an attribute");
+ }
+ return true;
+ }
}
diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def
index 2da6d621973..9018d41d5d9 100644
--- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def
+++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def
@@ -4,7 +4,18 @@ namespace=search.config
## The name of this schema
schema[].name string
-## The name of the summary class
+## Information about a schema field (currently incomplete)
+schema[].field[].name string
+schema[].field[].type string
+schema[].field[].alias[] string
+schema[].field[].attribute bool
+schema[].field[].index bool
+
+## Field sets
+schema[].fieldset[].name string
+schema[].fieldset[].field[] string
+
+## Information about a summary class
schema[].summaryclass[].name string
## The name of a field in the summary class
schema[].summaryclass[].fields[].name string
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index 06ae9923dae..c164fd3eb1c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -277,7 +277,7 @@ public class ClusterSearcherTestCase {
schemaBuilder.add(new RankProfile.Builder("testprofile").build());
schemas.add(schemaBuilder.build());
}
- return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, Map.of())));
+ return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, List.of())));
} finally {
cluster.deconstruct();
}
@@ -462,7 +462,7 @@ public class ClusterSearcherTestCase {
qrSearchersConfig.build(),
clusterConfig.build(),
documentDbConfig.build(),
- new SchemaInfo(List.of(schema.build()), Map.of()),
+ new SchemaInfo(List.of(schema.build()), List.of()),
dispatchers,
null,
vipStatus,
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index 73975ecaa96..7a63eb07641 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -129,7 +129,7 @@ public class FastSearcherTestCase {
new SummaryParameters(null),
new ClusterParams("testhittype"),
documentDb,
- new SchemaInfo(List.of(schema.build()), Map.of()));
+ new SchemaInfo(List.of(schema.build()), List.of()));
Query q = new Query("?query=foo");
Result result = doSearch(backend, q, 0, 10);
assertFalse(backend.summaryNeedsQuery(q));
@@ -210,7 +210,7 @@ public class FastSearcherTestCase {
private SchemaInfo schemaInfo(String schemaName) {
var schema = new Schema.Builder(schemaName);
schema.add(new RankProfile.Builder("default").build());
- return new SchemaInfo(List.of(schema.build()), Map.of());
+ return new SchemaInfo(List.of(schema.build()), List.of());
}
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
index d70b42aa36b..07394676e09 100644
--- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
@@ -261,14 +261,14 @@ public class QueryTestCase {
@Test
void testNoneHitsNegativeOffsetValue() {
assertQueryError(
- "?query=test&hits=(none)&offset=-10",
+ "?query=test&hits=(none)",
"Could not set 'hits' to '(none)': '(none)' is not a valid integer");
}
@Test
void testFeedbackIsTransferredToResult() {
assertQueryError(
- "?query=test&hits=(none)&offset=-10",
+ "?query=test&hits=(none)",
"Could not set 'hits' to '(none)': '(none)' is not a valid integer");
}
diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
index cbe4ddcbc63..03b53970550 100644
--- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
+++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
@@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.language.Language;
import com.yahoo.language.process.Embedder;
import com.yahoo.search.Query;
+import com.yahoo.search.schema.Cluster;
import com.yahoo.search.schema.RankProfile;
import com.yahoo.search.schema.Schema;
import com.yahoo.search.schema.SchemaInfo;
@@ -259,9 +260,9 @@ public class RankProfileInputTest {
.addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})"))
.build())
.build());
- Map<String, List<String>> clusters = new HashMap<>();
- clusters.put("ab", List.of("a", "b"));
- clusters.put("a", List.of("a"));
+ List<Cluster> clusters = new ArrayList<>();
+ clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build());
+ clusters.add(new Cluster.Builder("a").addSchema("a").build());
return new SchemaInfo(schemas, clusters);
}
diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
index a46f3480d50..4aced1b5e25 100644
--- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
+++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
@@ -83,17 +83,14 @@ public class SchemaInfoTester {
.addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})"))
.build())
.build());
- Map<String, List<String>> clusters = new HashMap<>();
- clusters.put("ab", List.of("a", "b"));
- clusters.put("a", List.of("a"));
+ List<Cluster> clusters = new ArrayList<>();
+ clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build());
+ clusters.add(new Cluster.Builder("a").addSchema("a").setStreaming(true).build());
return new SchemaInfo(schemas, clusters);
}
/** Creates the same schema info as createSchemaInfo from config objects. */
static SchemaInfo createSchemaInfoFromConfig() {
-
- var indexInfoConfig = new IndexInfoConfig.Builder();
-
var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder();
rankProfileCommon.name("commonProfile");
rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})"));
@@ -141,7 +138,7 @@ public class SchemaInfoTester {
schemaInfoInfoConfig.schema(schemaB);
- // ----- Info about which schemas are in which clusters
+ // ----- Info about clusters
var qrSearchersConfig = new QrSearchersConfig.Builder();
var clusterAB = new QrSearchersConfig.Searchcluster.Builder();
clusterAB.name("ab");
@@ -149,10 +146,11 @@ public class SchemaInfoTester {
qrSearchersConfig.searchcluster(clusterAB);
var clusterA = new QrSearchersConfig.Searchcluster.Builder();
clusterA.name("a");
+ clusterA.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING);
clusterA.searchdef("a");
qrSearchersConfig.searchcluster(clusterA);
- return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build());
+ return new SchemaInfo(schemaInfoInfoConfig.build(), qrSearchersConfig.build());
}
}
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java
new file mode 100644
index 00000000000..9367d9f335a
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java
@@ -0,0 +1,57 @@
+package com.yahoo.search.searchers.test;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.FieldSet;
+import com.yahoo.search.schema.Schema;
+import com.yahoo.search.schema.SchemaInfo;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchers.QueryValidator;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author bratseth
+ */
+public class QueryValidatorFieldTypeTest {
+
+ @Test
+ void testTensorsCannotBeSearchedForTerms() {
+ var test = new Schema.Builder("test")
+ .add(new Field.Builder("mytensor1", "tensor(x[100])").build())
+ .add(new Field.Builder("mytensor2", "tensor<float>(x[100])").build())
+ .add(new Field.Builder("mystring", "string").addAlias("fieldAlias").build())
+ .add(new FieldSet.Builder("myFieldSet").addField("mystring").build())
+ .build();
+ var schemaInfo = new SchemaInfo(List.of(test), List.of());
+ Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+
+ assertSucceeds("?query=mystring:foo", execution);
+ assertSucceeds("?query=fieldAlias:foo", execution);
+ assertSucceeds("?query=myFieldSet:foo", execution);
+ assertSucceeds("?query=none:foo", execution);
+ assertFails("Cannot search for terms in 'mytensor1': It is a tensor field",
+ "?query=mytensor1:foo", execution);
+ assertFails("Cannot search for terms in 'mytensor2': It is a tensor field",
+ "?query=mytensor2:foo", execution);
+ }
+ private void assertSucceeds(String query, Execution execution) {
+ new QueryValidator().search(new Query(query), execution);
+ }
+
+ private void assertFails(String expectedError, String query, Execution execution) {
+ try {
+ new QueryValidator().search(new Query(query), execution);
+ fail("Expected validation error from " + query);
+ }
+ catch (IllegalArgumentException e) {
+ // success
+ assertEquals(expectedError, e.getMessage());
+ }
+ }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java
new file mode 100644
index 00000000000..b653e4d97aa
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java
@@ -0,0 +1,83 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers.test;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.schema.Cluster;
+import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.Schema;
+import com.yahoo.search.schema.SchemaInfo;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchers.QueryValidator;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Disabled;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author bratseth
+ */
+public class QueryValidatorPrefixTest {
+
+ @Disabled
+ @Test
+ void testPrefixRequiresAttribute() {
+ var indexing = new Cluster.Builder("indexing").addSchema("test1").build();
+ var streaming = new Cluster.Builder("streaming").addSchema("test1").addSchema("test2").setStreaming(true).build();
+ var schemaInfo = new SchemaInfo(List.of(schema("test1"), schema("test2")), List.of(indexing, streaming));
+
+ assertIndexingValidation("", schemaInfo);
+ assertIndexingValidation("sources=indexing", schemaInfo);
+ assertIndexingValidation("sources=indexing,streaming", schemaInfo);
+ assertIndexingValidation("sources=indexing,streaming,ignored", schemaInfo);
+ assertStreamingValidation("sources=streaming", schemaInfo);
+ assertStreamingValidation("sources=streaming,ignored", schemaInfo);
+ assertIndexingValidation("sources=test1", schemaInfo);
+ assertIndexingValidation("sources=test1,streaming", schemaInfo);
+ assertStreamingValidation("sources=test2,streaming", schemaInfo);
+ assertIndexingValidation("sources=test1,test2", schemaInfo);
+ assertStreamingValidation("sources=test2", schemaInfo);
+ }
+
+ private Schema schema(String name) {
+ return new Schema.Builder(name)
+ .add(new Field.Builder("attributeOnly", "string").setAttribute(true).build())
+ .add(new Field.Builder("indexOnly", "string").setIndex(true).build())
+ .add(new Field.Builder("attributeAndIndex", "string").setAttribute(true).setIndex(true).build())
+ .build();
+ }
+
+ private void assertIndexingValidation(String sourcesParameter, SchemaInfo schemaInfo) {
+ Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+ assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution);
+ assertFails("'indexOnly' is not an attribute field: Prefix matching is not supported",
+ "?query=indexOnly:foo*&" + sourcesParameter, execution);
+ assertFails("'attributeAndIndex' is an index field: Prefix matching is not supported even when it is also an attribute",
+ "?query=attributeAndIndex:foo*&" + sourcesParameter, execution);
+ }
+
+ private void assertStreamingValidation(String sourcesParameter, SchemaInfo schemaInfo) {
+ Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+ assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution);
+ assertSucceeds("?query=indexOnly:foo*&" + sourcesParameter, execution);
+ assertSucceeds("?query=attributeAndIndex:foo*&" + sourcesParameter, execution);
+ }
+
+ private void assertSucceeds(String query, Execution execution) {
+ new QueryValidator().search(new Query(query), execution);
+ }
+
+ private void assertFails(String expectedError, String query, Execution execution) {
+ try {
+ new QueryValidator().search(new Query(query), execution);
+ fail("Expected validation error from " + query);
+ }
+ catch (IllegalArgumentException e) {
+ // success
+ assertEquals(expectedError, e.getMessage());
+ }
+ }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java
deleted file mode 100644
index 64fb4354003..00000000000
--- a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.searchers.test;
-
-import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.IndexModel;
-import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.search.Query;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.search.searchers.QueryValidator;
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.fail;
-
-/**
- * @author bratseth
- */
-public class QueryValidatorTestCase {
-
- @Test
- void testValidation() {
- SearchDefinition sd = new SearchDefinition("test");
- sd.addCommand("mytensor1", "type tensor(x[100]");
- sd.addCommand("mytensor2", "type tensor<float>(x[100]");
- sd.addCommand("mystring", "type string");
- IndexModel model = new IndexModel(sd);
-
- IndexFacts indexFacts = new IndexFacts(model);
- Execution execution = new Execution(Execution.Context.createContextStub(indexFacts));
- new QueryValidator().search(new Query("?query=mystring:foo"), execution);
-
- try {
- new QueryValidator().search(new Query("?query=mytensor1:foo"), execution);
- fail("Expected validation error");
- }
- catch (IllegalArgumentException e) {
- // success
- assertEquals("Cannot search 'mytensor1': It is a tensor field", e.getMessage());
- }
-
- try {
- new QueryValidator().search(new Query("?query=mytensor2:foo"), execution);
- fail("Expected validation error");
- }
- catch (IllegalArgumentException e) {
- // success
- assertEquals("Cannot search 'mytensor2': It is a tensor field", e.getMessage());
- }
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
index 0bb3095fa9d..3a7641e7dc0 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
@@ -74,7 +74,7 @@ public class YqlFieldAndSourceTestCase {
.addField(FIELD2, "string").build())
.add((new DocumentSummary.Builder(SORTABLE_ATTRIBUTES_SUMMARY_CLASS).addField(FIELD2, "string").build()))
.add((new DocumentSummary.Builder(THIRD_OPTION).addField(FIELD3, "string").build()));
- return new SchemaInfo(List.of(schema.build()), Map.of());
+ return new SchemaInfo(List.of(schema.build()), List.of());
}
@AfterEach
diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
index 10d3a5aeabe..2eb136056ac 100644
--- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
+++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
@@ -76,9 +76,8 @@ public class SelectTestCase {
"my.nested.title:madonna");
}
-
@Test
- void testOr() throws Exception {
+ void testOr() {
ObjectNode json_two_or = jsonMapper.createObjectNode();
ObjectNode json_three_or = jsonMapper.createObjectNode();
ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna");
@@ -100,7 +99,7 @@ public class SelectTestCase {
}
@Test
- void testAnd() throws Exception {
+ void testAnd() {
ObjectNode json_two_and = jsonMapper.createObjectNode();
ObjectNode json_three_and = jsonMapper.createObjectNode();
ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna");
diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
index 578ccec7f40..19c03faae66 100644
--- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
@@ -240,7 +240,7 @@ public class VdsStreamingSearcherTestCase {
new SummaryParameters("default"),
new ClusterParams("clusterName"),
new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test")).build(),
- new SchemaInfo(List.of(schema.build()), Map.of()));
+ new SchemaInfo(List.of(schema.build()), List.of()));
// Magic query values are used to trigger specific behaviors from mock visitor.
checkError(searcher, "/?query=noselection",