From 66c9edce7bbe11cb99c636e433feb2085f8d2e8a Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 27 Apr 2022 13:45:41 +0200 Subject: Use an already exported package --- .../yahoo/prelude/fastsearch/DocumentDatabase.java | 2 +- .../prelude/fastsearch/VespaBackEndSearcher.java | 2 +- .../src/main/java/com/yahoo/search/Query.java | 3 +- .../java/com/yahoo/search/config/RankProfile.java | 94 ------------- .../main/java/com/yahoo/search/config/Schema.java | 71 ---------- .../java/com/yahoo/search/config/SchemaInfo.java | 147 -------------------- .../yahoo/search/config/SchemaInfoConfigurer.java | 50 ------- .../search/config/internal/TensorConverter.java | 95 ------------- .../java/com/yahoo/search/config/package-info.java | 11 -- .../main/java/com/yahoo/search/query/Model.java | 2 +- .../query/profile/types/TensorFieldType.java | 2 +- .../properties/RankProfileInputProperties.java | 5 +- .../java/com/yahoo/search/schema/RankProfile.java | 94 +++++++++++++ .../main/java/com/yahoo/search/schema/Schema.java | 71 ++++++++++ .../java/com/yahoo/search/schema/SchemaInfo.java | 148 +++++++++++++++++++++ .../yahoo/search/schema/SchemaInfoConfigurer.java | 50 +++++++ .../search/schema/internal/TensorConverter.java | 95 +++++++++++++ .../java/com/yahoo/search/schema/package-info.java | 11 ++ .../com/yahoo/search/searchchain/Execution.java | 3 +- .../yahoo/search/searchchain/ExecutionFactory.java | 4 +- .../container.search.schema-info.def | 2 +- .../com/yahoo/search/config/SchemaInfoTest.java | 50 ------- .../com/yahoo/search/config/SchemaInfoTester.java | 132 ------------------ .../yahoo/search/query/RankProfileInputTest.java | 6 +- .../com/yahoo/search/schema/SchemaInfoTest.java | 49 +++++++ .../com/yahoo/search/schema/SchemaInfoTester.java | 136 +++++++++++++++++++ 26 files changed, 668 insertions(+), 667 deletions(-) delete mode 100644 container-search/src/main/java/com/yahoo/search/config/RankProfile.java delete mode 100644 container-search/src/main/java/com/yahoo/search/config/Schema.java delete mode 100644 container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java delete mode 100644 container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java delete mode 100644 container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java delete mode 100644 container-search/src/main/java/com/yahoo/search/config/package-info.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/RankProfile.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/Schema.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java create mode 100644 container-search/src/main/java/com/yahoo/search/schema/package-info.java delete mode 100644 container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java delete mode 100644 container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java create mode 100644 container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java (limited to 'container-search/src') diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index f35559ad2f4..67038e0e771 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.search.config.RankProfile; +import com.yahoo.search.schema.RankProfile; import com.yahoo.tensor.TensorType; import java.util.ArrayList; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index a6da823d990..3847e80d3c7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -14,7 +14,7 @@ import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.config.RankProfile; +import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 354881c763b..5e46815a152 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -13,7 +13,7 @@ import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import com.yahoo.search.federation.FederationSearcher; @@ -60,7 +60,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; diff --git a/container-search/src/main/java/com/yahoo/search/config/RankProfile.java b/container-search/src/main/java/com/yahoo/search/config/RankProfile.java deleted file mode 100644 index 944a23f2964..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/RankProfile.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.tensor.TensorType; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * Information about a rank profile - * - * @author bratseth - */ -public class RankProfile { - - private final String name; - private final boolean hasSummaryFeatures; - private final boolean hasRankFeatures; - private final Map inputs; - - private RankProfile(Builder builder) { - this.name = builder.name; - this.hasSummaryFeatures = builder.hasSummaryFeatures; - this.hasRankFeatures = builder.hasRankFeatures; - this.inputs = Map.copyOf(builder.inputs); - } - - public String name() { return name; } - - /** Returns true if this rank profile has summary features. */ - public boolean hasSummaryFeatures() { return hasSummaryFeatures; } - - /** Returns true if this rank profile has rank features. */ - public boolean hasRankFeatures() { return hasRankFeatures; } - - /** Returns the inputs explicitly declared in this rank profile. */ - public Map inputs() { return inputs; } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof RankProfile)) return false; - RankProfile other = (RankProfile)o; - if ( ! other.name.equals(this.name)) return false; - if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; - if ( other.hasRankFeatures != this.hasRankFeatures) return false; - if ( ! other.inputs.equals(this.inputs)) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); - } - - @Override - public String toString() { - return "rank profile '" + name + "'"; - } - - public static class Builder { - - private final String name; - private boolean hasSummaryFeatures = true; - private boolean hasRankFeatures = true; - private final Map inputs = new HashMap<>(); - - public Builder(String name) { - this.name = Objects.requireNonNull(name); - } - - public Builder setHasSummaryFeatures(boolean hasSummaryFeatures) { - this.hasSummaryFeatures = hasSummaryFeatures; - return this; - } - - public Builder setHasRankFeatures(boolean hasRankFeatures) { - this.hasRankFeatures = hasRankFeatures; - return this; - } - - public Builder addInput(String name, TensorType type) { - inputs.put(name, type); - return this; - } - - public RankProfile build() { - return new RankProfile(this); - } - - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/config/Schema.java b/container-search/src/main/java/com/yahoo/search/config/Schema.java deleted file mode 100644 index 57712c731f4..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/Schema.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.api.annotations.Beta; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * Information about a schema which is part of the application running this. - * - * This is immutable. - * - * @author bratseth - */ -@Beta -public class Schema { - - private final String name; - private final Map rankProfiles; - - private Schema(Builder builder) { - this.name = builder.name; - this.rankProfiles = Map.copyOf(builder.rankProfiles); - } - - public String name() { return name; } - public Map rankProfiles() { return rankProfiles; } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof Schema)) return false; - Schema other = (Schema)o; - if ( ! other.name.equals(this.name)) return false; - if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(name, rankProfiles); - } - - @Override - public String toString() { - return "schema '" + name + "'"; - } - - public static class Builder { - - private final String name; - private final Map rankProfiles = new HashMap<>(); - - public Builder(String name) { - this.name = Objects.requireNonNull(name); - } - - public Builder add(RankProfile profile) { - rankProfiles.put(profile.name(), Objects.requireNonNull(profile)); - return this; - } - - public Schema build() { - return new Schema(this); - } - - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java deleted file mode 100644 index 4a8ec83c847..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.api.annotations.Beta; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.search.SchemaInfoConfig; -import com.yahoo.search.Query; -import com.yahoo.tensor.TensorType; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Information about all the schemas configured in the application this container is a part of. - * - * Usage: - * - * SchemaInfo.Session session = schemaInfo.newSession(query); // once when starting to process a query - * session.get(...) // access information about the schema(s) relevant to the query - * - * - * This is immutable. - * - * @author bratseth - */ -// NOTES: -// This should replace IndexFacts, and probably DocumentDatabase. -// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information. -// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation -// given a set of possible schemas: The session mechanism is present here to make that efficient when added -// (resolving schema subsets for every field lookup is too expensive). -@Beta -public class SchemaInfo { - - private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); - - private final List schemas; - - /** The schemas contained in each content cluster indexed by cluster name */ - private final Map> clusters; - - public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future - SchemaInfoConfig schemaInfoConfig, - QrSearchersConfig qrSearchersConfig) { - this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); - } - - public SchemaInfo(List schemas, Map> clusters) { - this.schemas = List.copyOf(schemas); - this.clusters = Map.copyOf(clusters); - } - - public Session newSession(Query query) { - return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); - } - - public static SchemaInfo empty() { return empty; } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof SchemaInfo)) return false; - SchemaInfo other = (SchemaInfo)o; - if ( ! other.schemas.equals(this.schemas)) return false; - if ( ! other.clusters.equals(this.clusters)) return false; - return true; - } - - @Override - public int hashCode() { return Objects.hash(schemas, clusters); } - - /** The schema information resolved to be relevant to this session. */ - public static class Session { - - private final List schemas; - - private Session(Set sources, - Set restrict, - Map> clusters, - List candidates) { - this.schemas = resolveSchemas(sources, restrict, clusters, candidates); - } - - /** - * Given a search list which is a mixture of schemas and cluster - * names, and a restrict list which is a list of schemas, return a - * set of all valid schemas for this combination. - * - * @return the possibly empty list of schemas matching the arguments - */ - private static List resolveSchemas(Set sources, - Set restrict, - Map> clusters, - List candidates) { - if (sources.isEmpty()) - return restrict.isEmpty() ? candidates : keep(restrict, candidates); - - Set schemaNames = new HashSet<>(); - for (String source : sources) { - if (clusters.containsKey(source)) // source is a cluster - schemaNames.addAll(clusters.get(source)); - else // source is a schema - schemaNames.add(source); - } - candidates = keep(schemaNames, candidates); - return restrict.isEmpty() ? candidates : keep(restrict, candidates); - } - - private static List keep(Set names, List schemas) { - return schemas.stream().filter(schema -> names.contains(schema.name())).collect(Collectors.toList()); - } - - /** - * Returns the type of the given rank feature name in the given profile, - * if it can be uniquely determined. - * - * @param rankFeature the rank feature name, a string on the form "query(name)" - * @param rankProfile the name of the rank profile in which to locate the input declaration - * @return the type of the declared input, or null if it is not declared or the rank profile is not found - * @throws IllegalArgumentException if the feature is declared in this rank profile in multiple schemas - * of this session with conflicting types - */ - public TensorType rankProfileInput(String rankFeature, String rankProfile) { - TensorType foundType = null; - Schema declaringSchema = null; - for (Schema schema : schemas) { - RankProfile profile = schema.rankProfiles().get(rankProfile); - if (profile == null) continue; - TensorType newlyFoundType = profile.inputs().get(rankFeature); - if (newlyFoundType == null) continue; - if (foundType != null && ! newlyFoundType.equals(foundType)) - throw new IllegalArgumentException("Conflicting input type declarations for '" + rankFeature + "': " + - "Declared as " + foundType + " in " + profile + " in " + declaringSchema + - ", and as " + newlyFoundType + " in " + profile + " in " + schema); - foundType = newlyFoundType; - declaringSchema = schema; - } - return foundType; - } - - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java deleted file mode 100644 index 49acf589ba3..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.search.SchemaInfoConfig; -import com.yahoo.tensor.TensorType; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Translation between schema info configuration and schema objects. - * - * @author bratseth - */ -class SchemaInfoConfigurer { - - static List toSchemas(SchemaInfoConfig documentdbInfoConfig) { - return documentdbInfoConfig.schema().stream().map(config -> toSchema(config)).collect(Collectors.toList()); - } - - static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { - Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); - for (var profileConfig : schemaInfoConfig.rankprofile()) { - RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); - profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); - profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); - for (var inputConfig : profileConfig.input()) - profileBuilder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); - builder.add(profileBuilder.build()); - } - return builder.build(); - } - - static Map> toClusters(QrSearchersConfig config) { - Map> clusters = new HashMap<>(); - for (int i = 0; i < config.searchcluster().size(); ++i) { - List schemas = new ArrayList<>(); - String clusterName = config.searchcluster(i).name(); - for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j) - schemas.add(config.searchcluster(i).searchdef(j)); - clusters.put(clusterName, schemas); - } - return clusters; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java b/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java deleted file mode 100644 index fbe2ffb8984..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config.internal; - -import com.yahoo.language.Language; -import com.yahoo.language.process.Embedder; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * A class which knows how to convert an Object value to a tensor of a given type. - * - * @author bratseth - */ -public class TensorConverter { - - private static final Pattern embedderArgumentRegexp = Pattern.compile("^([A-Za-z0-9_\\-.]+),\\s*([\"'].*[\"'])"); - - private final Map embedders; - - public TensorConverter(Map embedders) { - this.embedders = embedders; - } - - public Tensor convertTo(TensorType type, String key, Object value, Language language) { - var context = new Embedder.Context(key).setLanguage(language); - Tensor tensor = toTensor(type, value, context); - if (tensor == null) return null; - if (! tensor.type().isAssignableTo(type)) - throw new IllegalArgumentException("Require a tensor of type " + type); - return tensor; - } - - private Tensor toTensor(TensorType type, Object value, Embedder.Context context) { - if (value instanceof Tensor) return (Tensor)value; - if (value instanceof String && isEmbed((String)value)) return embed((String)value, type, context); - if (value instanceof String) return Tensor.from(type, (String)value); - return null; - } - - static boolean isEmbed(String value) { - return value.startsWith("embed("); - } - - private Tensor embed(String s, TensorType type, Embedder.Context embedderContext) { - if ( ! s.endsWith(")")) - throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); - String argument = s.substring("embed(".length(), s.length() - 1); - Embedder embedder; - - // Check if arguments specifies an embedder with the format embed(embedder, "text to encode") - Matcher matcher = embedderArgumentRegexp.matcher(argument); - if (matcher.matches()) { - String embedderId = matcher.group(1); - argument = matcher.group(2); - if ( ! embedders.containsKey(embedderId)) { - throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + - "Valid embedders are " + validEmbedders(embedders)); - } - embedder = embedders.get(embedderId); - } else if (embedders.size() == 0) { - throw new IllegalStateException("No embedders provided"); // should never happen - } else if (embedders.size() > 1) { - throw new IllegalArgumentException("Multiple embedders are provided but no embedder id is given. " + - "Valid embedders are " + validEmbedders(embedders)); - } else { - embedder = embedders.entrySet().stream().findFirst().get().getValue(); - } - - return embedder.embed(removeQuotes(argument), embedderContext, type); - } - - private static String removeQuotes(String s) { - if (s.startsWith("'") && s.endsWith("'")) { - return s.substring(1, s.length() - 1); - } - if (s.startsWith("\"") && s.endsWith("\"")) { - return s.substring(1, s.length() - 1); - } - return s; - } - - private static String validEmbedders(Map embedders) { - List embedderIds = new ArrayList<>(); - embedders.forEach((key, value) -> embedderIds.add(key)); - embedderIds.sort(null); - return String.join(",", embedderIds); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/config/package-info.java b/container-search/src/main/java/com/yahoo/search/config/package-info.java deleted file mode 100644 index dd9c7bfcf04..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/package-info.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi -package com.yahoo.search.config; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; - -/** - * Information about the current configuration this is running as a part of. - */ \ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 82cda9e8a1b..1b12f3f3bb8 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.query.TaggableItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index e0dea744075..ccb9b2d0676 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.profile.types; import com.yahoo.processing.request.Properties; -import com.yahoo.search.config.internal.TensorConverter; +import com.yahoo.search.schema.internal.TensorConverter; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.SubstituteString; import com.yahoo.tensor.Tensor; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 7f4cee07e8c..6c65a5e898a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -5,10 +5,9 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.language.process.Embedder; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.config.SchemaInfo; -import com.yahoo.search.config.internal.TensorConverter; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.search.schema.internal.TensorConverter; import com.yahoo.search.query.Properties; -import com.yahoo.search.query.Ranking; import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java new file mode 100644 index 00000000000..8267e5c937b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.tensor.TensorType; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Information about a rank profile + * + * @author bratseth + */ +public class RankProfile { + + private final String name; + private final boolean hasSummaryFeatures; + private final boolean hasRankFeatures; + private final Map inputs; + + private RankProfile(Builder builder) { + this.name = builder.name; + this.hasSummaryFeatures = builder.hasSummaryFeatures; + this.hasRankFeatures = builder.hasRankFeatures; + this.inputs = Map.copyOf(builder.inputs); + } + + public String name() { return name; } + + /** Returns true if this rank profile has summary features. */ + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + + /** Returns true if this rank profile has rank features. */ + public boolean hasRankFeatures() { return hasRankFeatures; } + + /** Returns the inputs explicitly declared in this rank profile. */ + public Map inputs() { return inputs; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof RankProfile)) return false; + RankProfile other = (RankProfile)o; + if ( ! other.name.equals(this.name)) return false; + if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; + if ( other.hasRankFeatures != this.hasRankFeatures) return false; + if ( ! other.inputs.equals(this.inputs)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); + } + + @Override + public String toString() { + return "rank profile '" + name + "'"; + } + + public static class Builder { + + private final String name; + private boolean hasSummaryFeatures = true; + private boolean hasRankFeatures = true; + private final Map inputs = new HashMap<>(); + + public Builder(String name) { + this.name = Objects.requireNonNull(name); + } + + public Builder setHasSummaryFeatures(boolean hasSummaryFeatures) { + this.hasSummaryFeatures = hasSummaryFeatures; + return this; + } + + public Builder setHasRankFeatures(boolean hasRankFeatures) { + this.hasRankFeatures = hasRankFeatures; + return this; + } + + public Builder addInput(String name, TensorType type) { + inputs.put(name, type); + return this; + } + + public RankProfile build() { + return new RankProfile(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java new file mode 100644 index 00000000000..b66e6ce957a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Information about a schema which is part of the application running this. + * + * This is immutable. + * + * @author bratseth + */ +@Beta +public class Schema { + + private final String name; + private final Map rankProfiles; + + private Schema(Builder builder) { + this.name = builder.name; + this.rankProfiles = Map.copyOf(builder.rankProfiles); + } + + public String name() { return name; } + public Map rankProfiles() { return rankProfiles; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Schema)) return false; + Schema other = (Schema)o; + if ( ! other.name.equals(this.name)) return false; + if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, rankProfiles); + } + + @Override + public String toString() { + return "schema '" + name + "'"; + } + + public static class Builder { + + private final String name; + private final Map rankProfiles = new HashMap<>(); + + public Builder(String name) { + this.name = Objects.requireNonNull(name); + } + + public Builder add(RankProfile profile) { + rankProfiles.put(profile.name(), Objects.requireNonNull(profile)); + return this; + } + + public Schema build() { + return new Schema(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java new file mode 100644 index 00000000000..2d1d391640f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -0,0 +1,148 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.tensor.TensorType; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Information about all the schemas configured in the application this container is a part of. + * + * Usage: + * + * SchemaInfo.Session session = schemaInfo.newSession(query); // once when starting to process a query + * session.get(...) // access information about the schema(s) relevant to the query + * + * + * This is immutable. + * + * @author bratseth + */ +// NOTES: +// This should replace IndexFacts, and probably DocumentDatabase. +// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information. +// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation +// given a set of possible schemas: The session mechanism is present here to make that efficient when added +// (resolving schema subsets for every field lookup is too expensive). +@Beta +public class SchemaInfo { + + private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); + + private final List schemas; + + /** The schemas contained in each content cluster indexed by cluster name */ + private final Map> clusters; + + public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future + SchemaInfoConfig schemaInfoConfig, + QrSearchersConfig qrSearchersConfig) { + this(SchemaInfoConfigurer.toSchemas(schemaInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); + } + + public SchemaInfo(List schemas, Map> clusters) { + this.schemas = List.copyOf(schemas); + this.clusters = Map.copyOf(clusters); + } + + public Session newSession(Query query) { + return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); + } + + public static SchemaInfo empty() { return empty; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof SchemaInfo)) return false; + SchemaInfo other = (SchemaInfo)o; + if ( ! other.schemas.equals(this.schemas)) return false; + if ( ! other.clusters.equals(this.clusters)) return false; + return true; + } + + @Override + public int hashCode() { return Objects.hash(schemas, clusters); } + + /** The schema information resolved to be relevant to this session. */ + public static class Session { + + private final List schemas; + + private Session(Set sources, + Set restrict, + Map> clusters, + List candidates) { + this.schemas = resolveSchemas(sources, restrict, clusters, candidates); + } + + /** + * Given a search list which is a mixture of schemas and cluster + * names, and a restrict list which is a list of schemas, return a + * set of all valid schemas for this combination. + * + * @return the possibly empty list of schemas matching the arguments + */ + private static List resolveSchemas(Set sources, + Set restrict, + Map> clusters, + List candidates) { + if (sources.isEmpty()) + return restrict.isEmpty() ? candidates : keep(restrict, candidates); + + Set schemaNames = new HashSet<>(); + for (String source : sources) { + if (clusters.containsKey(source)) // source is a cluster + schemaNames.addAll(clusters.get(source)); + else // source is a schema + schemaNames.add(source); + } + candidates = keep(schemaNames, candidates); + return restrict.isEmpty() ? candidates : keep(restrict, candidates); + } + + private static List keep(Set names, List schemas) { + return schemas.stream().filter(schema -> names.contains(schema.name())).collect(Collectors.toList()); + } + + /** + * Returns the type of the given rank feature name in the given profile, + * if it can be uniquely determined. + * + * @param rankFeature the rank feature name, a string on the form "query(name)" + * @param rankProfile the name of the rank profile in which to locate the input declaration + * @return the type of the declared input, or null if it is not declared or the rank profile is not found + * @throws IllegalArgumentException if the feature is declared in this rank profile in multiple schemas + * of this session with conflicting types + */ + public TensorType rankProfileInput(String rankFeature, String rankProfile) { + TensorType foundType = null; + Schema declaringSchema = null; + for (Schema schema : schemas) { + RankProfile profile = schema.rankProfiles().get(rankProfile); + if (profile == null) continue; + TensorType newlyFoundType = profile.inputs().get(rankFeature); + if (newlyFoundType == null) continue; + if (foundType != null && ! newlyFoundType.equals(foundType)) + throw new IllegalArgumentException("Conflicting input type declarations for '" + rankFeature + "': " + + "Declared as " + foundType + " in " + profile + " in " + declaringSchema + + ", and as " + newlyFoundType + " in " + profile + " in " + schema); + foundType = newlyFoundType; + declaringSchema = schema; + } + return foundType; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java new file mode 100644 index 00000000000..84ed9ae8e3d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Translation between schema info configuration and schema objects. + * + * @author bratseth + */ +class SchemaInfoConfigurer { + + static List toSchemas(SchemaInfoConfig schemaInfoConfig) { + return schemaInfoConfig.schema().stream().map(config -> toSchema(config)).collect(Collectors.toList()); + } + + static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { + Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); + for (var profileConfig : schemaInfoConfig.rankprofile()) { + RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); + profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); + profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); + for (var inputConfig : profileConfig.input()) + profileBuilder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); + builder.add(profileBuilder.build()); + } + return builder.build(); + } + + static Map> toClusters(QrSearchersConfig config) { + Map> clusters = new HashMap<>(); + for (int i = 0; i < config.searchcluster().size(); ++i) { + List schemas = new ArrayList<>(); + String clusterName = config.searchcluster(i).name(); + for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j) + schemas.add(config.searchcluster(i).searchdef(j)); + clusters.put(clusterName, schemas); + } + return clusters; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java new file mode 100644 index 00000000000..2370513dba2 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema.internal; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class which knows how to convert an Object value to a tensor of a given type. + * + * @author bratseth + */ +public class TensorConverter { + + private static final Pattern embedderArgumentRegexp = Pattern.compile("^([A-Za-z0-9_\\-.]+),\\s*([\"'].*[\"'])"); + + private final Map embedders; + + public TensorConverter(Map embedders) { + this.embedders = embedders; + } + + public Tensor convertTo(TensorType type, String key, Object value, Language language) { + var context = new Embedder.Context(key).setLanguage(language); + Tensor tensor = toTensor(type, value, context); + if (tensor == null) return null; + if (! tensor.type().isAssignableTo(type)) + throw new IllegalArgumentException("Require a tensor of type " + type); + return tensor; + } + + private Tensor toTensor(TensorType type, Object value, Embedder.Context context) { + if (value instanceof Tensor) return (Tensor)value; + if (value instanceof String && isEmbed((String)value)) return embed((String)value, type, context); + if (value instanceof String) return Tensor.from(type, (String)value); + return null; + } + + static boolean isEmbed(String value) { + return value.startsWith("embed("); + } + + private Tensor embed(String s, TensorType type, Embedder.Context embedderContext) { + if ( ! s.endsWith(")")) + throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); + String argument = s.substring("embed(".length(), s.length() - 1); + Embedder embedder; + + // Check if arguments specifies an embedder with the format embed(embedder, "text to encode") + Matcher matcher = embedderArgumentRegexp.matcher(argument); + if (matcher.matches()) { + String embedderId = matcher.group(1); + argument = matcher.group(2); + if ( ! embedders.containsKey(embedderId)) { + throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + + "Valid embedders are " + validEmbedders(embedders)); + } + embedder = embedders.get(embedderId); + } else if (embedders.size() == 0) { + throw new IllegalStateException("No embedders provided"); // should never happen + } else if (embedders.size() > 1) { + throw new IllegalArgumentException("Multiple embedders are provided but no embedder id is given. " + + "Valid embedders are " + validEmbedders(embedders)); + } else { + embedder = embedders.entrySet().stream().findFirst().get().getValue(); + } + + return embedder.embed(removeQuotes(argument), embedderContext, type); + } + + private static String removeQuotes(String s) { + if (s.startsWith("'") && s.endsWith("'")) { + return s.substring(1, s.length() - 1); + } + if (s.startsWith("\"") && s.endsWith("\"")) { + return s.substring(1, s.length() - 1); + } + return s; + } + + private static String validEmbedders(Map embedders) { + List embedderIds = new ArrayList<>(); + embedders.forEach((key, value) -> embedderIds.add(key)); + embedderIds.sort(null); + return String.join(",", embedderIds); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/package-info.java b/container-search/src/main/java/com/yahoo/search/schema/package-info.java new file mode 100644 index 00000000000..f9c86afc3e1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/package-info.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.search.schema; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; + +/** + * Information about the current configuration this is running as a part of. + */ \ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0a4f3e04955..aba72cb3404 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -16,8 +16,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.config.SchemaInfo; -import com.yahoo.search.rendering.Renderer; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; import java.util.Objects; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 4af2b5c9073..54874dbee3e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -11,7 +11,7 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; -import com.yahoo.container.search.SchemaInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; @@ -20,7 +20,7 @@ import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 8bd34c21ad3..e5b14db9b4e 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.search +namespace=search.config ## The name of this schema schema[].name string diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java deleted file mode 100644 index 728ebbf8f7f..00000000000 --- a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class SchemaInfoTest { - - @Test - public void testSchemaInfoConfiguration() { - assertEquals(SchemaInfoTester.createSchemaInfoFromConfig(), SchemaInfoTester.createSchemaInfo()); - } - - @Test - public void testInputResolution() { - var tester = new SchemaInfoTester(); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "", "", "commonProfile", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "ab", "", "commonProfile", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "a", "", "commonProfile", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "b", "", "commonProfile", "query(myTensor1)"); - - tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), - "", "", "inconsistent", "query(myTensor1)"); - tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), - "ab", "", "inconsistent", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "ab", "a", "inconsistent", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(x[10])"), - "ab", "b", "inconsistent", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "a", "", "inconsistent", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(x[10])"), - "b", "", "inconsistent", "query(myTensor1)"); - tester.assertInput(null, - "a", "", "bOnly", "query(myTensor1)"); - tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), - "ab", "", "bOnly", "query(myTensor1)"); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java deleted file mode 100644 index 4e84324a779..00000000000 --- a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.config; - -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.search.SchemaInfoConfig; -import com.yahoo.search.Query; -import com.yahoo.tensor.TensorType; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class SchemaInfoTester { - - private final SchemaInfo schemaInfo; - - SchemaInfoTester() { - this.schemaInfo = createSchemaInfo(); - } - - SchemaInfo schemaInfo() { return schemaInfo; } - - Query query(String sources, String restrict) { - Map params = new HashMap<>(); - if ( ! sources.isEmpty()) - params.put("sources", sources); - if ( ! restrict.isEmpty()) - params.put("restrict", restrict); - return new Query.Builder().setSchemaInfo(schemaInfo) - .setRequestMap(params) - .build(); - } - - void assertInput(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { - assertEquals(expectedType, - schemaInfo.newSession(query(sources, restrict)).rankProfileInput(feature, rankProfile)); - } - - void assertInputConflict(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { - try { - assertInput(expectedType, sources, restrict, rankProfile, feature); - } - catch (IllegalArgumentException e) { - assertEquals("Conflicting input type declarations for '" + feature + "'", - e.getMessage().split(":")[0]); - } - } - - static SchemaInfo createSchemaInfo() { - List schemas = new ArrayList<>(); - RankProfile common = new RankProfile.Builder("commonProfile") - .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) - .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) - .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) - .addInput("query(myTensor4)", TensorType.fromSpec("tensor(x[5])")) - .build(); - schemas.add(new Schema.Builder("a") - .add(common) - .add(new RankProfile.Builder("inconsistent") - .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) - .build()) - .build()); - schemas.add(new Schema.Builder("b") - .add(common) - .add(new RankProfile.Builder("inconsistent") - .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) - .build()) - .add(new RankProfile.Builder("bOnly") - .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) - .build()) - .build()); - Map> clusters = new HashMap<>(); - clusters.put("ab", List.of("a", "b")); - clusters.put("a", List.of("a")); - return new SchemaInfo(schemas, clusters); - } - - /** Creates the same schema info as createSchemaInfo from config objects. */ - static SchemaInfo createSchemaInfoFromConfig() { - var indexInfoConfig = new IndexInfoConfig.Builder(); - - var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileCommon.name("commonProfile"); - rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor2)").type("tensor(x[2],y[2])")); - rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor3)").type("tensor(x[2],y[2])")); - rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor4)").type("tensor(x[5])")); - - var schemaInfoInfoConfig = new SchemaInfoConfig.Builder(); - - var schemaA = new SchemaInfoConfig.Schema.Builder(); - schemaA.name("a"); - schemaA.rankprofile(rankProfileCommon); - var rankProfileInconsistentA = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileInconsistentA.name("inconsistent"); - rankProfileInconsistentA.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - schemaA.rankprofile(rankProfileInconsistentA); - schemaInfoInfoConfig.schema(schemaA); - - var schemaB = new SchemaInfoConfig.Schema.Builder(); - schemaB.name("b"); - schemaB.rankprofile(rankProfileCommon); - var rankProfileInconsistentB = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileInconsistentB.name("inconsistent"); - rankProfileInconsistentB.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); - schemaB.rankprofile(rankProfileInconsistentB); - var rankProfileBOnly = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileBOnly.name("bOnly"); - rankProfileBOnly.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); - schemaB.rankprofile(rankProfileBOnly); - schemaInfoInfoConfig.schema(schemaB); - - var qrSearchersConfig = new QrSearchersConfig.Builder(); - var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); - clusterAB.name("ab"); - clusterAB.searchdef("a").searchdef("b"); - qrSearchersConfig.searchcluster(clusterAB); - var clusterA = new QrSearchersConfig.Searchcluster.Builder(); - clusterA.name("a"); - clusterA.searchdef("a"); - qrSearchersConfig.searchcluster(clusterA); - - return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index 1b10e4cd0ba..f34ad783277 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -5,9 +5,9 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.search.Query; -import com.yahoo.search.config.RankProfile; -import com.yahoo.search.config.Schema; -import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java new file mode 100644 index 00000000000..d2cd2c4bb33 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.tensor.TensorType; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaInfoTest { + + @Test + public void testSchemaInfoConfiguration() { + assertEquals(SchemaInfoTester.createSchemaInfoFromConfig(), SchemaInfoTester.createSchemaInfo()); + } + + @Test + public void testInputResolution() { + var tester = new SchemaInfoTester(); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "a", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "b", "", "commonProfile", "query(myTensor1)"); + + tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), + "", "", "inconsistent", "query(myTensor1)"); + tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "a", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(x[10])"), + "ab", "b", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "a", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(x[10])"), + "b", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(null, + "a", "", "bOnly", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "bOnly", "query(myTensor1)"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java new file mode 100644 index 00000000000..ff996464c4a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -0,0 +1,136 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaInfoTester { + + private final SchemaInfo schemaInfo; + + SchemaInfoTester() { + this.schemaInfo = createSchemaInfo(); + } + + SchemaInfo schemaInfo() { return schemaInfo; } + + Query query(String sources, String restrict) { + Map params = new HashMap<>(); + if ( ! sources.isEmpty()) + params.put("sources", sources); + if ( ! restrict.isEmpty()) + params.put("restrict", restrict); + return new Query.Builder().setSchemaInfo(schemaInfo) + .setRequestMap(params) + .build(); + } + + void assertInput(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { + assertEquals(expectedType, + schemaInfo.newSession(query(sources, restrict)).rankProfileInput(feature, rankProfile)); + } + + void assertInputConflict(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { + try { + assertInput(expectedType, sources, restrict, rankProfile, feature); + } + catch (IllegalArgumentException e) { + assertEquals("Conflicting input type declarations for '" + feature + "'", + e.getMessage().split(":")[0]); + } + } + + static SchemaInfo createSchemaInfo() { + List schemas = new ArrayList<>(); + RankProfile common = new RankProfile.Builder("commonProfile") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor4)", TensorType.fromSpec("tensor(x[5])")) + .build(); + schemas.add(new Schema.Builder("a") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + schemas.add(new Schema.Builder("b") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) + .build()) + .add(new RankProfile.Builder("bOnly") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + Map> clusters = new HashMap<>(); + clusters.put("ab", List.of("a", "b")); + clusters.put("a", List.of("a")); + return new SchemaInfo(schemas, clusters); + } + + /** Creates the same schema info as createSchemaInfo from config objects. */ + static SchemaInfo createSchemaInfoFromConfig() { + var indexInfoConfig = new IndexInfoConfig.Builder(); + + var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileCommon.name("commonProfile"); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor2)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor3)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor4)").type("tensor(x[5])")); + + var schemaInfoInfoConfig = new SchemaInfoConfig.Builder(); + + var schemaA = new SchemaInfoConfig.Schema.Builder(); + schemaA.name("a"); + schemaA.rankprofile(rankProfileCommon); + var rankProfileInconsistentA = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileInconsistentA.name("inconsistent"); + rankProfileInconsistentA.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + schemaA.rankprofile(rankProfileInconsistentA); + schemaInfoInfoConfig.schema(schemaA); + + var schemaB = new SchemaInfoConfig.Schema.Builder(); + schemaB.name("b"); + schemaB.rankprofile(rankProfileCommon); + var rankProfileInconsistentB = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileInconsistentB.name("inconsistent"); + rankProfileInconsistentB.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); + schemaB.rankprofile(rankProfileInconsistentB); + var rankProfileBOnly = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileBOnly.name("bOnly"); + rankProfileBOnly.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + schemaB.rankprofile(rankProfileBOnly); + schemaInfoInfoConfig.schema(schemaB); + + var qrSearchersConfig = new QrSearchersConfig.Builder(); + var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); + clusterAB.name("ab"); + clusterAB.searchdef("a").searchdef("b"); + qrSearchersConfig.searchcluster(clusterAB); + var clusterA = new QrSearchersConfig.Searchcluster.Builder(); + clusterA.name("a"); + clusterA.searchdef("a"); + qrSearchersConfig.searchcluster(clusterA); + + return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build()); + } + +} -- cgit v1.2.3