diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
7 files changed, 246 insertions, 24 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 7a5ef94069d..57f7ea34a97 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -215,7 +215,7 @@ public class RpcProtobufFillInvoker extends FillInvoker { for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); if (summary.valid()) { - hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.getName()); + hits.get(i).setField(Hit.SDDOCNAME_FIELD, documentDb.schema().name()); hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsum(summaryClass), summary); hits.get(i).setFilled(summaryClass); } else { diff --git a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java new file mode 100644 index 00000000000..12037ee6633 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java @@ -0,0 +1,160 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * A document summary definition: Defines the schema on which a document hit may be + * represented in a Result. + * + * @author bratseth + */ +public class DocumentSummary { + + private final String name; + private final List<Field> fields; + private final boolean dynamic; + + private DocumentSummary(Builder builder) { + this.name = builder.name; + this.fields = List.copyOf(builder.fields); + this.dynamic = builder.dynamic; + } + + public String name() { return name; } + public List<Field> fields() { return fields; } + + /** Returns whether this contains fields which are generated dynamically from the query and field data. */ + public boolean isDynamic() { return dynamic; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof DocumentSummary)) return false; + var other = (DocumentSummary)o; + if ( ! other.name.equals(this.name)) return false; + if ( other.dynamic != this.dynamic) return false; + if ( ! other.fields.equals(this.fields)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, dynamic, fields); + } + + @Override + public String toString() { + return "document summary '" + name + "'"; + } + + public static class Builder { + + private final String name; + private final List<Field> fields = new ArrayList<>(); + private boolean dynamic; + + public Builder(String name) { + this.name = name; + } + + public Builder add(Field field) { + fields.add(field); + return this; + } + + public Builder setDynamic(boolean dynamic) { + this.dynamic = dynamic; + return this; + } + + public DocumentSummary build() { return new DocumentSummary(this); } + + } + + public static class Field { + + public enum Type { + bool, + byteType("byte"), + shortType("short"), + integer, + int64, + float16, + floatType("float"), + doubleType("double"), + string, + data, + raw, + longstring, + longdata, + jsonstring, + featuredata, + xmlstring, + tensor; + + private final String name; + + Type() { + this(null); + } + + Type(String name) { + this.name = name; + } + + /** Use this, not name() to retrieve the string value of this. */ + public String asString() { + return name != null ? name : name(); + } + + @Override + public String toString() { return asString(); } + + public static Type fromString(String name) { + return Arrays.stream(Type.values()).filter(t -> name.equals(t.asString())).findAny().orElseThrow(); + } + + } + + private final String name; + private final Type type; + + public Field(String name, String type) { + this(name, Type.fromString(type)); + } + + public Field(String name, Type type) { + this.name = name; + this.type = type; + } + + public String name() { return name; } + public Type type() { return type; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Field)) return false; + var other = (Field)o; + if ( ! other.name.equals(this.name)) return false; + if ( other.type != this.type) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } + + @Override + public String toString() { + return "summary field '" + name + "' " + type; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index 8267e5c937b..39d4a389e6f 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -3,7 +3,9 @@ package com.yahoo.search.schema; import com.yahoo.tensor.TensorType; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -23,7 +25,7 @@ public class RankProfile { this.name = builder.name; this.hasSummaryFeatures = builder.hasSummaryFeatures; this.hasRankFeatures = builder.hasRankFeatures; - this.inputs = Map.copyOf(builder.inputs); + this.inputs = Collections.unmodifiableMap(builder.inputs); } public String name() { return name; } @@ -64,7 +66,7 @@ public class RankProfile { private final String name; private boolean hasSummaryFeatures = true; private boolean hasRankFeatures = true; - private final Map<String, TensorType> inputs = new HashMap<>(); + private final Map<String, TensorType> inputs = new LinkedHashMap<>(); public Builder(String name) { this.name = Objects.requireNonNull(name); diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java index b66e6ce957a..2ab5a30fbd7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/Schema.java +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -3,7 +3,9 @@ package com.yahoo.search.schema; import com.yahoo.api.annotations.Beta; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -19,14 +21,17 @@ public class Schema { private final String name; private final Map<String, RankProfile> rankProfiles; + private final Map<String, DocumentSummary> documentSummaries; private Schema(Builder builder) { this.name = builder.name; - this.rankProfiles = Map.copyOf(builder.rankProfiles); + this.rankProfiles = Collections.unmodifiableMap(builder.rankProfiles); + this.documentSummaries = Collections.unmodifiableMap(builder.documentSummaries); } public String name() { return name; } public Map<String, RankProfile> rankProfiles() { return rankProfiles; } + public Map<String, DocumentSummary> documentSummaries() { return documentSummaries; } @Override public boolean equals(Object o) { @@ -35,12 +40,13 @@ public class Schema { Schema other = (Schema)o; if ( ! other.name.equals(this.name)) return false; if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; + if ( ! other.documentSummaries.equals(this.documentSummaries)) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, rankProfiles); + return Objects.hash(name, rankProfiles, documentSummaries); } @Override @@ -51,14 +57,20 @@ public class Schema { public static class Builder { private final String name; - private final Map<String, RankProfile> rankProfiles = new HashMap<>(); + private final Map<String, RankProfile> rankProfiles = new LinkedHashMap<>(); + private final Map<String, DocumentSummary> documentSummaries = new LinkedHashMap<>(); public Builder(String name) { this.name = Objects.requireNonNull(name); } public Builder add(RankProfile profile) { - rankProfiles.put(profile.name(), Objects.requireNonNull(profile)); + rankProfiles.put(profile.name(), profile); + return this; + } + + public Builder add(DocumentSummary documentSummary) { + documentSummaries.put(documentSummary.name(), documentSummary); return this; } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index 4daf110fc54..7bd66445512 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -2,13 +2,17 @@ package com.yahoo.search.schema; import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -39,11 +43,12 @@ public class SchemaInfo { private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); - private final List<Schema> schemas; + private final Map<String, Schema> schemas; /** The schemas contained in each content cluster indexed by cluster name */ private final Map<String, List<String>> clusters; + @Inject public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future SchemaInfoConfig schemaInfoConfig, QrSearchersConfig qrSearchersConfig) { @@ -51,12 +56,14 @@ public class SchemaInfo { } public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) { - this.schemas = List.copyOf(schemas); - this.clusters = Map.copyOf(clusters); + Map<String, Schema> schemaMap = new LinkedHashMap<>(); + schemas.forEach(schema -> schemaMap.put(schema.name(), schema)); + this.schemas = Collections.unmodifiableMap(schemaMap); + this.clusters = Collections.unmodifiableMap(clusters); } - /** Returns all schemas configured in this application. */ - public List<Schema> schemas() { return schemas; } + /** Returns all schemas configured in this application, indexed by schema name. */ + public Map<String, Schema> schemas() { return schemas; } public Session newSession(Query query) { return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); @@ -80,13 +87,13 @@ public class SchemaInfo { /** The schema information resolved to be relevant to this session. */ public static class Session { - private final List<Schema> schemas; + private final Collection<Schema> schemas; private Session(Set<String> sources, Set<String> restrict, Map<String, List<String>> clusters, - List<Schema> candidates) { - this.schemas = resolveSchemas(sources, restrict, clusters, candidates); + Map<String, Schema> candidates) { + this.schemas = resolveSchemas(sources, restrict, clusters, candidates.values()); } /** @@ -96,10 +103,10 @@ public class SchemaInfo { * * @return the possibly empty list of schemas matching the arguments */ - private static List<Schema> resolveSchemas(Set<String> sources, - Set<String> restrict, - Map<String, List<String>> clusters, - List<Schema> candidates) { + private static Collection<Schema> resolveSchemas(Set<String> sources, + Set<String> restrict, + Map<String, List<String>> clusters, + Collection<Schema> candidates) { if (sources.isEmpty()) return restrict.isEmpty() ? candidates : keep(restrict, candidates); @@ -114,7 +121,7 @@ public class SchemaInfo { return restrict.isEmpty() ? candidates : keep(restrict, candidates); } - private static List<Schema> keep(Set<String> names, List<Schema> schemas) { + private static List<Schema> keep(Set<String> names, Collection<Schema> schemas) { return schemas.stream().filter(schema -> names.contains(schema.name())).collect(Collectors.toList()); } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 84ed9ae8e3d..c5a74ece866 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -1,11 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.DocsumDefinition; +import com.yahoo.prelude.fastsearch.DocsumField; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,6 +30,7 @@ class SchemaInfoConfigurer { static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); + for (var profileConfig : schemaInfoConfig.rankprofile()) { RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); @@ -32,6 +39,17 @@ class SchemaInfoConfigurer { profileBuilder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); builder.add(profileBuilder.build()); } + + for (var summaryConfig : schemaInfoConfig.summaryclass()) { + DocumentSummary.Builder summaryBuilder = new DocumentSummary.Builder(summaryConfig.name()); + for (var field : summaryConfig.fields()) { + if (field.dynamic()) + summaryBuilder.setDynamic(true); + summaryBuilder.add(new DocumentSummary.Field(field.name(), field.type())); + } + builder.add(summaryBuilder.build()); + } + return builder.build(); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 3e44b02618e..bf8ef39001f 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -2,6 +2,7 @@ package com.yahoo.search.searchchain; import com.google.inject.Inject; +import com.yahoo.api.annotations.Beta; import com.yahoo.component.AbstractComponent; import com.yahoo.component.chain.Chain; import com.yahoo.component.chain.ChainsConfigurer; @@ -48,6 +49,8 @@ public class ExecutionFactory extends AbstractComponent { private final RendererRegistry rendererRegistry; private final Executor executor; + // TODO: Fix tests depending on HandlersConfigurerTestWrapper so that this constructor can be removed + @Beta @Inject public ExecutionFactory(ChainsConfig chainsConfig, IndexInfoConfig indexInfo, @@ -58,9 +61,29 @@ public class ExecutionFactory extends AbstractComponent { Linguistics linguistics, ComponentRegistry<Renderer> renderers, Executor executor) { + this(chainsConfig, + indexInfo, + new SchemaInfo(indexInfo, schemaInfo, clusters), + clusters, + searchers, + specialTokens, + linguistics, + renderers, + executor); + } + + public ExecutionFactory(ChainsConfig chainsConfig, + IndexInfoConfig indexInfo, + SchemaInfo schemaInfo, + QrSearchersConfig clusters, + ComponentRegistry<Searcher> searchers, + SpecialtokensConfig specialTokens, + Linguistics linguistics, + ComponentRegistry<Renderer> renderers, + Executor executor) { this.searchChainRegistry = createSearchChainRegistry(searchers, chainsConfig); this.indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters)).freeze(); - this.schemaInfo = new SchemaInfo(indexInfo, schemaInfo, clusters); + this.schemaInfo = schemaInfo; this.specialTokens = new SpecialTokenRegistry(specialTokens); this.linguistics = linguistics; this.renderingExecutor = createRenderingExecutor(); @@ -78,7 +101,7 @@ public class ExecutionFactory extends AbstractComponent { Linguistics linguistics, ComponentRegistry<Renderer> renderers, Executor executor) { - this(chainsConfig, indexInfo, new SchemaInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, executor); + this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, executor); } /** @deprecated pass the container threadpool */ @@ -90,7 +113,7 @@ public class ExecutionFactory extends AbstractComponent { SpecialtokensConfig specialTokens, Linguistics linguistics, ComponentRegistry<Renderer> renderers) { - this(chainsConfig, indexInfo, new SchemaInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, null); + this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, null); } private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, @@ -144,7 +167,7 @@ public class ExecutionFactory extends AbstractComponent { public static ExecutionFactory empty() { return new ExecutionFactory(new ChainsConfig.Builder().build(), new IndexInfoConfig.Builder().build(), - new SchemaInfoConfig.Builder().build(), + SchemaInfo.empty(), new QrSearchersConfig.Builder().build(), new ComponentRegistry<>(), new SpecialtokensConfig.Builder().build(), |