diff options
50 files changed, 501 insertions, 292 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 99a4f7f9425..2ac535b5491 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -2540,9 +2540,9 @@ eslint-plugin-react@^7: string.prototype.matchall "^4.0.8" eslint-plugin-unused-imports@^3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-3.0.0.tgz#d25175b0072ff16a91892c3aa72a09ca3a9e69e7" - integrity sha512-sduiswLJfZHeeBJ+MQaG+xYzSWdRXoSw61DpU13mzWumCkR0ufD0HmO4kdNokjrkluMHpj/7PJeN35pgbhW3kw== + version "3.1.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-3.1.0.tgz#db015b569d3774e17a482388c95c17bd303bc602" + integrity sha512-9l1YFCzXKkw1qtAru1RWUtG2EVDZY0a0eChKXcL+EZ5jitG7qxdctu4RnvhOJHv4xfmUf7h+JJPINlVpGhZMrw== dependencies: eslint-rule-composer "^0.3.0" @@ -3089,9 +3089,9 @@ human-signals@^2.1.0: integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== husky@^9.0.0: - version "9.0.10" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.0.10.tgz#ddca8908deb5f244e9286865ebc80b54387672c2" - integrity sha512-TQGNknoiy6bURzIO77pPRu+XHi6zI7T93rX+QnJsoYFf3xdjKOur+IlfqzJGMHIK/wXrLg+GsvMs8Op7vI2jVA== + version "9.0.11" + resolved "https://registry.yarnpkg.com/husky/-/husky-9.0.11.tgz#fc91df4c756050de41b3e478b2158b87c1e79af9" + integrity sha512-AB6lFlbwwyIqMdHYhwPe+kjOC3Oc5P3nThEoW/AaO2BX3vJDjWPFxYLxokUZOo6RNX20He3AaT8sESs9NJcmEw== ignore@^5.2.0, ignore@^5.3.0: version "5.3.0" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index a6d063592c1..28e11ae0deb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -26,25 +26,14 @@ import java.util.Optional; public class FederationSearcher extends Searcher<FederationSearcherModel> implements FederationConfig.Producer { private final Optional<Component> targetSelector; + private final Map<ComponentId, Target> resolvedTargets = new LinkedHashMap<>(); /** * Generates config for a single search chain contained in a target. */ - private static final class SearchChainConfig { - - private final SearchChain searchChain; - final ComponentId providerId; - final FederationOptions targetOptions; - final List<String> documentTypes; - - SearchChainConfig(SearchChain searchChain, ComponentId providerId, - FederationOptions targetOptions, List<String> documentTypes) { - this.searchChain = searchChain; - this.providerId = providerId; - this.targetOptions = targetOptions; - this.documentTypes = documentTypes; - } - + private record SearchChainConfig(SearchChain searchChain, ComponentId providerId, + FederationOptions targetOptions, List<String> documentTypes) + { FederationConfig.Target.SearchChain.Builder getSearchChainConfig() { FederationConfig.Target.SearchChain.Builder sB = new FederationConfig.Target.SearchChain.Builder(); FederationOptions resolvedOptions = targetOptions.inherit(searchChain.federationOptions()); @@ -77,9 +66,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem FederationConfig.Target.Builder getTargetConfig() { FederationConfig.Target.Builder tb = new FederationConfig.Target.Builder(); - tb. - id(id.stringValue()). - useByDefault(targetOptions.getUseByDefault()); + tb.id(id.stringValue()).useByDefault(targetOptions.getUseByDefault()); getSearchChainsConfig(tb); return tb; } @@ -137,10 +124,9 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } } - private static class TargetResolver { - - final ComponentRegistry<SearchChain> searchChainRegistry; - final SourceGroupRegistry sourceGroupRegistry; + private record TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, + SourceGroupRegistry sourceGroupRegistry) + { /** Returns true if searchChain.id newer than sourceGroup.id */ private boolean newerVersion(SearchChain searchChain, SourceGroup sourceGroup) { @@ -153,12 +139,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem return a.compareTo(b) > 0; } - TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { - this.searchChainRegistry = searchChainRegistry; - this.sourceGroupRegistry = sourceGroupRegistry; - } - - Target resolve(FederationSearcherModel.TargetSpec specification) { + Target resolve(TargetSpec specification) { SearchChain searchChain = searchChainRegistry.getComponent(specification.sourceSpec); SourceGroup sourceGroup = sourceGroupRegistry.getComponent(specification.sourceSpec); @@ -172,13 +153,10 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } } - private final Map<ComponentId, Target> resolvedTargets = new LinkedHashMap<>(); - public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) { super(searcherModel); this.targetSelector = targetSelector; - - targetSelector.ifPresent(selector -> addChild(selector)); + targetSelector.ifPresent(this::addChild); } @Override @@ -196,16 +174,13 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem void initialize(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { TargetResolver targetResolver = new TargetResolver(searchChainRegistry, sourceGroupRegistry); - addSourceTargets(targetResolver, model.targets); - if (model.inheritDefaultSources) addDefaultTargets(targetResolver, searchChainRegistry); } private void addSourceTargets(TargetResolver targetResolver, List<TargetSpec> targets) { for (TargetSpec targetSpec : targets) { - Target target = targetResolver.resolve(targetSpec); if (target == null) { throw new IllegalArgumentException("Can't find source " + targetSpec.sourceSpec + @@ -220,7 +195,6 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } } - private void addDefaultTargets(TargetResolver targetResolver, ComponentRegistry<SearchChain> searchChainRegistry) { for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) { ComponentSpecification specification = genericTarget.getComponentId().toSpecification(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java index 037da825e74..4c382a75b24 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -98,7 +98,7 @@ public class SearchChains extends Chains<SearchChain> { return allChains; } - private void addSources(ComponentRegistry<SearchChain> chains, Provider provider) { + private static void addSources(ComponentRegistry<SearchChain> chains, Provider provider) { for (Source source : provider.getSources()) { chains.register(source.getId(), source); } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java index 0512560e0b7..f92d218390f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java @@ -10,7 +10,6 @@ import com.yahoo.jdisc.Metric; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 5e59d998e86..95b93617b6f 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -34,7 +34,7 @@ reuseAddress bool default=true idleTimeout double default=180.0 # The idle timeout that takes effect during graceful shutdown of Jetty -shutdownIdleTimeout double default=5.0 +shutdownIdleTimeout double default=1.0 # TODO Vespa 9 Remove # Has no effect since Jetty 11 upgrade diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 2304743873f..35d097f23f1 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -8517,6 +8517,7 @@ ], "methods" : [ "public boolean isStreaming()", + "public java.util.Collection schemas()", "public java.util.Optional fieldInfo(java.lang.String)", "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" ], diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index c0dce3734b2..e4e50f8f2ff 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -3,9 +3,7 @@ package com.yahoo.prelude; import com.yahoo.search.Query; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,8 +27,6 @@ import static com.yahoo.text.Lowercase.toLowerCase; // TODO: Complete migration to SchemaInfo public class IndexFacts { - private final Map<String, List<String>> clusterByDocument; - private record DocumentTypeListOffset(int offset, SearchDefinition searchDefinition) { } /** A Map of all known search definitions indexed by name */ @@ -56,7 +52,6 @@ public class IndexFacts { public IndexFacts() { searchDefinitions = Map.of(); clusters = Map.of(); - clusterByDocument = Map.of(); } public IndexFacts(IndexModel indexModel) { @@ -65,28 +60,6 @@ public class IndexFacts { this.unionSearchDefinition = indexModel.getUnionSearchDefinition(); } this.clusters = indexModel.getMasterClusters(); - clusterByDocument = invert(clusters); - } - - private static Map<String, List<String>> invert(Map<String, List<String>> clusters) { - Map<String, List<String>> result = new HashMap<>(); - for (Map.Entry<String,List<String>> entry : clusters.entrySet()) { - for (String value : entry.getValue()) { - addEntry(result, value, entry.getKey()); - } - } - return result; - } - - private static void addEntry(Map<String, List<String>> result, String key, String value) { - List<String> values = result.computeIfAbsent(key, k -> new ArrayList<>()); - values.add(value); - } - - // Assumes that document names are equal to the search definition that contain them. - public List<String> clustersHavingSearchDefinition(String searchDefinitionName) { - List<String> clusters = clusterByDocument.get(searchDefinitionName); - return clusters != null ? clusters : List.of(); } private boolean notInitialized() { diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 88cc7ad7b2d..b0456b941f4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -8,7 +8,6 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FastSearcher; @@ -22,6 +21,7 @@ import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.streamingvisitors.StreamingSearcher; @@ -29,7 +29,6 @@ import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -39,6 +38,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; +import java.util.stream.Collectors; import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; @@ -59,6 +59,7 @@ public class ClusterSearcher extends Searcher { // The set of document types contained in this search cluster private final Set<String> schemas; + private final SchemaInfo schemaInfo; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds @@ -80,6 +81,7 @@ public class ClusterSearcher extends Searcher { VespaDocumentAccess access) { super(id); this.executor = executor; + this.schemaInfo = schemaInfo; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -156,19 +158,20 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { - this.schemas = schemas; + ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { + this.schemaInfo = schemaInfo; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; this.globalPhaseRanker = null; + this.schemas = schemas; } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(Set<String> schemas) { - this(schemas, null, null); + ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas) { + this(schemaInfo, schemas, null, null); } @Override @@ -283,7 +286,7 @@ public class ClusterSearcher extends Searcher { } private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { - Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); + Set<String> schemas = resolveSchemas(query); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { return perSchemaSearch(searcher, queries.get(0), execution); @@ -316,13 +319,24 @@ public class ClusterSearcher extends Searcher { } } - Set<String> resolveSchemas(Query query, IndexFacts indexFacts) { + private Set<String> resolveSourceSubset(Set<String> sources) { + Set<String> candidates = new HashSet<>(); + for (String source : sources) { + Cluster cluster = schemaInfo.clusters().get(source); + if (cluster != null) + candidates.addAll(cluster.schemas()); + } + return (candidates.isEmpty() ? sources : candidates).stream() + .filter(schemas::contains).collect(Collectors.toUnmodifiableSet()); + } + + Set<String> resolveSchemas(Query query) { Set<String> restrict = query.getModel().getRestrict(); if (restrict == null || restrict.isEmpty()) { Set<String> sources = query.getModel().getSources(); return (sources == null || sources.isEmpty()) ? schemas - : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), schemas).documentTypes()); + : resolveSourceSubset(sources); } else { return filterValidDocumentTypes(restrict); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 5dab9d2988f..2e635d21f01 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -9,7 +9,6 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.prelude.querytransform.QueryRewrite; -import com.yahoo.processing.request.CompoundName; import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -223,7 +222,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove List<Result> parts = partitionHits(result, summaryClass); - if (parts.size() > 0) { // anything to fill at all? + if (!parts.isEmpty()) { // anything to fill at all? for (Result r : parts) { doPartialFill(r, summaryClass); mergeErrorsInto(result, r); @@ -379,11 +378,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new FillHitsResult(skippedHits, lastError); } - protected DocsumDefinitionSet getDocsumDefinitionSet(Query query) { - DocumentDatabase db = getDocumentDatabase(query); - return db.getDocsumDefinitionSet(); - } - private String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata) { DocumentDatabase db = getDocumentDatabase(hit.getQuery()); hit.setField(Hit.SDDOCNAME_FIELD, db.schema().name()); diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index e5393fe9b85..f40caac1562 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -11,7 +11,6 @@ import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.errorhandling.Results; import com.yahoo.errorhandling.Results.Builder; -import com.yahoo.prelude.IndexFacts; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -25,11 +24,14 @@ import com.yahoo.search.federation.sourceref.SingleTarget; import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.federation.sourceref.VirtualSourceResolver; import com.yahoo.search.query.Properties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.HitOrderer; +import com.yahoo.search.schema.Cluster; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.AsyncExecution; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.ForkingSearcher; @@ -41,6 +43,7 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.LinkedHashSet; @@ -77,25 +80,34 @@ public class FederationSearcher extends ForkingSearcher { private final SearchChainResolver searchChainResolver; private final SourceRefResolver sourceRefResolver; + private final VirtualSourceResolver virtualSourceResolver; private final TargetSelector<?> targetSelector; private final Clock clock = Clock.systemUTC(); @Inject - public FederationSearcher(FederationConfig config, ComponentRegistry<TargetSelector> targetSelectors) { - this(createResolver(config), resolveSelector(config.targetSelector(), targetSelectors)); + public FederationSearcher(FederationConfig config, SchemaInfo schemaInfo, + ComponentRegistry<TargetSelector> targetSelectors) { + this(createResolver(config), + VirtualSourceResolver.of(config), + resolveSelector(config.targetSelector(), targetSelectors), + createSchema2Clusters(schemaInfo)); } // for testing - public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, null); + public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver, + Map<String, List<String>> schema2Clusters) { + this(searchChainResolver, VirtualSourceResolver.of(), null, schema2Clusters); } private FederationSearcher(SearchChainResolver searchChainResolver, - TargetSelector targetSelector) { + VirtualSourceResolver virtualSourceResolver, + TargetSelector targetSelector, + Map<String, List<String>> schema2Clusters) { this.searchChainResolver = searchChainResolver; - sourceRefResolver = new SourceRefResolver(searchChainResolver); + sourceRefResolver = new SourceRefResolver(searchChainResolver, schema2Clusters); this.targetSelector = targetSelector; + this.virtualSourceResolver = virtualSourceResolver; } private static TargetSelector resolveSelector(String selectorId, @@ -105,6 +117,16 @@ public class FederationSearcher extends ForkingSearcher { "Missing target selector with id '" + selectorId + "'"); } + private static Map<String, List<String>> createSchema2Clusters(SchemaInfo schemaInfo) { + Map<String, List<String>> schema2Clusters = new HashMap<>(); + for (Cluster cluster : schemaInfo.clusters().values()) { + for (String schema : cluster.schemas()) { + schema2Clusters.computeIfAbsent(schema, key -> new ArrayList<>()).add(cluster.name()); + } + } + return schema2Clusters; + } + private static SearchChainResolver createResolver(FederationConfig config) { SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); @@ -160,7 +182,7 @@ public class FederationSearcher extends ForkingSearcher { Result mergedResults = execution.search(query); Results<SearchChainInvocationSpec, UnresolvedSearchChainException> targets = - getTargets(query.getModel().getSources(), query.properties(), execution.context().getIndexFacts()); + getTargets(query.getModel().getSources(), query.properties()); warnIfUnresolvedSearchChains(targets.errors(), mergedResults.hits()); Collection<SearchChainInvocationSpec> prunedTargets = @@ -277,7 +299,7 @@ public class FederationSearcher extends ForkingSearcher { return descriptions; } - private Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { + private static Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { Set<String> messages = new LinkedHashSet<>(); for (UnresolvedSearchChainException exception : unresolvedSearchChainExceptions) { messages.add(exception.getMessage()); @@ -370,9 +392,9 @@ public class FederationSearcher extends ForkingSearcher { else { // fill timed out: Remove these hits as they are incomplete and may cause a race when accessed later result.hits().addError(futureFilledResult.getSecond().createTimeoutError()); for (Iterator<Hit> i = futureFilledResult.getFirst().hits().unorderedDeepIterator(); i.hasNext(); ) { - // Note that some of these hits may be filled, but as the fill thread may still be working on them - // and we do not synchronize with it we need to discard all - Hit removed = result.hits().remove(i.next().getId()); + // Note that some of these hits may be filled, but as the fill thread may still be working on them, + // and we do not synchronize with it, we need to discard all. + result.hits().remove(i.next().getId()); } } } @@ -421,18 +443,20 @@ public class FederationSearcher extends ForkingSearcher { return orderer; } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties, IndexFacts indexFacts) { + private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties) { return sources.isEmpty() ? defaultSearchChains(properties): - resolveSources(sources, properties, indexFacts); + resolveSources(sources, properties); } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sources, Properties properties, IndexFacts indexFacts) { + + private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sourcesInQuery, Properties properties) { Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + Set<String> sources = virtualSourceResolver.resolve(sourcesInQuery); for (String source : sources) { try { - result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties, indexFacts)); + result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties)); } catch (UnresolvedSearchChainException e) { result.addError(e); } @@ -578,11 +602,7 @@ public class FederationSearcher extends ForkingSearcher { /** Returns a result to fill for a query and chain, by creating it if necessary */ public Result get(Chain<Searcher> chain, Query query) { - Map<Query,Result> resultsToFillForAChain = resultsToFill.get(chain); - if (resultsToFillForAChain == null) { - resultsToFillForAChain = new IdentityHashMap<>(); - resultsToFill.put(chain,resultsToFillForAChain); - } + Map<Query, Result> resultsToFillForAChain = resultsToFill.computeIfAbsent(chain, k -> new IdentityHashMap<>()); Result resultsToFillForAChainAndQuery = resultsToFillForAChain.get(query); if (resultsToFillForAChainAndQuery == null) { @@ -686,27 +706,18 @@ public class FederationSearcher extends ForkingSearcher { } - private static class Window { - - private final int hits; - private final int offset; - - public Window(int hits, int offset) { - this.hits = hits; - this.offset = offset; - } + private record Window(int hits, int offset) { public Integer get(CompoundName parameterName) { if (parameterName.equals(Query.HITS)) return hits; if (parameterName.equals(Query.OFFSET)) return offset; return null; } - + public static Window from(Query query) { return new Window(query.getHits(), query.getOffset()); } - public static Window from(Collection<Target> targets, Query query) { if (targets.size() == 1) // preserve requested top-level offsets return Window.from(query); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index bfb5bf1a9ab..df91b968750 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -56,12 +56,8 @@ public class SearchChainResolver { } }; - public Builder addSearchChain(ComponentId searchChainId) { - return addSearchChain(searchChainId, Collections.<String>emptyList()); - } - public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions) { - return addSearchChain(searchChainId, federationOptions, Collections.<String>emptyList()); + return addSearchChain(searchChainId, federationOptions, List.of()); } public Builder addSearchChain(ComponentId searchChainId, List<String> documentTypes) { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java index 7345868cae7..2e7849dd85a 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java @@ -2,11 +2,11 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; import com.yahoo.processing.request.Properties; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -17,30 +17,30 @@ import java.util.Set; public class SourceRefResolver { private final SearchChainResolver searchChainResolver; + private final Map<String, List<String>> schema2Clusters; - public SourceRefResolver(SearchChainResolver searchChainResolver) { + public SourceRefResolver(SearchChainResolver searchChainResolver, Map<String, List<String>> schema2Clusters) { this.searchChainResolver = searchChainResolver; + this.schema2Clusters = schema2Clusters; } public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, - Properties sourceToProviderMap, - IndexFacts indexFacts) throws UnresolvedSearchChainException { + Properties sourceToProviderMap) throws UnresolvedSearchChainException { try { - return new LinkedHashSet<>(List.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap))); + return Set.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap)); } catch (UnresolvedSourceRefException e) { - return resolveClustersWithDocument(sourceRef, sourceToProviderMap, indexFacts); + return resolveClustersWithDocument(sourceRef, sourceToProviderMap); } } private Set<SearchChainInvocationSpec> resolveClustersWithDocument(ComponentSpecification sourceRef, - Properties sourceToProviderMap, - IndexFacts indexFacts) + Properties sourceToProviderMap) throws UnresolvedSearchChainException { if (hasOnlyName(sourceRef)) { Set<SearchChainInvocationSpec> clusterSearchChains = new LinkedHashSet<>(); - List<String> clusters = indexFacts.clustersHavingSearchDefinition(sourceRef.getName()); + List<String> clusters = schema2Clusters.getOrDefault(sourceRef.getName(), List.of()); for (String cluster : clusters) { clusterSearchChains.add(resolveClusterSearchChain(cluster, sourceRef, sourceToProviderMap)); } @@ -48,9 +48,7 @@ public class SourceRefResolver { if ( ! clusterSearchChains.isEmpty()) return clusterSearchChains; } - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); - } private SearchChainInvocationSpec resolveClusterSearchChain(String cluster, diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java index 54b022e0b97..b6d99758c7b 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.federation.sourceref; - import com.google.common.base.Joiner; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java index 3cf2776259c..0c8562e6032 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java @@ -5,7 +5,6 @@ package com.yahoo.search.federation.sourceref; * Thrown if a search chain can not be resolved from one or more ids. * @author Tony Vaagenes */ -@SuppressWarnings("serial") public class UnresolvedSearchChainException extends Exception { public UnresolvedSearchChainException(String msg) { super(msg); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java index a7da7a7ee04..fa2c1da13f0 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java @@ -6,7 +6,6 @@ import com.yahoo.component.ComponentSpecification; /** * @author Tony Vaagenes */ -@SuppressWarnings("serial") class UnresolvedSourceRefException extends UnresolvedSearchChainException { UnresolvedSourceRefException(String msg) { super(msg); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java new file mode 100644 index 00000000000..fc07d12d429 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java @@ -0,0 +1,65 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +import com.yahoo.search.federation.FederationConfig; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Multiple sources like contentcluster.schema1, contencluster.schema2 needs to respond + * when source is the prefix contentcluster. This is done by generating map from virtual source + * to the fully qualified ones, and resolving from there. + * + * @author baldersheim + */ +public class VirtualSourceResolver { + private final Map<String, Set<String>> virtualSources; + private VirtualSourceResolver(Map<String, Set<String>> virtualSources) { + this.virtualSources = virtualSources; + } + public static VirtualSourceResolver of() { + return new VirtualSourceResolver(Map.of()); + } + public static VirtualSourceResolver of(Set<String> targets) { + return new VirtualSourceResolver(createVirtualSources(targets)); + } + private static Map<String, Set<String>> createVirtualSources(Set<String> targets) { + Set<String> virtualSources = targets.stream() + .filter(id -> id.contains(".")) + .map(id -> id.substring(0, id.indexOf('.'))) + .collect(Collectors.toUnmodifiableSet()); + if (virtualSources.isEmpty()) return Map.of(); + Map<String, Set<String>> virtualSourceMap = new HashMap<>(); + for (String virtualSource : virtualSources) { + String prefix = virtualSource + "."; + Set<String> sources = targets.stream() + .filter(id -> id.startsWith(prefix)) + .collect(Collectors.toUnmodifiableSet()); + virtualSourceMap.put(virtualSource, sources); + } + return virtualSourceMap; + } + public static VirtualSourceResolver of(FederationConfig config) { + return of(config.target().stream().map(FederationConfig.Target::id).collect(Collectors.toUnmodifiableSet())); + } + public Set<String> resolve(Set<String> sourcesInQuery) { + boolean hasMapping = sourcesInQuery.stream().anyMatch(virtualSources::containsKey); + if (hasMapping) { + Set<String> resolved = new HashSet<>(); + for (String source : sourcesInQuery) { + var subSources = virtualSources.get(source); + if (subSources != null) { + resolved.addAll(subSources); + } else { + resolved.add(source); + } + } + return resolved; + } + return sourcesInQuery; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java index 86684603a45..4cb19bff740 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java @@ -33,10 +33,6 @@ public class DefaultPositionSearcher extends Searcher { this.useV8GeoPositions = cfg.usev8geopositions(); } - DefaultPositionSearcher() { - this.useV8GeoPositions = false; - } - @Override public com.yahoo.search.Result search(Query query, Execution execution) { Location location = query.getRanking().getLocation(); diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index bbd303039cf..263fa4058c7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -5,11 +5,9 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.component.annotation.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; -import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -65,7 +63,7 @@ public class SchemaInfo { /** Returns all schemas configured in this application, indexed by schema name. */ public Map<String, Schema> schemas() { return schemas; } - /** Returns information about all clusters available for searching in this applications, indexed by cluyster name. */ + /** Returns information about all clusters available for searching in this application, indexed by cluster name. */ public Map<String, Cluster> clusters() { return clusters; } public Session newSession(Query query) { @@ -103,6 +101,8 @@ public class SchemaInfo { /** Returns true if this only searches streaming clusters. */ public boolean isStreaming() { return isStreaming; } + public Collection<Schema> schemas() { return schemas; } + /** * Looks up a field or field set by the given name or alias * in the schemas resolved for this query. diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 84cf1744e27..b70f5145e56 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -16,7 +16,7 @@ import java.util.List; class SchemaInfoConfigurer { static List<Schema> toSchemas(SchemaInfoConfig schemaInfoConfig) { - return schemaInfoConfig.schema().stream().map(config -> toSchema(config)).toList(); + return schemaInfoConfig.schema().stream().map(SchemaInfoConfigurer::toSchema).toList(); } static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index f2ce555a068..8d4e3364ce4 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -7,9 +7,6 @@ import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; -import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -18,6 +15,7 @@ import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.result.Hit; +import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; @@ -36,6 +34,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -54,7 +53,7 @@ public class ClusterSearcherTestCase { @Test void testNoBackends() { - ClusterSearcher cluster = new ClusterSearcher(new LinkedHashSet<>(List.of("dummy"))); + ClusterSearcher cluster = new ClusterSearcher(createSchemaInfo(), Set.of("dummy")); try { Execution execution = new Execution(cluster, Execution.Context.createContextStub()); Query query = new Query("query=hello"); @@ -67,55 +66,58 @@ public class ClusterSearcherTestCase { } } - private IndexFacts createIndexFacts() { - Map<String, List<String>> clusters = new LinkedHashMap<>(); - clusters.put("cluster1", List.of("type1", "type2", "type3")); - clusters.put("cluster2", List.of("type4", "type5")); - clusters.put("type1", List.of("type6")); - Collection<SearchDefinition> searchDefs = List.of( - new SearchDefinition("type1"), - new SearchDefinition("type2"), - new SearchDefinition("type3"), - new SearchDefinition("type4"), - new SearchDefinition("type5"), - new SearchDefinition("type6")); - return new IndexFacts(new IndexModel(clusters, searchDefs)); + private static SchemaInfo createSchemaInfo() { + var schemas = Stream.of("type1", "type2", "type3", "type4", "type5", "type6") + .map(name -> new Schema.Builder(name).build()).toList(); + var clusters = List.of(new Cluster.Builder("cluster1").addSchema("type1").addSchema("type2").addSchema("type3").build(), + new Cluster.Builder("cluster2").addSchema("type4").addSchema("type5").build(), + new Cluster.Builder("type1").addSchema("type6").build()); + return new SchemaInfo(schemas, clusters); } private Set<String> resolve(ClusterSearcher searcher, String query) { - return searcher.resolveSchemas(new Query("?query=hello" + query), createIndexFacts()); + return searcher.resolveSchemas(new Query("?query=hello" + query)); + } + + private static SchemaInfo toSchemaInfo(Collection<String> schemaNames, String clusterName) { + Cluster.Builder clusterBuilder = new Cluster.Builder(clusterName); + + schemaNames.forEach(clusterBuilder::addSchema); + return new SchemaInfo(schemaNames.stream().map(name -> new Schema.Builder(name).build()).toList(), + List.of(clusterBuilder.build())); } @Test void testThatDocumentTypesAreResolved() { - ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type1", "type2", "type3"))); + SchemaInfo schemaInfo = createSchemaInfo(); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); try { - ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type6"))); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); try { - assertEquals(new LinkedHashSet<>(List.of("type1", "type2", "type3")), resolve(cluster1, "")); - assertEquals(new LinkedHashSet<>(List.of("type6")), resolve(type1, "")); + assertEquals(Set.of("type1", "type2", "type3"), resolve(cluster1, "")); + assertEquals(Set.of("type6"), resolve(type1, "")); { // specify restrict - assertEquals(new LinkedHashSet<>(List.of("type1")), resolve(cluster1, "&restrict=type1")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&restrict=type2")); - assertEquals(new LinkedHashSet<>(List.of("type2", "type3")), resolve(cluster1, "&restrict=type2,type3")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&restrict=type2,type4")); - assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&restrict=type4")); + assertEquals(Set.of("type1"), resolve(cluster1, "&restrict=type1")); + assertEquals(Set.of("type2"), resolve(cluster1, "&restrict=type2")); + assertEquals(Set.of("type2", "type3"), resolve(cluster1, "&restrict=type2,type3")); + assertEquals(Set.of("type2"), resolve(cluster1, "&restrict=type2,type4")); + assertEquals(Set.of(), resolve(cluster1, "&restrict=type4")); } { // specify sources - assertEquals(new LinkedHashSet<>(List.of("type1", "type2", "type3")), resolve(cluster1, "&sources=cluster1")); - assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&sources=cluster2")); - assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&sources=type1")); - assertEquals(new LinkedHashSet<>(List.of("type6")), resolve(type1, "&sources=type1")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&sources=type2")); - assertEquals(new LinkedHashSet<>(List.of("type2", "type3")), resolve(cluster1, "&sources=type2,type3")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&sources=type2,type4")); - assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&sources=type4")); + assertEquals(Set.of("type1", "type2", "type3"), resolve(cluster1, "&sources=cluster1")); + assertEquals(Set.of(), resolve(cluster1, "&sources=cluster2")); + assertEquals(Set.of(), resolve(cluster1, "&sources=type1")); + assertEquals(Set.of("type6"), resolve(type1, "&sources=type1")); + assertEquals(Set.of("type2"), resolve(cluster1, "&sources=type2")); + assertEquals(Set.of("type2", "type3"), resolve(cluster1, "&sources=type2,type3")); + assertEquals(Set.of("type2"), resolve(cluster1, "&sources=type2,type4")); + assertEquals(Set.of(), resolve(cluster1, "&sources=type4")); } { // specify both - assertEquals(new LinkedHashSet<>(List.of("type1")), resolve(cluster1, "&sources=cluster1&restrict=type1")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&sources=cluster1&restrict=type2")); - assertEquals(new LinkedHashSet<>(List.of("type2", "type3")), resolve(cluster1, "&sources=cluster1&restrict=type2,type3")); - assertEquals(new LinkedHashSet<>(List.of("type2")), resolve(cluster1, "&sources=cluster2&restrict=type2")); + assertEquals(Set.of("type1"), resolve(cluster1, "&sources=cluster1&restrict=type1")); + assertEquals(Set.of("type2"), resolve(cluster1, "&sources=cluster1&restrict=type2")); + assertEquals(Set.of("type2", "type3"), resolve(cluster1, "&sources=cluster1&restrict=type2,type3")); + assertEquals(Set.of("type2"), resolve(cluster1, "&sources=cluster2&restrict=type2")); } } finally { type1.deconstruct(); @@ -127,11 +129,12 @@ public class ClusterSearcherTestCase { @Test void testThatDocumentTypesAreResolvedTODO_REMOVE() { - ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type1", "type2", "type3"))); + SchemaInfo schemaInfo = createSchemaInfo(); + ClusterSearcher cluster1 = new ClusterSearcher(schemaInfo, Set.of("type1", "type2", "type3")); try { - ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type6"))); + ClusterSearcher type1 = new ClusterSearcher(schemaInfo, Set.of("type6")); try { - assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&sources=cluster2")); + assertEquals(Set.of(), resolve(cluster1, "&sources=cluster2")); } finally { type1.deconstruct(); } @@ -265,7 +268,8 @@ public class ClusterSearcherTestCase { private Execution createExecution(List<String> docTypesList, boolean expectAttributePrefetch) { Set<String> documentTypes = new LinkedHashSet<>(docTypesList); - ClusterSearcher cluster = new ClusterSearcher(documentTypes, + ClusterSearcher cluster = new ClusterSearcher(toSchemaInfo(documentTypes, "mycluster"), + documentTypes, new MyMockSearcher(expectAttributePrefetch), new InThreadExecutorService()); try { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 3e5c4564be0..4ce1cd5a10d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -23,17 +23,21 @@ import com.yahoo.search.grouping.result.Group; import com.yahoo.search.grouping.result.RootGroup; import com.yahoo.search.grouping.result.StringBucketId; import com.yahoo.search.grouping.result.StringId; -import com.yahoo.search.grouping.result.ValueGroupId; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.Relevance; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChain; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * Tests the BlendingSearcher class @@ -46,12 +50,12 @@ public class BlendingSearcherTestCase { private static final double delta = 0.00000001; - public class BlendingSearcherWrapper extends Searcher { + public static class BlendingSearcherWrapper extends Searcher { private SearchChain blendingChain; private final FederationConfig.Builder builder = new FederationConfig.Builder(); private final Map<String, Searcher> searchers = new HashMap<>(); - private SearchChainRegistry chainRegistry = new SearchChainRegistry(); + private final SearchChainRegistry chainRegistry = new SearchChainRegistry(); private final String blendingField; @@ -63,12 +67,6 @@ public class BlendingSearcherTestCase { this.blendingField = blendingField; } - @SuppressWarnings("serial") - public BlendingSearcherWrapper(QrSearchersConfig cfg) { - QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); - blendingField = s.docid().length() > 0 ? s.docid() : null; - } - /** Adds a source implemented as a single searcher */ public void addSource(String sourceName, Searcher searcher) { addSource(createSearchChain(new ComponentId(sourceName), searcher)); @@ -110,7 +108,7 @@ public class BlendingSearcherTestCase { } FederationSearcher fedSearcher = - new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>()); + new FederationSearcher(new FederationConfig(builder), SchemaInfo.empty(), new ComponentRegistry<>()); BlendingSearcher blendingSearcher = new BlendingSearcher(blendingField); blendingChain = new SearchChain(ComponentId.createAnonymousComponentId("blendingChain"), blendingSearcher, fedSearcher); return true; diff --git a/container-search/src/test/java/com/yahoo/search/federation/DuplicateSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/DuplicateSourceTestCase.java index b691d8050dd..514a087bed1 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/DuplicateSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/DuplicateSourceTestCase.java @@ -3,8 +3,6 @@ package com.yahoo.search.federation; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -13,7 +11,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChainRegistry; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -31,20 +28,20 @@ public class DuplicateSourceTestCase { @Test void testDuplicateSource() { + String chain1 = "chain1"; + String schema1 = "doc1", schema2 = "doc2"; // Set up a single cluster and chain (chain1), containing a MockBackendSearcher and having 2 doc types (doc1, doc2) MockBackendSearcher mockBackendSearcher = new MockBackendSearcher(); SearchChainRegistry searchChains = new SearchChainRegistry(); - searchChains.register(new Chain<>("chain1", mockBackendSearcher)); - Map<String, List<String>> clusters = new HashMap<>(); - clusters.put("chain1", List.of("doc1", "doc2")); - IndexFacts indexFacts = new IndexFacts(new IndexModel(clusters, List.of())); + searchChains.register(new Chain<>(chain1, mockBackendSearcher)); SearchChainResolver resolver = new SearchChainResolver.Builder() - .addSearchChain(new ComponentId("chain1"), List.of("doc1", "doc2")) + .addSearchChain(new ComponentId(chain1), List.of(schema1, schema2)) .build(); - FederationSearcher searcher = new FederationSearcher(new ComponentId("test"), resolver); + FederationSearcher searcher = new FederationSearcher(new ComponentId("test"), resolver, + Map.of(schema1, List.of(chain1), schema2, List.of(chain1))); Result result = searcher.search(new Query("?query=test&sources=doc1%2cdoc2"), - new Execution(Execution.Context.createContextStub(searchChains, indexFacts))); + new Execution(Execution.Context.createContextStub(searchChains))); assertNull(result.hits().getError()); assertEquals(1, mockBackendSearcher.getInvocationCount()); diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java index 0d6f78a4bf6..4d3e3c18e0b 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java @@ -16,6 +16,7 @@ import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.Execution.Context; import com.yahoo.search.searchchain.model.federation.FederationOptions; @@ -168,6 +169,7 @@ public class FederationSearcherTest { FederationSearcher searcher = new FederationSearcher( new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + SchemaInfo.empty(), targetSelectors); Query query = new Query(); @@ -186,6 +188,7 @@ public class FederationSearcherTest { FederationSearcher searcher = new FederationSearcher( new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + SchemaInfo.empty(), targetSelectors); Query query = new Query(); diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java index 3c7e122dc7d..af0e0ef69a8 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java @@ -11,6 +11,7 @@ import com.yahoo.search.federation.sourceref.SearchChainResolver; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChain; import com.yahoo.search.searchchain.SearchChainRegistry; @@ -24,6 +25,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @@ -81,7 +83,7 @@ public class FederationSearcherTestCase { } private Searcher createFederationSearcher() { - return new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>()); + return new FederationSearcher(new FederationConfig(builder), SchemaInfo.empty(), new ComponentRegistry<>()); } private SearchChain createSearchChain(ComponentId chainId,Searcher searcher) { @@ -107,7 +109,7 @@ public class FederationSearcherTestCase { @Test void testTraceTwoSources() { - Chain<Searcher> mainChain = twoTracingSources(false); + Chain<Searcher> mainChain = twoTracingSources(); Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1")); @@ -120,7 +122,7 @@ public class FederationSearcherTestCase { assertTrue(lookForTraces.traceFromSource2); } - private Chain<Searcher> twoTracingSources(boolean strictContracts) { + private Chain<Searcher> twoTracingSources() { addChained(new Searcher() { @Override public Result search(Query query, Execution execution) { @@ -140,12 +142,12 @@ public class FederationSearcherTestCase { }, SOURCE2); return new Chain<>("default", - new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>())); + new FederationSearcher(new FederationConfig(builder), SchemaInfo.empty(), new ComponentRegistry<>())); } @Test void testTraceOneSourceNoCloning() { - Chain<Searcher> mainChain = twoTracingSources(true); + Chain<Searcher> mainChain = twoTracingSources(); Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); @@ -160,7 +162,7 @@ public class FederationSearcherTestCase { @Test void testTraceOneSourceWithCloning() { - Chain<Searcher> mainChain = twoTracingSources(false); + Chain<Searcher> mainChain = twoTracingSources(); Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); @@ -282,7 +284,7 @@ public class FederationSearcherTestCase { builder.addSourceForProvider(news, provider1, provider1, true, options, List.of()); builder.addSourceForProvider(news, provider2, provider2, false, options, List.of()); - return new FederationSearcher(new ComponentId("federation"), builder.build()); + return new FederationSearcher(new ComponentId("federation"), builder.build(), Map.of()); } private static class MockProvider extends Searcher { diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationTester.java b/container-search/src/test/java/com/yahoo/search/federation/FederationTester.java index 3991a8d18b7..ec540e43dfd 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationTester.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationTester.java @@ -11,7 +11,8 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.searchchain.model.federation.FederationOptions; -import java.util.Collections; +import java.util.List; +import java.util.Map; /** * @author Tony Vaagenes @@ -30,7 +31,7 @@ class FederationTester { void addSearchChain(String id, FederationOptions federationOptions, Searcher... searchers) { ComponentId searchChainId = ComponentId.fromString(id); - builder.addSearchChain(searchChainId, federationOptions, Collections.<String>emptyList()); + builder.addSearchChain(searchChainId, federationOptions, List.of()); Chain<Searcher> chain = new Chain<>(searchChainId, searchers); registry.register(chain); @@ -46,7 +47,7 @@ class FederationTester { } FederationSearcher buildFederationSearcher() { - return new FederationSearcher(ComponentId.fromString("federation"), builder.build()); + return new FederationSearcher(ComponentId.fromString("federation"), builder.build(), Map.of()); } public Result search() { diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java index d575be603c1..d9046075f38 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref.test; +package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.processing.request.Properties; -import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; -import com.yahoo.search.federation.sourceref.SearchChainResolver; -import com.yahoo.search.federation.sourceref.Target; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; @@ -16,7 +12,9 @@ import java.util.Collections; import java.util.Iterator; import java.util.SortedSet; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Tony Vaagenes diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java index 1b3baebac6f..b32135afc94 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java @@ -1,25 +1,23 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref.test; +package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; -import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; -import com.yahoo.search.federation.sourceref.SearchChainResolver; -import com.yahoo.search.federation.sourceref.SourceRefResolver; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeMap; -import static com.yahoo.search.federation.sourceref.test.SearchChainResolverTestCase.emptySourceToProviderMap; -import static org.junit.jupiter.api.Assertions.*; +import static com.yahoo.search.federation.sourceref.SearchChainResolverTestCase.emptySourceToProviderMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test for SourceRefResolver. @@ -31,35 +29,20 @@ public class SourceRefResolverTestCase { private static final String cluster1 = "cluster1"; private static final String cluster2 = "cluster2"; private static final String cluster3 = "cluster3"; - private static IndexFacts indexFacts; + private static final String schema1 = "document1"; + private static final String schema2 = "document2"; + private static final String schema3 = "document3"; private static final SourceRefResolver sourceRefResolver = createSourceRefResolver(); - static { - setupIndexFacts(); - } - private static SourceRefResolver createSourceRefResolver() { SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); - builder.addSearchChain(ComponentId.fromString(cluster1), new FederationOptions().setUseByDefault(true), - Collections.emptyList()); - builder.addSearchChain(ComponentId.fromString(cluster2), new FederationOptions().setUseByDefault(true), - Collections.emptyList()); - - return new SourceRefResolver(builder.build()); - } + builder.addSearchChain(ComponentId.fromString(cluster1), new FederationOptions().setUseByDefault(true), List.of()); + builder.addSearchChain(ComponentId.fromString(cluster2), new FederationOptions().setUseByDefault(true), List.of()); - private static void setupIndexFacts() { - TreeMap<String, List<String>> masterClusters = new TreeMap<>(); - masterClusters.put(cluster1, List.of("document1", "document2")); - masterClusters.put(cluster2, List.of("document1")); - masterClusters.put(cluster3, List.of("document3")); - indexFacts = new IndexFacts(new IndexModel(masterClusters, Collections.emptyList())); - } - - @Test - void check_test_assumptions() { - assertTrue(indexFacts.clustersHavingSearchDefinition("document1").containsAll(List.of("cluster1", "cluster2"))); + return new SourceRefResolver(builder.build(), Map.of(schema1, List.of(cluster1, cluster2), + schema2, List.of(cluster1), + schema3, List.of(cluster3))); } @Test @@ -107,7 +90,7 @@ public class SourceRefResolverTestCase { } private Set<SearchChainInvocationSpec> resolve(String documentName) throws UnresolvedSearchChainException { - return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap(), indexFacts); + return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap()); } } diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java new file mode 100644 index 00000000000..bf6605a43b2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java @@ -0,0 +1,29 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Test of VirtualSourceResolver + * + * @author baldersheim + */ +public class VirtualSourceResolverTestCase { + @Test + void testThatOriginalIsReturnedIfNoMapping() { + var input = Set.of("a","b", "b.c"); + assertSame(input, VirtualSourceResolver.of().resolve(input)); + assertSame(input, VirtualSourceResolver.of(Set.of("x.a","x.b")).resolve(input)); + } + @Test + void testResolution() { + var input = Set.of("a","b", "b.c"); + assertEquals(Set.of("a.x", "a.y", "b.c", "b.x"), + VirtualSourceResolver.of(Set.of("a.x","a.y", "b.x")).resolve(input)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java index e16c4164fbb..36c80e852d1 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java @@ -17,6 +17,7 @@ import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; import java.util.Collections; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -42,7 +43,7 @@ public class FutureDataTestCase { new SearchChainResolver.Builder().addSearchChain(new ComponentId("sync"), new FederationOptions().setUseByDefault(true)). addSearchChain(new ComponentId("async"), new FederationOptions().setUseByDefault(true)). build(); - Chain<Searcher> main = new Chain<>(new FederationSearcher(new ComponentId("federator"), searchChainResolver)); + Chain<Searcher> main = new Chain<>(new FederationSearcher(new ComponentId("federator"), searchChainResolver, Map.of())); SearchChainRegistry searchChainRegistry = new SearchChainRegistry(); searchChainRegistry.register(main); searchChainRegistry.register(syncSource); diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 68dbb5b8a76..398f2261eec 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -65,7 +65,7 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.657</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.658</aws-sdk.vespa.version> <athenz.vespa.version>1.11.51</athenz.vespa.version> <!-- Athenz END --> @@ -120,7 +120,7 @@ <mimepull.vespa.version>1.10.0</mimepull.vespa.version> <mockito.vespa.version>5.10.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> - <netty.vespa.version>4.1.106.Final</netty.vespa.version> + <netty.vespa.version>4.1.107.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.62.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.16.3</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.2</opennlp.vespa.version> diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 510c6be4bf3..ee7c201f093 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -1278,6 +1278,10 @@ TEST("require that OR blueprint use saturated sum as estimate") { TEST_DO(verify_or_est({{100, false},{300, false},{200, false}}, {300, false})); } +std::vector<FlowStats> child_stats({{0.2, 1.1, 0.2*1.1}, + {0.3, 1.2, 0.3*1.2}, + {0.5, 1.3, 1.3}}); + void verify_relative_estimate(make &&mk, double expect) { EXPECT_EQUAL(mk.making->estimate(), 0.0); Blueprint::UP bp = std::move(mk).leafs({200,300,950}); @@ -1316,8 +1320,10 @@ TEST("relative estimate for ONEAR") { } TEST("relative estimate for WEAKAND") { - verify_relative_estimate(make::WEAKAND(1000), 1.0-0.8*0.7*0.5); - verify_relative_estimate(make::WEAKAND(50), 0.05); + double est1 = (Blueprint::abs_to_rel_est(1000, 1000) + OrFlow::estimate_of(child_stats)) / 2.0; + double est2 = (Blueprint::abs_to_rel_est(50, 1000) + OrFlow::estimate_of(child_stats)) / 2.0; + verify_relative_estimate(make::WEAKAND(1000), est1); + verify_relative_estimate(make::WEAKAND(50), est2); } void verify_cost(make &&mk, double expect, double expect_strict) { @@ -1333,10 +1339,6 @@ void verify_cost(make &&mk, double expect, double expect_strict) { EXPECT_EQUAL(bp->strict_cost(), expect_strict); } -std::vector<FlowStats> child_stats({{0.2, 1.1, 0.2*1.1}, - {0.3, 1.2, 0.3*1.2}, - {0.5, 1.3, 1.3}}); - TEST("cost for OR") { verify_cost(make::OR(), OrFlow::cost_of(child_stats, false), @@ -1377,9 +1379,10 @@ TEST("cost for ONEAR") { } TEST("cost for WEAKAND") { + double est = (Blueprint::abs_to_rel_est(1000, 1000) + OrFlow::estimate_of(child_stats)) / 2.0; verify_cost(make::WEAKAND(1000), OrFlow::cost_of(child_stats, false), - OrFlow::cost_of(child_stats, true)); + OrFlow::cost_of(child_stats, true) + flow::heap_cost(est, 3)); } TEST_MAIN() { TEST_DEBUG("lhs.out", "rhs.out"); TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index f358ee56dd4..8de8f6247c9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -31,6 +31,7 @@ #include <vespa/searchlib/queryeval/matching_elements_search.h> #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/queryeval/orlikesearch.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/queryeval/predicate_blueprint.h> #include <vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h> #include <vespa/searchlib/queryeval/wand/parallel_weak_and_search.h> @@ -229,8 +230,10 @@ struct LocationPreFilterIterator : public OrLikeSearch<is_strict, NoUnpack> class LocationPreFilterBlueprint : public ComplexLeafBlueprint { private: + using AttrHitEstimate = attribute::HitEstimate; const IAttributeVector &_attribute; std::vector<ISearchContext::UP> _rangeSearches; + std::vector<AttrHitEstimate> _estimates; bool _should_use; public: @@ -238,6 +241,7 @@ public: : ComplexLeafBlueprint(field), _attribute(attribute), _rangeSearches(), + _estimates(), _should_use(false) { uint64_t estHits(0); @@ -247,7 +251,8 @@ public: query::SimpleRangeTerm rt(qr, "", 0, query::Weight(0)); string stack(StackDumpCreator::create(rt)); _rangeSearches.push_back(attr.createSearchContext(QueryTermDecoder::decodeTerm(stack), scParams)); - estHits += _rangeSearches.back()->calc_hit_estimate().est_hits(); + _estimates.push_back(_rangeSearches.back()->calc_hit_estimate()); + estHits += _estimates.back().est_hits(); LOG(debug, "Range '%s' estHits %" PRId64, qr.getRangeString().c_str(), estHits); } if (estHits > attr.getNumDocs()) { @@ -266,9 +271,23 @@ public: bool should_use() const { return _should_use; } queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - return default_flow_stats(docid_limit, getState().estimate().estHits, _rangeSearches.size()); + using OrFlow = search::queryeval::OrFlow; + struct MyAdapter { + uint32_t docid_limit; + MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + double estimate(const AttrHitEstimate &est) const noexcept { + return est.is_unknown() ? 0.5 : abs_to_rel_est(est.est_hits(), docid_limit); + } + double cost(const AttrHitEstimate &) const noexcept { return 1.0; } + double strict_cost(const AttrHitEstimate &est) const noexcept { + return est.is_unknown() ? 1.0 : abs_to_rel_est(est.est_hits(), docid_limit); + } + }; + double est = OrFlow::estimate_of(MyAdapter(docid_limit), _estimates); + return {est, OrFlow::cost_of(MyAdapter(docid_limit), _estimates, false), + OrFlow::cost_of(MyAdapter(docid_limit), _estimates, true) + queryeval::flow::array_cost(est, _estimates.size())}; } - + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { @@ -458,7 +477,23 @@ public: } queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - return default_flow_stats(docid_limit, getState().estimate().estHits, _terms.size()); + using OrFlow = search::queryeval::OrFlow; + struct MyAdapter { + uint32_t docid_limit; + MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { + return abs_to_rel_est(term.posting_size, docid_limit); + } + double cost(const IDirectPostingStore::LookupResult &) const noexcept { return 1.0; } + double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { + return abs_to_rel_est(term.posting_size, docid_limit); + } + }; + double child_est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); + double my_est = abs_to_rel_est(_scores.getScoresToTrack(), docid_limit); + double est = (child_est + my_est) / 2.0; + return {est, OrFlow::cost_of(MyAdapter(docid_limit), _terms, false), + OrFlow::cost_of(MyAdapter(docid_limit), _terms, true) + queryeval::flow::heap_cost(est, _terms.size())}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index ede2ecd0ad5..18748641ca4 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -8,6 +8,8 @@ #include <vespa/searchlib/query/query_term_ucs4.h> #include <vespa/searchlib/queryeval/filter_wrapper.h> #include <vespa/searchlib/queryeval/orsearch.h> +#include <vespa/searchlib/queryeval/flow.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/queryeval/weighted_set_term_search.h> #include <vespa/vespalib/objects/visit.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -20,6 +22,7 @@ namespace { using attribute::ISearchContext; using attribute::IAttributeVector; +using queryeval::OrFlow; class AttrWrapper { @@ -94,7 +97,8 @@ AttributeWeightedSetBlueprint::AttributeWeightedSetBlueprint(const queryeval::Fi _estHits(0), _weights(), _attr(attr), - _contexts() + _contexts(), + _estimates() { set_allow_termwise_eval(true); } @@ -110,7 +114,8 @@ AttributeWeightedSetBlueprint::~AttributeWeightedSetBlueprint() void AttributeWeightedSetBlueprint::addToken(std::unique_ptr<ISearchContext> context, int32_t weight) { - _estHits = std::min(_estHits + context->calc_hit_estimate().est_hits(), _numDocs); + _estimates.push_back(context->calc_hit_estimate()); + _estHits = std::min(_estHits + _estimates.back().est_hits(), _numDocs); setEstimate(HitEstimate(_estHits, (_estHits == 0))); _weights.push_back(weight); _contexts.push_back(context.release()); @@ -119,7 +124,20 @@ AttributeWeightedSetBlueprint::addToken(std::unique_ptr<ISearchContext> context, queryeval::FlowStats AttributeWeightedSetBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, _estHits, _weights.size()); + struct MyAdapter { + uint32_t docid_limit; + MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + double estimate(const AttrHitEstimate &est) const noexcept { + return est.is_unknown() ? 0.5 : abs_to_rel_est(est.est_hits(), docid_limit); + } + double cost(const AttrHitEstimate &) const noexcept { return 1.0; } + double strict_cost(const AttrHitEstimate &est) const noexcept { + return est.is_unknown() ? 1.0 : abs_to_rel_est(est.est_hits(), docid_limit); + } + }; + double est = OrFlow::estimate_of(MyAdapter(docid_limit), _estimates); + return {est, OrFlow::cost_of(MyAdapter(docid_limit), _estimates, false), + OrFlow::cost_of(MyAdapter(docid_limit), _estimates, true) + queryeval::flow::heap_cost(est, _estimates.size())}; } queryeval::SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h index 16319654024..32632403e42 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h @@ -5,6 +5,7 @@ #include "attributeguard.h" #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchcommon/attribute/hit_estimate.h> #include <vector> namespace search { @@ -16,12 +17,14 @@ class AttributeWeightedSetBlueprint : public queryeval::ComplexLeafBlueprint private: using ISearchContext = attribute::ISearchContext; using IAttributeVector = attribute::IAttributeVector; + using AttrHitEstimate = attribute::HitEstimate; size_t _numDocs; size_t _estHits; std::vector<int32_t> _weights; const IAttributeVector & _attr; std::vector<ISearchContext*> _contexts; - + std::vector<AttrHitEstimate> _estimates; + public: AttributeWeightedSetBlueprint(const AttributeWeightedSetBlueprint &) = delete; AttributeWeightedSetBlueprint &operator=(const AttributeWeightedSetBlueprint &) = delete; diff --git a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h index e0b206dbdd9..076c375091a 100644 --- a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h @@ -8,6 +8,7 @@ #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/queryeval/field_spec.h> #include <vespa/searchlib/queryeval/matching_elements_search.h> #include <variant> @@ -72,7 +73,21 @@ public: } queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - return default_flow_stats(docid_limit, getState().estimate().estHits, _terms.size()); + using OrFlow = search::queryeval::OrFlow; + struct MyAdapter { + uint32_t docid_limit; + MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { + return abs_to_rel_est(term.posting_size, docid_limit); + } + double cost(const IDirectPostingStore::LookupResult &) const noexcept { return 1.0; } + double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { + return abs_to_rel_est(term.posting_size, docid_limit); + } + }; + double est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); + return {est, OrFlow::cost_of(MyAdapter(docid_limit), _terms, false), + OrFlow::cost_of(MyAdapter(docid_limit), _terms, true) + queryeval::flow::heap_cost(est, _terms.size())}; } std::unique_ptr<queryeval::SearchIterator> createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 3b8975f9883..7cd00d02bb3 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -2,6 +2,7 @@ #include "dot_product_blueprint.h" #include "dot_product_search.h" +#include "flow_tuning.h" #include "field_spec.hpp" #include <vespa/vespalib/objects/visit.hpp> @@ -41,7 +42,12 @@ DotProductBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimate & e FlowStats DotProductBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, getState().estimate().estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double est = OrFlow::estimate_of(_terms); + return {est, OrFlow::cost_of(_terms, false), + OrFlow::cost_of(_terms, true) + flow::heap_cost(est, _terms.size())}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp index 84745fdd0bd..27624abf515 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp @@ -3,6 +3,7 @@ #include "equiv_blueprint.h" #include "equivsearch.h" #include "field_spec.hpp" +#include "flow_tuning.h" #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -55,7 +56,12 @@ EquivBlueprint::~EquivBlueprint() = default; FlowStats EquivBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, _estimate.estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double est = OrFlow::estimate_of(_terms); + return {est, OrFlow::cost_of(_terms, false), + OrFlow::cost_of(_terms, true) + flow::array_cost(est, _terms.size())}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index ab0381fa12c..491f0ad5571 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -10,4 +10,8 @@ inline double heap_cost(double my_est, size_t num_children) { return my_est * std::log2(std::max(size_t(1),num_children)); } +inline double array_cost(double my_est, size_t num_children) { + return my_est * num_children; +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 938044c0c19..993639becf2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -421,9 +421,10 @@ FlowStats WeakAndBlueprint::calculate_flow_stats(uint32_t docid_limit) const { double child_est = OrFlow::estimate_of(get_children()); double my_est = abs_to_rel_est(_n, docid_limit); - return {std::min(my_est, child_est), + double est = (child_est + my_est) / 2.0; + return {est, OrFlow::cost_of(get_children(), false), - OrFlow::cost_of(get_children(), true)}; + OrFlow::cost_of(get_children(), true) + flow::heap_cost(est, get_children().size())}; } Blueprint::HitEstimate diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 3d4f917aeac..628020cfea2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -47,7 +47,13 @@ SameElementBlueprint::addTerm(Blueprint::UP term) FlowStats SameElementBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, _estimate.estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double est = AndFlow::estimate_of(_terms); + return {est, + AndFlow::cost_of(_terms, false) + est * _terms.size(), + AndFlow::cost_of(_terms, true) + est * _terms.size()}; } void diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index ace7d12c32b..953e7350074 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -48,7 +48,13 @@ SimplePhraseBlueprint::addTerm(Blueprint::UP term) FlowStats SimplePhraseBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, _estimate.estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double est = AndFlow::estimate_of(_terms); + return {est, + AndFlow::cost_of(_terms, false) + est * _terms.size(), + AndFlow::cost_of(_terms, true) + est * _terms.size()}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 4e400b3c055..78fe3882aab 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -5,6 +5,7 @@ #include "parallel_weak_and_search.h" #include <vespa/searchlib/queryeval/field_spec.hpp> #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> @@ -68,7 +69,14 @@ ParallelWeakAndBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimat FlowStats ParallelWeakAndBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, getState().estimate().estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double child_est = OrFlow::estimate_of(_terms); + double my_est = abs_to_rel_est(_scores.getScoresToTrack(), docid_limit); + double est = (child_est + my_est) / 2.0; + return {est, OrFlow::cost_of(_terms, false), + OrFlow::cost_of(_terms, true) + flow::heap_cost(est, _terms.size())}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index b3bbec8428f..8bf2dd53470 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -4,6 +4,7 @@ #include "weighted_set_term_search.h" #include "orsearch.h" #include "matching_elements_search.h" +#include "flow_tuning.h" #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/objects/visit.hpp> @@ -95,7 +96,12 @@ WeightedSetTermBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimat FlowStats WeightedSetTermBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - return default_flow_stats(docid_limit, getState().estimate().estHits, _terms.size()); + for (auto &term: _terms) { + term->update_flow_stats(docid_limit); + } + double est = OrFlow::estimate_of(_terms); + return {est, OrFlow::cost_of(_terms, false), + OrFlow::cost_of(_terms, true) + flow::heap_cost(est, _terms.size())}; } SearchIterator::UP diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index 75eccaef83c..15017fa3334 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -190,6 +190,30 @@ HitCollector::getFeatureSet(IRankProgram &rankProgram, return retval; } +FeatureSet::SP +HitCollector::getFeatureSet(IRankProgram &rankProgram, + search::DocumentIdT docId, + const FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map) +{ + LOG(debug, "docId = %d, _hits.size = %zu", docId, _hits.size()); + if (resolver.num_features() == 0 || _hits.empty()) { + return std::make_shared<FeatureSet>(); + } + auto names = FefUtils::extract_feature_names(resolver, feature_rename_map); + FeatureSet::SP retval = std::make_shared<FeatureSet>(names, _hits.size()); + for (const Hit & hit : _hits) { + LOG(debug, "Checking docId=%d", hit.getDocId()); + if (docId == hit.getDocId()) { + rankProgram.run(docId, hit.getMatchData()); + auto *f = retval->getFeaturesByIndex(retval->addDocId(docId)); + FefUtils::extract_feature_values(resolver, docId, f); + return retval; + } + } + return retval; +} + FeatureValues HitCollector::get_match_features(IRankProgram& rank_program, const FeatureResolver& resolver, diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index 3a05f9dca86..f3d1b68aa3f 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -153,6 +153,11 @@ public: const FeatureResolver &resolver, const search::StringStringMap &feature_rename_map) const; + vespalib::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, + search::DocumentIdT docId, + const FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map); + vespalib::FeatureValues get_match_features(IRankProgram& rank_program, const FeatureResolver& resolver, const search::StringStringMap& feature_rename_map) const; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 167d5ecde4c..ac7a61e5888 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -256,6 +256,19 @@ RankProcessor::calculateFeatureSet() return sf; } +FeatureSet::SP +RankProcessor::calculateFeatureSet(search::DocumentIdT docId) +{ + LOG(debug, "Calculate feature set for docId = %d", docId); + RankProgram &rankProgram = *(_summaryProgram ? _summaryProgram : _rankProgram); + search::fef::FeatureResolver resolver(rankProgram.get_seeds(false)); + LOG(debug, "Feature handles: numNames(%ld)", resolver.num_features()); + RankProgramWrapper wrapper(*_match_data); + FeatureSet::SP sf = _hitCollector->getFeatureSet(wrapper, docId, resolver, _rankSetup.get_feature_rename_map()); + LOG(debug, "Feature set: numFeatures(%u), numDocs(%u)", sf->numFeatures(), sf->numDocs()); + return sf; +} + FeatureValues RankProcessor::calculate_match_features() { diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index b9ed07f1170..476ed013d23 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -77,6 +77,7 @@ public: static void unpack_match_data(uint32_t docid, search::fef::MatchData& matchData, QueryWrapper& query); void runRankProgram(uint32_t docId); vespalib::FeatureSet::SP calculateFeatureSet(); + vespalib::FeatureSet::SP calculateFeatureSet(search::DocumentIdT docId); void fillSearchResult(vdslib::SearchResult & searchResult); const search::fef::MatchData &getMatchData() const { return *_match_data; } void setRankScore(double score) { _score = score; } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 9a0b720f054..28a2a521cf7 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -288,6 +288,7 @@ SearchVisitor::SearchVisitor(StorageComponent& component, _env(get_search_environment_snapshot(vEnv, params)), _params(params), _init_called(false), + _collectGroupingHits(false), _docSearchedCount(0), _hitCount(0), _hitsRejectedCount(0), @@ -753,6 +754,14 @@ SearchVisitor::RankController::collectMatchedDocument(bool hasSorting, } } +vespalib::FeatureSet::SP +SearchVisitor::RankController::getFeatureSet(search::DocumentIdT docId) { + if (_hasRanking && !_rankSetup->getSummaryFeatures().empty()) { + return _rankProcessor->calculateFeatureSet(docId); + } + return {}; +} + void SearchVisitor::RankController::onCompletedVisiting(vsm::GetDocsumsStateCallback & docsumsStateCallback, vdslib::SearchResult & searchResult) { @@ -763,15 +772,13 @@ SearchVisitor::RankController::onCompletedVisiting(vsm::GetDocsumsStateCallback // calculate summary features and set them on the callback object if (!_rankSetup->getSummaryFeatures().empty()) { LOG(debug, "Calculate summary features"); - vespalib::FeatureSet::SP sf = _rankProcessor->calculateFeatureSet(); - docsumsStateCallback.setSummaryFeatures(sf); + docsumsStateCallback.setSummaryFeatures(_rankProcessor->calculateFeatureSet()); } // calculate rank features and set them on the callback object if (_dumpFeatures) { LOG(debug, "Calculate rank features"); - vespalib::FeatureSet::SP rf = _dumpProcessor->calculateFeatureSet(); - docsumsStateCallback.setRankFeatures(rf); + docsumsStateCallback.setRankFeatures(_dumpProcessor->calculateFeatureSet()); } } } @@ -996,6 +1003,9 @@ SearchVisitor::setupGrouping(const std::vector<char> & groupingBlob) grouping.configureStaticStuff(stuff); HitsResultPreparator preparator(_summaryGenerator); grouping.select(preparator, preparator); + if (preparator.getNumHitsAggregators() > 0) { + _collectGroupingHits = true; + } grouping.preAggregate(false); if (!grouping.getAll() || (preparator.getNumHitsAggregators() == 0)) { _groupingList.push_back(groupingPtr); @@ -1082,6 +1092,9 @@ SearchVisitor::handleDocument(StorageDocument::SP documentSP) _syntheticFieldsController.onDocumentMatch(document, documentId); SingleDocumentStore single(document); _summaryGenerator.setDocsumCache(single); + if (_collectGroupingHits) { + _summaryGenerator.getDocsumCallback().setSummaryFeatures(_rankController.getFeatureSet(document.getDocId())); + } group(document.docDoc(), rp.getRankScore(), false); } else { _hitsRejectedCount++; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index d8d97830244..33d5a14084f 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -210,6 +210,7 @@ private: * @param docsumsStateCallback state object to store summary features and rank features. **/ void onCompletedVisiting(vsm::GetDocsumsStateCallback & docsumsStateCallback, vdslib::SearchResult & searchResult); + vespalib::FeatureSet::SP getFeatureSet(search::DocumentIdT docId); }; /** @@ -455,6 +456,7 @@ private: std::shared_ptr<const SearchEnvironmentSnapshot> _env; vdslib::Parameters _params; bool _init_called; + bool _collectGroupingHits; size_t _docSearchedCount; size_t _hitCount; size_t _hitsRejectedCount; diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index d3b3abd5fbc..a135f1d9fa2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -37,8 +37,8 @@ public: void fillSummaryFeatures(GetDocsumsState& state) override; void fillRankFeatures(GetDocsumsState& state) override; std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; - void setSummaryFeatures(const vespalib::FeatureSet::SP & sf) { _summaryFeatures = sf; } - void setRankFeatures(const vespalib::FeatureSet::SP & rf) { _rankFeatures = rf; } + void setSummaryFeatures(vespalib::FeatureSet::SP sf) { _summaryFeatures = std::move(sf); } + void setRankFeatures(vespalib::FeatureSet::SP rf) { _rankFeatures = std::move(rf); } void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler); ~GetDocsumsStateCallback() override; }; |