diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
10 files changed, 122 insertions, 59 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index e5393fe9b85..f40caac1562 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -11,7 +11,6 @@ import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.errorhandling.Results; import com.yahoo.errorhandling.Results.Builder; -import com.yahoo.prelude.IndexFacts; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -25,11 +24,14 @@ import com.yahoo.search.federation.sourceref.SingleTarget; import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.federation.sourceref.VirtualSourceResolver; import com.yahoo.search.query.Properties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.HitOrderer; +import com.yahoo.search.schema.Cluster; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.AsyncExecution; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.ForkingSearcher; @@ -41,6 +43,7 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.LinkedHashSet; @@ -77,25 +80,34 @@ public class FederationSearcher extends ForkingSearcher { private final SearchChainResolver searchChainResolver; private final SourceRefResolver sourceRefResolver; + private final VirtualSourceResolver virtualSourceResolver; private final TargetSelector<?> targetSelector; private final Clock clock = Clock.systemUTC(); @Inject - public FederationSearcher(FederationConfig config, ComponentRegistry<TargetSelector> targetSelectors) { - this(createResolver(config), resolveSelector(config.targetSelector(), targetSelectors)); + public FederationSearcher(FederationConfig config, SchemaInfo schemaInfo, + ComponentRegistry<TargetSelector> targetSelectors) { + this(createResolver(config), + VirtualSourceResolver.of(config), + resolveSelector(config.targetSelector(), targetSelectors), + createSchema2Clusters(schemaInfo)); } // for testing - public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, null); + public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver, + Map<String, List<String>> schema2Clusters) { + this(searchChainResolver, VirtualSourceResolver.of(), null, schema2Clusters); } private FederationSearcher(SearchChainResolver searchChainResolver, - TargetSelector targetSelector) { + VirtualSourceResolver virtualSourceResolver, + TargetSelector targetSelector, + Map<String, List<String>> schema2Clusters) { this.searchChainResolver = searchChainResolver; - sourceRefResolver = new SourceRefResolver(searchChainResolver); + sourceRefResolver = new SourceRefResolver(searchChainResolver, schema2Clusters); this.targetSelector = targetSelector; + this.virtualSourceResolver = virtualSourceResolver; } private static TargetSelector resolveSelector(String selectorId, @@ -105,6 +117,16 @@ public class FederationSearcher extends ForkingSearcher { "Missing target selector with id '" + selectorId + "'"); } + private static Map<String, List<String>> createSchema2Clusters(SchemaInfo schemaInfo) { + Map<String, List<String>> schema2Clusters = new HashMap<>(); + for (Cluster cluster : schemaInfo.clusters().values()) { + for (String schema : cluster.schemas()) { + schema2Clusters.computeIfAbsent(schema, key -> new ArrayList<>()).add(cluster.name()); + } + } + return schema2Clusters; + } + private static SearchChainResolver createResolver(FederationConfig config) { SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); @@ -160,7 +182,7 @@ public class FederationSearcher extends ForkingSearcher { Result mergedResults = execution.search(query); Results<SearchChainInvocationSpec, UnresolvedSearchChainException> targets = - getTargets(query.getModel().getSources(), query.properties(), execution.context().getIndexFacts()); + getTargets(query.getModel().getSources(), query.properties()); warnIfUnresolvedSearchChains(targets.errors(), mergedResults.hits()); Collection<SearchChainInvocationSpec> prunedTargets = @@ -277,7 +299,7 @@ public class FederationSearcher extends ForkingSearcher { return descriptions; } - private Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { + private static Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { Set<String> messages = new LinkedHashSet<>(); for (UnresolvedSearchChainException exception : unresolvedSearchChainExceptions) { messages.add(exception.getMessage()); @@ -370,9 +392,9 @@ public class FederationSearcher extends ForkingSearcher { else { // fill timed out: Remove these hits as they are incomplete and may cause a race when accessed later result.hits().addError(futureFilledResult.getSecond().createTimeoutError()); for (Iterator<Hit> i = futureFilledResult.getFirst().hits().unorderedDeepIterator(); i.hasNext(); ) { - // Note that some of these hits may be filled, but as the fill thread may still be working on them - // and we do not synchronize with it we need to discard all - Hit removed = result.hits().remove(i.next().getId()); + // Note that some of these hits may be filled, but as the fill thread may still be working on them, + // and we do not synchronize with it, we need to discard all. + result.hits().remove(i.next().getId()); } } } @@ -421,18 +443,20 @@ public class FederationSearcher extends ForkingSearcher { return orderer; } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties, IndexFacts indexFacts) { + private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties) { return sources.isEmpty() ? defaultSearchChains(properties): - resolveSources(sources, properties, indexFacts); + resolveSources(sources, properties); } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sources, Properties properties, IndexFacts indexFacts) { + + private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sourcesInQuery, Properties properties) { Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + Set<String> sources = virtualSourceResolver.resolve(sourcesInQuery); for (String source : sources) { try { - result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties, indexFacts)); + result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties)); } catch (UnresolvedSearchChainException e) { result.addError(e); } @@ -578,11 +602,7 @@ public class FederationSearcher extends ForkingSearcher { /** Returns a result to fill for a query and chain, by creating it if necessary */ public Result get(Chain<Searcher> chain, Query query) { - Map<Query,Result> resultsToFillForAChain = resultsToFill.get(chain); - if (resultsToFillForAChain == null) { - resultsToFillForAChain = new IdentityHashMap<>(); - resultsToFill.put(chain,resultsToFillForAChain); - } + Map<Query, Result> resultsToFillForAChain = resultsToFill.computeIfAbsent(chain, k -> new IdentityHashMap<>()); Result resultsToFillForAChainAndQuery = resultsToFillForAChain.get(query); if (resultsToFillForAChainAndQuery == null) { @@ -686,27 +706,18 @@ public class FederationSearcher extends ForkingSearcher { } - private static class Window { - - private final int hits; - private final int offset; - - public Window(int hits, int offset) { - this.hits = hits; - this.offset = offset; - } + private record Window(int hits, int offset) { public Integer get(CompoundName parameterName) { if (parameterName.equals(Query.HITS)) return hits; if (parameterName.equals(Query.OFFSET)) return offset; return null; } - + public static Window from(Query query) { return new Window(query.getHits(), query.getOffset()); } - public static Window from(Collection<Target> targets, Query query) { if (targets.size() == 1) // preserve requested top-level offsets return Window.from(query); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index bfb5bf1a9ab..df91b968750 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -56,12 +56,8 @@ public class SearchChainResolver { } }; - public Builder addSearchChain(ComponentId searchChainId) { - return addSearchChain(searchChainId, Collections.<String>emptyList()); - } - public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions) { - return addSearchChain(searchChainId, federationOptions, Collections.<String>emptyList()); + return addSearchChain(searchChainId, federationOptions, List.of()); } public Builder addSearchChain(ComponentId searchChainId, List<String> documentTypes) { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java index 7345868cae7..2e7849dd85a 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java @@ -2,11 +2,11 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; import com.yahoo.processing.request.Properties; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -17,30 +17,30 @@ import java.util.Set; public class SourceRefResolver { private final SearchChainResolver searchChainResolver; + private final Map<String, List<String>> schema2Clusters; - public SourceRefResolver(SearchChainResolver searchChainResolver) { + public SourceRefResolver(SearchChainResolver searchChainResolver, Map<String, List<String>> schema2Clusters) { this.searchChainResolver = searchChainResolver; + this.schema2Clusters = schema2Clusters; } public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, - Properties sourceToProviderMap, - IndexFacts indexFacts) throws UnresolvedSearchChainException { + Properties sourceToProviderMap) throws UnresolvedSearchChainException { try { - return new LinkedHashSet<>(List.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap))); + return Set.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap)); } catch (UnresolvedSourceRefException e) { - return resolveClustersWithDocument(sourceRef, sourceToProviderMap, indexFacts); + return resolveClustersWithDocument(sourceRef, sourceToProviderMap); } } private Set<SearchChainInvocationSpec> resolveClustersWithDocument(ComponentSpecification sourceRef, - Properties sourceToProviderMap, - IndexFacts indexFacts) + Properties sourceToProviderMap) throws UnresolvedSearchChainException { if (hasOnlyName(sourceRef)) { Set<SearchChainInvocationSpec> clusterSearchChains = new LinkedHashSet<>(); - List<String> clusters = indexFacts.clustersHavingSearchDefinition(sourceRef.getName()); + List<String> clusters = schema2Clusters.getOrDefault(sourceRef.getName(), List.of()); for (String cluster : clusters) { clusterSearchChains.add(resolveClusterSearchChain(cluster, sourceRef, sourceToProviderMap)); } @@ -48,9 +48,7 @@ public class SourceRefResolver { if ( ! clusterSearchChains.isEmpty()) return clusterSearchChains; } - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); - } private SearchChainInvocationSpec resolveClusterSearchChain(String cluster, diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java index 54b022e0b97..b6d99758c7b 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.federation.sourceref; - import com.google.common.base.Joiner; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java index 3cf2776259c..0c8562e6032 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java @@ -5,7 +5,6 @@ package com.yahoo.search.federation.sourceref; * Thrown if a search chain can not be resolved from one or more ids. * @author Tony Vaagenes */ -@SuppressWarnings("serial") public class UnresolvedSearchChainException extends Exception { public UnresolvedSearchChainException(String msg) { super(msg); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java index a7da7a7ee04..fa2c1da13f0 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java @@ -6,7 +6,6 @@ import com.yahoo.component.ComponentSpecification; /** * @author Tony Vaagenes */ -@SuppressWarnings("serial") class UnresolvedSourceRefException extends UnresolvedSearchChainException { UnresolvedSourceRefException(String msg) { super(msg); diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java new file mode 100644 index 00000000000..fc07d12d429 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java @@ -0,0 +1,65 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +import com.yahoo.search.federation.FederationConfig; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Multiple sources like contentcluster.schema1, contencluster.schema2 needs to respond + * when source is the prefix contentcluster. This is done by generating map from virtual source + * to the fully qualified ones, and resolving from there. + * + * @author baldersheim + */ +public class VirtualSourceResolver { + private final Map<String, Set<String>> virtualSources; + private VirtualSourceResolver(Map<String, Set<String>> virtualSources) { + this.virtualSources = virtualSources; + } + public static VirtualSourceResolver of() { + return new VirtualSourceResolver(Map.of()); + } + public static VirtualSourceResolver of(Set<String> targets) { + return new VirtualSourceResolver(createVirtualSources(targets)); + } + private static Map<String, Set<String>> createVirtualSources(Set<String> targets) { + Set<String> virtualSources = targets.stream() + .filter(id -> id.contains(".")) + .map(id -> id.substring(0, id.indexOf('.'))) + .collect(Collectors.toUnmodifiableSet()); + if (virtualSources.isEmpty()) return Map.of(); + Map<String, Set<String>> virtualSourceMap = new HashMap<>(); + for (String virtualSource : virtualSources) { + String prefix = virtualSource + "."; + Set<String> sources = targets.stream() + .filter(id -> id.startsWith(prefix)) + .collect(Collectors.toUnmodifiableSet()); + virtualSourceMap.put(virtualSource, sources); + } + return virtualSourceMap; + } + public static VirtualSourceResolver of(FederationConfig config) { + return of(config.target().stream().map(FederationConfig.Target::id).collect(Collectors.toUnmodifiableSet())); + } + public Set<String> resolve(Set<String> sourcesInQuery) { + boolean hasMapping = sourcesInQuery.stream().anyMatch(virtualSources::containsKey); + if (hasMapping) { + Set<String> resolved = new HashSet<>(); + for (String source : sourcesInQuery) { + var subSources = virtualSources.get(source); + if (subSources != null) { + resolved.addAll(subSources); + } else { + resolved.add(source); + } + } + return resolved; + } + return sourcesInQuery; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java index 86684603a45..4cb19bff740 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java @@ -33,10 +33,6 @@ public class DefaultPositionSearcher extends Searcher { this.useV8GeoPositions = cfg.usev8geopositions(); } - DefaultPositionSearcher() { - this.useV8GeoPositions = false; - } - @Override public com.yahoo.search.Result search(Query query, Execution execution) { Location location = query.getRanking().getLocation(); diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index bbd303039cf..263fa4058c7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -5,11 +5,9 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.component.annotation.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.Query; -import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -65,7 +63,7 @@ public class SchemaInfo { /** Returns all schemas configured in this application, indexed by schema name. */ public Map<String, Schema> schemas() { return schemas; } - /** Returns information about all clusters available for searching in this applications, indexed by cluyster name. */ + /** Returns information about all clusters available for searching in this application, indexed by cluster name. */ public Map<String, Cluster> clusters() { return clusters; } public Session newSession(Query query) { @@ -103,6 +101,8 @@ public class SchemaInfo { /** Returns true if this only searches streaming clusters. */ public boolean isStreaming() { return isStreaming; } + public Collection<Schema> schemas() { return schemas; } + /** * Looks up a field or field set by the given name or alias * in the schemas resolved for this query. diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 84cf1744e27..b70f5145e56 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -16,7 +16,7 @@ import java.util.List; class SchemaInfoConfigurer { static List<Schema> toSchemas(SchemaInfoConfig schemaInfoConfig) { - return schemaInfoConfig.schema().stream().map(config -> toSchema(config)).toList(); + return schemaInfoConfig.schema().stream().map(SchemaInfoConfigurer::toSchema).toList(); } static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { |