diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo')
21 files changed, 233 insertions, 272 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index f481d58d334..855a524473d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; +import com.yahoo.collections.TinyIdentitySet; import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; @@ -10,9 +11,8 @@ import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; -import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.IndexedBackend; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -24,14 +24,17 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.Cluster; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.streamingvisitors.StreamingSearcher; +import com.yahoo.vespa.streamingvisitors.StreamingBackend; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -40,8 +43,6 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; import java.util.stream.Collectors; -import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; - /** * A searcher which forwards to a cluster of monitored native Vespa backends. * @@ -58,13 +59,12 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; // The set of document types contained in this search cluster - private final Set<String> schemas; + private final Map<String, VespaBackend> schema2Searcher; private final SchemaInfo schemaInfo; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds - private final VespaBackEndSearcher server; private final Executor executor; private final GlobalPhaseRanker globalPhaseRanker; @@ -86,26 +86,28 @@ public class ClusterSearcher extends Searcher { searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = globalPhaseRanker; - schemas = new LinkedHashSet<>(); + schema2Searcher = new LinkedHashMap<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); - SummaryParameters docSumParams = new SummaryParameters(qrsConfig - .com().yahoo().prelude().fastsearch().FastSearcher().docsum() - .defaultclass()); - - for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) - schemas.add(docDb.name()); - - String uniqueServerId = UUID.randomUUID().toString(); - if (searchClusterConfig.indexingmode() == STREAMING) { - server = streamingCluster(uniqueServerId, searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); - vipStatus.addToRotation(server.getName()); - } else { - server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, - docSumParams, documentDbConfig, schemaInfo, dispatchers); + VespaBackend streaming = null, indexed = null; + ClusterParams clusterParams = makeClusterParams(searchClusterIndex, qrsConfig + .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() + .defaultclass(), documentDbConfig, schemaInfo); + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { + if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { + if (indexed == null) { + indexed = searchDispatch(clusterParams, searchClusterName, dispatchers); + } + schema2Searcher.put(docDb.name(), indexed); + } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) { + if (streaming == null) { + streaming = streamingCluster(clusterParams, searchClusterConfig, access); + vipStatus.addToRotation(streaming.getName()); + } + schema2Searcher.put(docDb.name(), streaming); + } } } @@ -115,91 +117,88 @@ public class ClusterSearcher extends Searcher { return searchCluster; } } - return null; + throw new IllegalStateException("No configured search cluster '" + name + "' among : " + + config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList()); } - private static ClusterParams makeClusterParams(int searchclusterIndex) { - return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); + private static ClusterParams makeClusterParams(int searchclusterIndex, String defaultSummary, + DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo) + { + return new ClusterParams("sc" + searchclusterIndex + ".num" + 0, UUID.randomUUID().toString(), + defaultSummary, documentDbConfig, schemaInfo); } - private static FastSearcher searchDispatch(int searchclusterIndex, - String searchClusterName, - String serverId, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - ComponentRegistry<Dispatcher> dispatchers) { - ClusterParams clusterParams = makeClusterParams(searchclusterIndex); + private static IndexedBackend searchDispatch(ClusterParams clusterParams, + String searchClusterName, + ComponentRegistry<Dispatcher> dispatchers) + { ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName); Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId); if (dispatcher == null) - throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + - " is configured"); - return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured"); + return new IndexedBackend(clusterParams, dispatcher); } - private static StreamingSearcher streamingCluster(String serverId, - int searchclusterIndex, - QrSearchersConfig.Searchcluster searchClusterConfig, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - VespaDocumentAccess access) { - if (searchClusterConfig.searchdef().size() != 1) - throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " + - searchClusterConfig.searchdef()); - ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - StreamingSearcher searcher = new StreamingSearcher(access); - searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid()); - searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); - searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); - return searcher; + private static StreamingBackend streamingCluster(ClusterParams clusterParams, + QrSearchersConfig.Searchcluster searchClusterConfig, + VespaDocumentAccess access) + { + return new StreamingBackend(clusterParams, searchClusterConfig.rankprofiles_configid(), + access, searchClusterConfig.storagecluster().routespec()); } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { + ClusterSearcher(SchemaInfo schemaInfo, Map<String, VespaBackend> schema2Searcher, Executor executor) { this.schemaInfo = schemaInfo; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; - server = searcher; this.executor = executor; this.globalPhaseRanker = null; - this.schemas = schemas; + this.schema2Searcher = schema2Searcher; } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(SchemaInfo schemaInfo, Set<String> schemas) { - this(schemaInfo, schemas, null, null); + ClusterSearcher(SchemaInfo schemaInfo, Map<String, VespaBackend> schema2Searcher) { + this(schemaInfo, schema2Searcher, null); } @Override public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - Searcher searcher = server; - if (searcher == null) { + if (schema2Searcher.isEmpty()) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } if (query.getTimeLeft() <= 0) { return new Result(query, ErrorMessage.createTimeout("No time left for searching")); } - return doSearch(searcher, query, execution); + return doSearch(query); } @Override - public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { + public void fill(Result result, String summaryClass, Execution execution) { + fill(result, summaryClass); + } + private void fill(Result result, String summaryClass) { Query query = result.getQuery(); - - Searcher searcher = server; - if (searcher != null) { - if (query.getTimeLeft() > 0) { - searcher.fill(result, summaryClass, execution); - } else { - if (result.hits().getErrorHit() == null) { - result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + - query.getTimeout() + " ms")); + var restrict = query.getModel().getRestrict(); + Collection<VespaBackend> servers = (restrict != null && ! restrict.isEmpty()) + ? query.getModel().getRestrict().stream() + .map(schema2Searcher::get) + .collect(Collectors.toCollection(TinyIdentitySet::new)) + : schema2Searcher.values().stream().collect(Collectors.toCollection(TinyIdentitySet::new)); + + if ( ! servers.isEmpty() ) { + for (var server : servers) { + if (query.getTimeLeft() > 0) { + server.fill(result, summaryClass); + } else { + if (result.hits().getErrorHit() == null) { + result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + + query.getTimeout() + " ms")); + } } } } else { @@ -230,22 +229,21 @@ public class ClusterSearcher extends Searcher { query.getRanking().setQueryCache(false); } - private Result doSearch(Searcher searcher, Query query, Execution execution) { - if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query, execution); + private Result doSearch(Query query) { + if (schema2Searcher.size() > 1) { + return searchMultipleDocumentTypes(query); } else { - String docType = schemas.iterator().next(); - query.getModel().setRestrict(docType); - return perSchemaSearch(searcher, query, execution); + String schema = schema2Searcher.keySet().iterator().next(); + query.getModel().setRestrict(schema); + return perSchemaSearch(schema, query); } } - private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + private Result perSchemaSearch(String schema, Query query) { Set<String> restrict = query.getModel().getRestrict(); if (restrict.size() != 1) { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); } - String schema = restrict.iterator().next(); int rerankCount = globalPhaseRanker != null ? globalPhaseRanker.getRerankCount(query, schema) : 0; boolean useGlobalPhase = rerankCount > 0; final int wantOffset = query.getOffset(); @@ -257,7 +255,7 @@ public class ClusterSearcher extends Searcher { query.setOffset(0); query.setHits(useHits); } - Result result = searcher.search(query, execution); + Result result = schema2Searcher.get(schema).search(schema, query); if (useGlobalPhase) { globalPhaseRanker.rerankHits(query, result, schema); result.hits().trim(wantOffset, wantHits); @@ -284,16 +282,17 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { + private Result searchMultipleDocumentTypes(Query query) { Set<String> schemas = resolveSchemas(query); - List<Query> queries = createQueries(query, schemas); - if (queries.size() == 1) { - return perSchemaSearch(searcher, queries.get(0), execution); + Map<String, Query> schemaQueries = createQueries(query, schemas); + if (schemaQueries.size() == 1) { + var entry = schemaQueries.entrySet().iterator().next(); + return perSchemaSearch(entry.getKey(), entry.getValue()); } else { Result mergedResult = new Result(query); - List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); - for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); + List<FutureTask<Result>> pending = new ArrayList<>(schemaQueries.size()); + for (var entry : schemaQueries.entrySet()) { + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(entry.getKey(), entry.getValue())); try { executor.execute(task); pending.add(task); @@ -309,7 +308,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS, execution); + fill(mergedResult, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim @@ -326,7 +325,7 @@ public class ClusterSearcher extends Searcher { candidates.addAll(cluster.schemas()); } return (candidates.isEmpty() ? sources : candidates).stream() - .filter(schemas::contains).collect(Collectors.toUnmodifiableSet()); + .filter(schema2Searcher::containsKey).collect(Collectors.toUnmodifiableSet()); } Set<String> resolveSchemas(Query query) { @@ -334,7 +333,7 @@ public class ClusterSearcher extends Searcher { if (restrict == null || restrict.isEmpty()) { Set<String> sources = query.getModel().getSources(); return (sources == null || sources.isEmpty()) - ? schemas + ? schema2Searcher.keySet() : resolveSourceSubset(sources); } else { return filterValidDocumentTypes(restrict); @@ -344,34 +343,40 @@ public class ClusterSearcher extends Searcher { private Set<String> filterValidDocumentTypes(Collection<String> restrict) { Set<String> retval = new LinkedHashSet<>(); for (String docType : restrict) { - if (docType != null && schemas.contains(docType)) { + if (docType != null && schema2Searcher.containsKey(docType)) { retval.add(docType); } } return retval; } - private List<Query> createQueries(Query query, Set<String> docTypes) { + private Map<String, Query> createQueries(Query query, Set<String> schemas) { query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - List<Query> retval = new ArrayList<>(docTypes.size()); - if (docTypes.size() == 1) { - query.getModel().setRestrict(docTypes.iterator().next()); - retval.add(query); - } else if ( ! docTypes.isEmpty() ) { - for (String docType : docTypes) { + if (schemas.size() == 1) { + String schema = schemas.iterator().next(); + query.getModel().setRestrict(schema); + return Map.of(schema, query); + } else if ( ! schemas.isEmpty() ) { + var schemaQueries = new HashMap<String, Query>(); + for (String schema : schemas) { Query q = query.clone(); q.setOffset(0); q.setHits(query.getOffset() + query.getHits()); - q.getModel().setRestrict(docType); - retval.add(q); + q.getModel().setRestrict(schema); + schemaQueries.put(schema, q); } + return schemaQueries; } - return retval; + return Map.of(); } @Override public void deconstruct() { - if (server != null) { + Map<String, VespaBackend> servers = new HashMap<>(); + for (var server : schema2Searcher.values()) { + servers.put(server.getName(), server); + } + for (var server : servers.values()) { server.shutDown(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java index c0f42f6924b..6a980c37f4e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -7,7 +7,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.search.result.NanNumber; /** * @author bratseth diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java index c34187e576b..2987500fdb3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java @@ -1,21 +1,40 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.yahoo.search.schema.SchemaInfo; + /** * Helper class for carrying around cluster-related - * config parameters to the FastSearcher class. + * config parameters to the VespaBackend class. * * @author arnej27959 */ public class ClusterParams { - public final String searcherName; + private final String searcherName; + private final String serverId; + private final String defaultSummary; + private final DocumentdbInfoConfig documentdbInfoConfig; + private final SchemaInfo schemaInfo; - /** - * Make up full ClusterParams - */ public ClusterParams(String name) { + this(name, "server.0", null, null, null); + } + public ClusterParams(String name, String serverId, String defaultSummary, + DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { this.searcherName = name; + this.serverId = serverId; + if (defaultSummary != null && defaultSummary.isEmpty()) + this.defaultSummary = null; + else + this.defaultSummary = defaultSummary; + this.documentdbInfoConfig = documentdbInfoConfig; + this.schemaInfo = schemaInfo; } + public String getServerId() { return serverId; } + public String getSearcherName() { return searcherName; } + public String getDefaultSummary() { return defaultSummary; } + public DocumentdbInfoConfig getDocumentdbInfoConfig() { return documentdbInfoConfig; } + public SchemaInfo getSchemaInfo() { return schemaInfo; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java index ddb36e007d3..9836934acc1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -15,8 +13,6 @@ import com.yahoo.search.query.Ranking; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; -import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.Optional; @@ -33,7 +29,7 @@ import java.util.Optional; // errors on results and returning them. It could be handy to create a QueryHandlingErrorException // or similar which could wrap an error message, and then just always throw that and // catch and unwrap into a results with an error in high level methods. -Jon -public class FastSearcher extends VespaBackEndSearcher { +public class IndexedBackend extends VespaBackend { /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ private final Dispatcher dispatcher; @@ -41,34 +37,18 @@ public class FastSearcher extends VespaBackEndSearcher { /** * Creates a Fastsearcher. * - * @param serverId the resource pool used to create direct connections to the local search nodes when - * bypassing the dispatch node * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. * Eventually we will move everything to this protocol and never use dispatch nodes. * At that point we won't need a cluster searcher above this to select and pass the right * backend. - * @param docSumParams document summary parameters * @param clusterParams the cluster number, and other cluster backend parameters - * @param documentdbInfoConfig document database parameters */ - public FastSearcher(String serverId, - Dispatcher dispatcher, - SummaryParameters docSumParams, - ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo) { - init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + public IndexedBackend(ClusterParams clusterParams, Dispatcher dispatcher) + { + super(clusterParams); this.dispatcher = dispatcher; } - /** - * Pings the backend. Does not propagate to other searchers. - */ - @Override - public Pong ping(Ping ping, Execution execution) { - throw new IllegalStateException("This ping should not have been called."); - } - @Override protected void transformQuery(Query query) { QueryRewrite.rewriteSddocname(query); @@ -83,11 +63,11 @@ public class FastSearcher extends VespaBackEndSearcher { } @Override - public Result doSearch2(Query query, Execution execution) { + public Result doSearch2(String schema, Query query) { if (dispatcher.allGroupsHaveSize1()) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { - Result result = invoker.search(query, execution); + Result result = invoker.search(query); injectSource(result.hits()); if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { @@ -97,7 +77,7 @@ public class FastSearcher extends VespaBackEndSearcher { // contain the data we need. If we fetch the default // one we end up fetching docsums twice unless the // user also requested the default one. - fill(result, query.getPresentation().getSummary(), execution); // ARGH + fill(result, query.getPresentation().getSummary()); // ARGH } return result; } catch (TimeoutException e) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java deleted file mode 100644 index 8751a730229..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - - -/** - * Wrapper for document summary parameters and configuration. - * - * @author Steinar Knutsen - */ -public class SummaryParameters { - - public final String defaultClass; - - public SummaryParameters(String defaultClass) { - if (defaultClass != null && defaultClass.isEmpty()) - this.defaultClass = null; - else - this.defaultClass = defaultClass; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java index 2e635d21f01..761cb22be57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -12,41 +12,37 @@ import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; -import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Superclass for backend searchers. * * @author baldersheim */ -public abstract class VespaBackEndSearcher extends PingableSearcher { +public abstract class VespaBackend { /** for vespa-internal use only; consider renaming the summary class */ public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; - private String serverId; + private final String serverId; /** The set of all document databases available in the backend handled by this searcher */ - private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); - private DocumentDatabase defaultDocumentDb = null; + private final Map<String, DocumentDatabase> documentDbs; + private final DocumentDatabase defaultDocumentDb; /** Default docsum class. null means "unset" and is the default value */ - private String defaultDocsumClass = null; + private final String defaultDocsumClass; /** Returns an iterator which returns all hits below this result **/ private static Iterator<Hit> hitIterator(Result result) { @@ -54,24 +50,36 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } /** The name of this source */ - private String name; + private final String name; - public final String getName() { return name; } - protected final String getDefaultDocsumClass() { return defaultDocsumClass; } + protected VespaBackend(ClusterParams clusterParams) { + this.serverId = clusterParams.getServerId(); + this.name = clusterParams.getSearcherName(); + this.defaultDocsumClass = clusterParams.getDefaultSummary(); - /** Sets default document summary class. Default is null */ - private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } + Validator.ensureNotNull("Name of Vespa backend integration", name); + + List<DocumentDatabase> dbs = new ArrayList<>(); + if (clusterParams.getDocumentdbInfoConfig() != null) { + for (DocumentdbInfoConfig.Documentdb docDb : clusterParams.getDocumentdbInfoConfig().documentdb()) { + DocumentDatabase db = new DocumentDatabase(clusterParams.getSchemaInfo().schemas().get(docDb.name())); + dbs.add(db); + } + } + this.defaultDocumentDb = dbs.isEmpty() ? null : dbs.get(0); + this.documentDbs = dbs.stream().collect(Collectors.toMap(db -> db.schema().name(), db -> db)); + } - public final Logger getLogger() { return super.getLogger(); } + public final String getName() { return name; } + protected final String getDefaultDocsumClass() { return defaultDocsumClass; } /** * Searches a search cluster * This is an endpoint - searchers will never propagate the search to any nested searcher. * * @param query the query to search - * @param execution the query execution context */ - protected abstract Result doSearch2(Query query, Execution execution); + protected abstract Result doSearch2(String schema, Query query); protected abstract void doPartialFill(Result result, String summaryClass); @@ -133,29 +141,9 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } } - public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { - this.serverId = serverId; - this.name = clusterParams.searcherName; - - Validator.ensureNotNull("Name of Vespa backend integration", getName()); - - setDefaultDocsumClass(docSumParams.defaultClass); - - if (documentdbInfoConfig != null) { - for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name())); - if (documentDbs.isEmpty()) - defaultDocumentDb = db; - documentDbs.put(docDb.name(), db); - } - } - } - protected void transformQuery(Query query) { } - @Override - public Result search(Query query, Execution execution) { + public Result search(String schema, Query query) { // query root should not be null here Item root = query.getModel().getQueryTree().getRoot(); if (root == null || root instanceof NullItem) { @@ -183,7 +171,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? return new Result(query); - Result result = doSearch2(query, execution); + Result result = doSearch2(schema, query); if (query.getTrace().getLevel() >= 1) query.trace(getName() + " dispatch response: " + result, false, 1); @@ -191,7 +179,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return result; } - private List<Result> partitionHits(Result result, String summaryClass) { + private static List<Result> partitionHits(Result result, String summaryClass) { List<Result> parts = new ArrayList<>(); TinyIdentitySet<Query> queryMap = new TinyIdentitySet<>(4); @@ -217,8 +205,8 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return parts; } - @Override - public void fill(Result result, String summaryClass, Execution execution) { + //TODO Add schema here too. + public void fill(Result result, String summaryClass) { if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove List<Result> parts = partitionHits(result, summaryClass); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 1128ad9570d..634a16d1dee 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,7 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -115,7 +115,7 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { // TODO: remove or rename this internal summary class for Vespa 9 - execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + execution.fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 8ec88bd7f45..1689f6d246e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -6,7 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.annotation.Inject; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -256,13 +256,13 @@ public class Dispatcher extends AbstractComponent { } } - public FillInvoker getFillInvoker(Result result, VespaBackEndSearcher searcher) { + public FillInvoker getFillInvoker(Result result, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. return items.register(items.get().invokerFactory.createFillInvoker(searcher, result)); } } - public SearchInvoker getSearchInvoker(Query query, VespaBackEndSearcher searcher) { + public SearchInvoker getSearchInvoker(Query query, VespaBackend searcher) { try (var items = volatileItems()) { // Take a snapshot, and release it when we're done. int maxHitsPerNode = dispatchConfig.maxHitsPerNode(); SearchInvoker invoker = getSearchPathInvoker(query, searcher, searchCluster.groupList(), items.get().invokerFactory, maxHitsPerNode) @@ -277,7 +277,7 @@ public class Dispatcher extends AbstractComponent { } /** Builds an invoker based on searchpath */ - private static Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackEndSearcher searcher, SearchGroups cluster, + private static Optional<SearchInvoker> getSearchPathInvoker(Query query, VespaBackend searcher, SearchGroups cluster, InvokerFactory invokerFactory, int maxHitsPerNode) { String searchPath = query.getModel().getSearchPath(); if (searchPath == null) return Optional.empty(); @@ -297,7 +297,7 @@ public class Dispatcher extends AbstractComponent { } } - private static SearchInvoker getInternalInvoker(Query query, VespaBackEndSearcher searcher, SearchCluster cluster, + private static SearchInvoker getInternalInvoker(Query query, VespaBackend searcher, SearchCluster cluster, LoadBalancer loadBalancer, InvokerFactory invokerFactory, int maxHitsPerNode) { Optional<Node> directNode = cluster.localCorpusDispatchTarget(); if (directNode.isPresent()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index be8524bedd2..a246589ec7e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -111,9 +111,9 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() throws IOException { InvokerResult result = new InvokerResult(query, query.getHits()); - List<LeanHit> merged = Collections.emptyList(); + List<LeanHit> merged = List.of(); long nextTimeout = query.getTimeLeft(); var groupingResultAggregator = new GroupingResultAggregator(); try { @@ -124,7 +124,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM coverageAggregator.getAnsweredNodes() + " responses received"); break; } else { - InvokerResult toMerge = invoker.getSearchResult(execution); + InvokerResult toMerge = invoker.getSearchResult(); merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f1d58122570..d7fad148c8c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -2,7 +2,7 @@ package com.yahoo.search.dispatch; import com.yahoo.concurrent.Timer; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; @@ -34,12 +34,12 @@ public abstract class InvokerFactory { this.hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); } - protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node); - public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); + public abstract FillInvoker createFillInvoker(VespaBackend searcher, Result result); /** * Creates a {@link SearchInvoker} for a list of content nodes. @@ -52,7 +52,7 @@ public abstract class InvokerFactory { * @return the invoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ - Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, + Optional<SearchInvoker> createSearchInvoker(VespaBackend searcher, Query query, List<Node> nodes, boolean acceptIncompleteCoverage, diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index b494b5b3819..74da2388155 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -35,7 +35,7 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected Object sendSearchRequest(Query query, Object context) throws IOException { + protected Object sendSearchRequest(Query query, Object context) { this.query = query; if (monitor != null) { monitor.responseAvailable(this); @@ -44,7 +44,7 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() { Result res = new Result(query, message); if (coverage != null) { res.setCoverage(coverage); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index ad332494179..4ad89674b7b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -31,9 +31,9 @@ public abstract class SearchInvoker extends CloseableInvoker { * nodes, the provided {@link Execution} may be used to retrieve document summaries required * for correct result windowing. */ - public Result search(Query query, Execution execution) throws IOException { + public Result search(Query query) throws IOException { sendSearchRequest(query, null); - InvokerResult result = getSearchResult(execution); + InvokerResult result = getSearchResult(); setFinalStatus(result.getResult().hits().getError() == null); result.complete(); return result.getResult(); @@ -48,7 +48,7 @@ public abstract class SearchInvoker extends CloseableInvoker { */ protected abstract Object sendSearchRequest(Query query, Object context) throws IOException; - protected abstract InvokerResult getSearchResult(Execution execution) throws IOException; + protected abstract InvokerResult getSearchResult() throws IOException; protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 9da4c91cd16..284b4a4fcbc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -14,7 +14,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; @@ -227,7 +227,7 @@ public class ProtobufSerialization { return convertFromResult(searchResult).toByteArray(); } - static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) + static InvokerResult deserializeToSearchResult(byte[] payload, Query query, VespaBackend searcher, int partId, int distKey) throws InvalidProtocolBufferException { var protobuf = SearchProtocol.SearchReply.parseFrom(payload); return convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 8221391afe6..9fd602032e9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.FillInvoker; @@ -37,12 +37,12 @@ public class RpcInvokerFactory extends InvokerFactory { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, int maxHits, Node node) { + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackend searcher, Query query, int maxHits, Node node) { return Optional.of(new RpcSearchInvoker(searcher, compressor, node, rpcResourcePool, maxHits)); } @Override - public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackend searcher, Result result) { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index e34e5c97bc4..fb028358caf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -2,14 +2,13 @@ package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.Compressor; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.Optional; @@ -26,7 +25,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private static final String RPC_METHOD = "vespa.searchprotocol.search"; - private final VespaBackEndSearcher searcher; + private final VespaBackend searcher; private final Node node; private final RpcConnectionPool resourcePool; private final BlockingQueue<Client.ResponseOrError<ProtobufResponse>> responses; @@ -35,7 +34,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe private Query query; - RpcSearchInvoker(VespaBackEndSearcher searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { + RpcSearchInvoker(VespaBackend searcher, CompressPayload compressor, Node node, RpcConnectionPool resourcePool, int maxHits) { super(Optional.of(node)); this.searcher = searcher; this.node = node; @@ -86,7 +85,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() throws IOException { long timeLeftMs = query.getTimeLeft(); if (timeLeftMs <= 0) { return errorResult(query, ErrorMessage.createTimeout("Timeout while waiting for " + getName())); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 0b3179f5ad6..6eb69c76afd 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -534,7 +534,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { */ @Deprecated // TODO Remove on Vespa 9. public void fillAttributes(Result result) { - fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); + fill(result, VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 0685a77d7a9..64a989fd0e7 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -5,7 +5,7 @@ import java.util.HashSet; import java.util.Set; import com.yahoo.component.chain.dependencies.After; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index 5fef90c2012..e367bb58f80 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import static com.yahoo.vespa.streamingvisitors.StreamingSearcher.STREAMING_STATISTICS; +import static com.yahoo.vespa.streamingvisitors.StreamingBackend.STREAMING_STATISTICS; /** * Generates mail-specific query metrics. diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java index 180fdc389d8..6b81ab0fa97 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java @@ -11,10 +11,11 @@ import com.yahoo.fs4.DocsumPacket; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.TimeoutException; -import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -44,8 +45,9 @@ import java.util.logging.Logger; * @author baldersheim * @author Ulf Carlin */ -public class StreamingSearcher extends VespaBackEndSearcher { +public class StreamingBackend extends VespaBackend { + private static final Logger log = Logger.getLogger(StreamingBackend.class.getName()); private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); @@ -53,41 +55,35 @@ public class StreamingSearcher extends VespaBackEndSearcher { static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; private final TracingOptions tracingOptions; - private static final Logger log = Logger.getLogger(StreamingSearcher.class.getName()); - private Route route; + private final Route route; /** The configId used to access the searchcluster. */ - private String searchClusterName = null; + private final String searchClusterName; /** The route to the storage cluster. */ - private String storageClusterRouteSpec = null; + private final String storageClusterRouteSpec; - StreamingSearcher(VisitorFactory visitorFactory) { - this.visitorFactory = visitorFactory; - tracingOptions = TracingOptions.DEFAULT; + StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec) { + this(clusterParams, searchClusterName, visitorFactory, storageClusterRouteSpec, TracingOptions.DEFAULT); } - StreamingSearcher(VisitorFactory visitorFactory, TracingOptions tracingOptions) { + StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec, TracingOptions tracingOptions) { + super(clusterParams); this.visitorFactory = visitorFactory; this.tracingOptions = tracingOptions; + this.searchClusterName = searchClusterName; + this.storageClusterRouteSpec = storageClusterRouteSpec; + this.route = Route.parse(storageClusterRouteSpec); } - public StreamingSearcher(VespaDocumentAccess access) { - this(new VespaVisitorFactory(access)); + public StreamingBackend(ClusterParams clusterParams, String searchClusterName, VespaDocumentAccess access, String storageClusterRouteSpec) { + this(clusterParams, searchClusterName, new VespaVisitorFactory(access), storageClusterRouteSpec); } private String getSearchClusterName() { return searchClusterName; } - private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } - public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; } - public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { - this.storageClusterRouteSpec = storageClusterRouteSpec; - } - - @Override - protected void doPartialFill(Result result, String summaryClass) { - } + @Override protected void doPartialFill(Result result, String summaryClass) { } private double durationInMillisFromNanoTime(long startTimeNanos) { return (tracingOptions.getClock().nanoTimeNow() - startTimeNanos) / (double)TimeUnit.MILLISECONDS.toNanos(1); @@ -126,7 +122,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { } @Override - public Result doSearch2(Query query, Execution execution) { + public Result doSearch2(String schema, Query query) { if (query.getTimeLeft() <= 0) return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); @@ -135,8 +131,6 @@ public class StreamingSearcher extends VespaBackEndSearcher { return new Result(query, ErrorMessage.createIllegalQuery("Streaming search requires either " + "streaming.groupname or streaming.selection")); } - // Cluster searcher guarantees that there will be one, and only one schema here - String schema = query.getModel().getRestrict().iterator().next(); if (query.getTrace().isTraceable(4)) query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + schema, 4); @@ -167,11 +161,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { } private void initializeMissingQueryFields(Query query) { - lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec()); - - if (route == null) { - route = Route.parse(getStorageClusterRouteSpec()); - } + lazyTrace(query, 7, "Routing to storage cluster ", storageClusterRouteSpec); lazyTrace(query, 8, "Route is ", route); lazyTrace(query, 7, "doSearch2(): query docsum class=", @@ -264,7 +254,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { lazyTrace(query, 8, "Returning result ", result); if (skippedHits > 0) { - getLogger().info("skipping " + skippedHits + " hits for query: " + result.getQuery()); + log.info("skipping " + skippedHits + " hits for query: " + result.getQuery()); result.hits().addError(ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); } @@ -366,8 +356,8 @@ public class StreamingSearcher extends VespaBackEndSearcher { } @Override - public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { - return new StreamingVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); + public Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride) { + return new StreamingVisitor(query, searchCluster, route, schema, this, traceLevelOverride); } } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java index 658eea0b526..bb155e3f934 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java @@ -28,6 +28,7 @@ import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.objects.BufferSerializer; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -76,13 +77,13 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { } public StreamingVisitor(Query query, String searchCluster, Route route, - String documentType, VisitorSessionFactory visitorSessionFactory, + String schema, VisitorSessionFactory visitorSessionFactory, int traceLevelOverride) { this.query = query; this.visitorSessionFactory = visitorSessionFactory; this.traceLevelOverride = traceLevelOverride; - setVisitorParameters(searchCluster, route, documentType); + setVisitorParameters(searchCluster, route, schema); } private int inferSessionTraceLevel(Query query) { @@ -112,8 +113,8 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { return query.properties().getString(streamingSelection); } - private void setVisitorParameters(String searchCluster, Route route, String documentType) { - params.setDocumentSelection(createSelectionString(documentType, createQuerySelectionString())); + private void setVisitorParameters(String searchCluster, Route route, String schema) { + params.setDocumentSelection(createSelectionString(schema, createQuerySelectionString())); params.setTimeoutMs(query.getTimeout()); // Per bucket visitor timeout params.setSessionTimeoutMs(query.getTimeout()); params.setVisitorLibrary("searchvisitor"); @@ -146,7 +147,8 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { encodeQueryData(query, 0, ed); params.setLibraryParameter("query", ed.getEncodedData()); params.setLibraryParameter("querystackcount", String.valueOf(ed.getReturned())); - params.setLibraryParameter("searchcluster", searchCluster.getBytes()); + params.setLibraryParameter("searchcluster", searchCluster.getBytes(StandardCharsets.UTF_8)); + params.setLibraryParameter("schema", schema.getBytes(StandardCharsets.UTF_8)); if (query.getPresentation().getSummary() != null) { params.setLibraryParameter("summaryclass", query.getPresentation().getSummary()); } else { @@ -265,7 +267,7 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { log.log(Level.FINE, () -> "StreamingVisitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); - throw new TimeoutException("Query timed out in " + StreamingSearcher.class.getName()); + throw new TimeoutException("Query timed out in " + StreamingBackend.class.getName()); } } finally { session.destroy(); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java index 10387065128..e33bbd206be 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java @@ -11,6 +11,6 @@ import com.yahoo.search.Query; */ interface VisitorFactory { - Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride); + Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride); } |