diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo')
6 files changed, 87 insertions, 128 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index a8605ab1597..855a524473d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -12,7 +12,6 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.IndexedBackend; -import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.prelude.fastsearch.VespaBackend; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -92,23 +91,19 @@ public class ClusterSearcher extends Searcher { maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); - SummaryParameters docSumParams = new SummaryParameters(qrsConfig - .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() - .defaultclass()); - - String uniqueServerId = UUID.randomUUID().toString(); VespaBackend streaming = null, indexed = null; + ClusterParams clusterParams = makeClusterParams(searchClusterIndex, qrsConfig + .com().yahoo().prelude().fastsearch().IndexedBackend().docsum() + .defaultclass(), documentDbConfig, schemaInfo); for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.INDEX) { if (indexed == null) { - indexed = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, - docSumParams, documentDbConfig, schemaInfo, dispatchers); + indexed = searchDispatch(clusterParams, searchClusterName, dispatchers); } schema2Searcher.put(docDb.name(), indexed); } else if (docDb.mode() == DocumentdbInfoConfig.Documentdb.Mode.Enum.STREAMING) { if (streaming == null) { - streaming = streamingCluster(uniqueServerId, searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); + streaming = streamingCluster(clusterParams, searchClusterConfig, access); vipStatus.addToRotation(streaming.getName()); } schema2Searcher.put(docDb.name(), streaming); @@ -126,42 +121,30 @@ public class ClusterSearcher extends Searcher { config.searchcluster().stream().map(QrSearchersConfig.Searchcluster::name).toList()); } - private static ClusterParams makeClusterParams(int searchclusterIndex) { - return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); + private static ClusterParams makeClusterParams(int searchclusterIndex, String defaultSummary, + DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo) + { + return new ClusterParams("sc" + searchclusterIndex + ".num" + 0, UUID.randomUUID().toString(), + defaultSummary, documentDbConfig, schemaInfo); } - private static IndexedBackend searchDispatch(int searchclusterIndex, + private static IndexedBackend searchDispatch(ClusterParams clusterParams, String searchClusterName, - String serverId, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - ComponentRegistry<Dispatcher> dispatchers) { - ClusterParams clusterParams = makeClusterParams(searchclusterIndex); + ComponentRegistry<Dispatcher> dispatchers) + { ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName); Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId); if (dispatcher == null) - throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + - " is configured"); - return new IndexedBackend(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId + " is configured"); + return new IndexedBackend(clusterParams, dispatcher); } - private static StreamingBackend streamingCluster(String serverId, - int searchclusterIndex, + private static StreamingBackend streamingCluster(ClusterParams clusterParams, QrSearchersConfig.Searchcluster searchClusterConfig, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - VespaDocumentAccess access) { - if (searchClusterConfig.searchdef().size() != 1) - throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " + - searchClusterConfig.searchdef()); - ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - StreamingBackend searcher = new StreamingBackend(access); - searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid()); - searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); - searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); - return searcher; + VespaDocumentAccess access) + { + return new StreamingBackend(clusterParams, searchClusterConfig.rankprofiles_configid(), + access, searchClusterConfig.storagecluster().routespec()); } /** Do not use, for internal testing purposes only. **/ diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java index c34187e576b..2987500fdb3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java @@ -1,21 +1,40 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.yahoo.search.schema.SchemaInfo; + /** * Helper class for carrying around cluster-related - * config parameters to the FastSearcher class. + * config parameters to the VespaBackend class. * * @author arnej27959 */ public class ClusterParams { - public final String searcherName; + private final String searcherName; + private final String serverId; + private final String defaultSummary; + private final DocumentdbInfoConfig documentdbInfoConfig; + private final SchemaInfo schemaInfo; - /** - * Make up full ClusterParams - */ public ClusterParams(String name) { + this(name, "server.0", null, null, null); + } + public ClusterParams(String name, String serverId, String defaultSummary, + DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { this.searcherName = name; + this.serverId = serverId; + if (defaultSummary != null && defaultSummary.isEmpty()) + this.defaultSummary = null; + else + this.defaultSummary = defaultSummary; + this.documentdbInfoConfig = documentdbInfoConfig; + this.schemaInfo = schemaInfo; } + public String getServerId() { return serverId; } + public String getSearcherName() { return searcherName; } + public String getDefaultSummary() { return defaultSummary; } + public DocumentdbInfoConfig getDocumentdbInfoConfig() { return documentdbInfoConfig; } + public SchemaInfo getSchemaInfo() { return schemaInfo; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java index 294aff8d78b..9836934acc1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java @@ -13,7 +13,6 @@ import com.yahoo.search.query.Ranking; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; -import com.yahoo.search.schema.SchemaInfo; import java.io.IOException; import java.util.Optional; @@ -38,23 +37,15 @@ public class IndexedBackend extends VespaBackend { /** * Creates a Fastsearcher. * - * @param serverId the resource pool used to create direct connections to the local search nodes when - * bypassing the dispatch node * @param dispatcher the dispatcher used (when enabled) to send summary requests over the rpc protocol. * Eventually we will move everything to this protocol and never use dispatch nodes. * At that point we won't need a cluster searcher above this to select and pass the right * backend. - * @param docSumParams document summary parameters * @param clusterParams the cluster number, and other cluster backend parameters - * @param documentdbInfoConfig document database parameters */ - public IndexedBackend(String serverId, - Dispatcher dispatcher, - SummaryParameters docSumParams, - ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo) { - init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); + public IndexedBackend(ClusterParams clusterParams, Dispatcher dispatcher) + { + super(clusterParams); this.dispatcher = dispatcher; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java deleted file mode 100644 index 8751a730229..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - - -/** - * Wrapper for document summary parameters and configuration. - * - * @author Steinar Knutsen - */ -public class SummaryParameters { - - public final String defaultClass; - - public SummaryParameters(String defaultClass) { - if (defaultClass != null && defaultClass.isEmpty()) - this.defaultClass = null; - else - this.defaultClass = defaultClass; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java index 5191bfc4f41..761cb22be57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -16,15 +16,14 @@ import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; -import com.yahoo.search.schema.SchemaInfo; import com.yahoo.searchlib.aggregation.Grouping; import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * Superclass for backend searchers. @@ -36,14 +35,14 @@ public abstract class VespaBackend { /** for vespa-internal use only; consider renaming the summary class */ public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; - private String serverId; + private final String serverId; /** The set of all document databases available in the backend handled by this searcher */ - private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); - private DocumentDatabase defaultDocumentDb = null; + private final Map<String, DocumentDatabase> documentDbs; + private final DocumentDatabase defaultDocumentDb; /** Default docsum class. null means "unset" and is the default value */ - private String defaultDocsumClass = null; + private final String defaultDocsumClass; /** Returns an iterator which returns all hits below this result **/ private static Iterator<Hit> hitIterator(Result result) { @@ -51,14 +50,29 @@ public abstract class VespaBackend { } /** The name of this source */ - private String name; + private final String name; + + protected VespaBackend(ClusterParams clusterParams) { + this.serverId = clusterParams.getServerId(); + this.name = clusterParams.getSearcherName(); + this.defaultDocsumClass = clusterParams.getDefaultSummary(); + + Validator.ensureNotNull("Name of Vespa backend integration", name); + + List<DocumentDatabase> dbs = new ArrayList<>(); + if (clusterParams.getDocumentdbInfoConfig() != null) { + for (DocumentdbInfoConfig.Documentdb docDb : clusterParams.getDocumentdbInfoConfig().documentdb()) { + DocumentDatabase db = new DocumentDatabase(clusterParams.getSchemaInfo().schemas().get(docDb.name())); + dbs.add(db); + } + } + this.defaultDocumentDb = dbs.isEmpty() ? null : dbs.get(0); + this.documentDbs = dbs.stream().collect(Collectors.toMap(db -> db.schema().name(), db -> db)); + } public final String getName() { return name; } protected final String getDefaultDocsumClass() { return defaultDocsumClass; } - /** Sets default document summary class. Default is null */ - private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } - /** * Searches a search cluster * This is an endpoint - searchers will never propagate the search to any nested searcher. @@ -127,25 +141,6 @@ public abstract class VespaBackend { } } - public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { - this.serverId = serverId; - this.name = clusterParams.searcherName; - - Validator.ensureNotNull("Name of Vespa backend integration", getName()); - - setDefaultDocsumClass(docSumParams.defaultClass); - - if (documentdbInfoConfig != null) { - for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name())); - if (documentDbs.isEmpty()) - defaultDocumentDb = db; - documentDbs.put(docDb.name(), db); - } - } - } - protected void transformQuery(Query query) { } public Result search(String schema, Query query) { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java index 9953d76f50a..6b81ab0fa97 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingBackend.java @@ -11,6 +11,7 @@ import com.yahoo.fs4.DocsumPacket; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.TimeoutException; @@ -46,6 +47,7 @@ import java.util.logging.Logger; */ public class StreamingBackend extends VespaBackend { + private static final Logger log = Logger.getLogger(StreamingBackend.class.getName()); private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); @@ -53,41 +55,35 @@ public class StreamingBackend extends VespaBackend { static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; private final TracingOptions tracingOptions; - private static final Logger log = Logger.getLogger(StreamingBackend.class.getName()); - private Route route; + private final Route route; /** The configId used to access the searchcluster. */ - private String searchClusterName = null; + private final String searchClusterName; /** The route to the storage cluster. */ - private String storageClusterRouteSpec = null; + private final String storageClusterRouteSpec; - StreamingBackend(VisitorFactory visitorFactory) { - this.visitorFactory = visitorFactory; - tracingOptions = TracingOptions.DEFAULT; + StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec) { + this(clusterParams, searchClusterName, visitorFactory, storageClusterRouteSpec, TracingOptions.DEFAULT); } - StreamingBackend(VisitorFactory visitorFactory, TracingOptions tracingOptions) { + StreamingBackend(ClusterParams clusterParams, String searchClusterName, VisitorFactory visitorFactory, String storageClusterRouteSpec, TracingOptions tracingOptions) { + super(clusterParams); this.visitorFactory = visitorFactory; this.tracingOptions = tracingOptions; + this.searchClusterName = searchClusterName; + this.storageClusterRouteSpec = storageClusterRouteSpec; + this.route = Route.parse(storageClusterRouteSpec); } - public StreamingBackend(VespaDocumentAccess access) { - this(new VespaVisitorFactory(access)); + public StreamingBackend(ClusterParams clusterParams, String searchClusterName, VespaDocumentAccess access, String storageClusterRouteSpec) { + this(clusterParams, searchClusterName, new VespaVisitorFactory(access), storageClusterRouteSpec); } private String getSearchClusterName() { return searchClusterName; } - private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } - public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; } - public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { - this.storageClusterRouteSpec = storageClusterRouteSpec; - } - - @Override - protected void doPartialFill(Result result, String summaryClass) { - } + @Override protected void doPartialFill(Result result, String summaryClass) { } private double durationInMillisFromNanoTime(long startTimeNanos) { return (tracingOptions.getClock().nanoTimeNow() - startTimeNanos) / (double)TimeUnit.MILLISECONDS.toNanos(1); @@ -165,11 +161,7 @@ public class StreamingBackend extends VespaBackend { } private void initializeMissingQueryFields(Query query) { - lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec()); - - if (route == null) { - route = Route.parse(getStorageClusterRouteSpec()); - } + lazyTrace(query, 7, "Routing to storage cluster ", storageClusterRouteSpec); lazyTrace(query, 8, "Route is ", route); lazyTrace(query, 7, "doSearch2(): query docsum class=", |