summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@verizonmedia.com>2019-10-25 13:33:09 +0200
committerJon Bratseth <bratseth@verizonmedia.com>2019-10-25 13:33:09 +0200
commitcc7153b5c502bfdfa9603182cfe6848f955075de (patch)
tree2da646cad6d24c5c4c35ef26d6ac1a1940897169
parent0087412bb6e46c06ba66705a11121f522f0bfc02 (diff)
Create one Dispatcher component per search cluster
This avoids creating an excessive number of connections to search clusters when the application (incorrectly) creates many local provider chains to the same search cluster.
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java3
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java92
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java88
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java6
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java3
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java106
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java28
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java43
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java1
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java5
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java17
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java11
25 files changed, 311 insertions, 223 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 922c6488659..0bdc3fbef9a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -46,8 +46,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
RankProfilesConfig.Producer,
RankingConstantsConfig.Producer,
ServletPathsConfig.Producer,
- ContainerMbusConfig.Producer
-{
+ ContainerMbusConfig.Producer {
private final Set<FileReference> applicationBundles = new LinkedHashSet<>();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index e4f5791b6b3..f5eb128e7f3 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -99,9 +99,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
DocprocConfig.Producer,
ClusterInfoConfig.Producer,
RoutingProviderConfig.Producer,
- ConfigserverConfig.Producer
-
-{
+ ConfigserverConfig.Producer {
/**
* URI prefix used for internal, usually programmatic, APIs. URIs using this
@@ -268,6 +266,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
*
* @return the removed component, or null if it was not present
*/
+ @SuppressWarnings("unused") // Used from other repositories
public Component removeComponent(ComponentId componentId) {
return componentGroup.removeComponent(componentId);
}
@@ -298,7 +297,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public void addContainer(CONTAINER container) {
container.setClusterName(name);
container.setProp("clustername", name)
- .setProp("index", this.containers.size());
+ .setProp("index", this.containers.size());
containers.add(container);
}
@@ -330,7 +329,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public SearchChains getSearchChains() {
if (containerSearch == null)
throw new IllegalStateException("Search components not found in container cluster '" + getSubId() +
- "': Add <search/> to the cluster in services.xml");
+ "': Add <search/> to the cluster in services.xml");
return containerSearch.getChains();
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
index f02878fe545..67870dc049d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
@@ -48,9 +48,8 @@ public class ContainerModel extends ConfigModel {
List<AbstractSearchCluster> searchClusters = Content.getSearchClusters(configModelRepo);
Map<String, AbstractSearchCluster> searchClustersByName = new TreeMap<>();
- for (AbstractSearchCluster c : searchClusters) {
+ for (AbstractSearchCluster c : searchClusters)
searchClustersByName.put(c.getClusterName(), c);
- }
getCluster().initialize(searchClustersByName);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
index 66294d3fcef..c671749cff0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
@@ -55,9 +55,7 @@ public class ConfigProducerGroup<CHILD extends AbstractConfigProducer<?>> extend
return Collections.unmodifiableCollection(result);
}
- /**
- * @return A map of all components in this group, with (local) component ID as key.
- */
+ /** Returns a map of all components in this group, with (local) component ID as key. */
public Map<ComponentId, CHILD> getComponentMap() {
return Collections.unmodifiableMap(producerById);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
index 62aae85b55a..e07c3216850 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
@@ -11,17 +11,13 @@ import java.util.Collections;
import java.util.List;
/**
- * <p>
* Models a jdisc RequestHandler (including ClientProvider).
* RequestHandlers always have at least one server binding,
* while ClientProviders have at least one client binding.
- * </p>
* <p>
* Note that this is also used to model vespa handlers (which do not have any bindings)
- * </p>
*
* @author gjoranv
- * @since 5.1.6
*/
public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
index c4e988e6749..46ee0d0bd37 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
@@ -9,7 +9,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
* A component that only needs a simple ComponentModel.
*
* @author gjoranv
- * @since 5.1.9
*/
public class SimpleComponent extends Component<AbstractConfigProducer<?>, ComponentModel> {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
index 6b4f8c8f8b5..97a09aa48d8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
@@ -26,7 +26,6 @@ public class Chain<T extends ChainedComponent<?>> extends AbstractConfigProducer
private final ComponentGroup<T> innerComponentsGroup;
private static final Type.Enum TYPE = Type.SEARCH;
-
public Chain(ChainSpecification specWithoutInnerComponents) {
super(specWithoutInnerComponents.componentId.stringValue());
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index 5266fc46f6e..e19d81e7fb2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import com.yahoo.vespa.model.search.StreamingSearchCluster;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
@@ -33,7 +34,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
SemanticRulesConfig.Producer,
PageTemplatesConfig.Producer {
- private final List<AbstractSearchCluster> systems = new LinkedList<>();
+ private ApplicationContainerCluster owningCluster;
+ private final List<AbstractSearchCluster> searchClusters = new LinkedList<>();
private final Options options;
private QueryProfiles queryProfiles;
@@ -42,26 +44,36 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains, Options options) {
super(chains);
+ this.owningCluster = cluster;
this.options = options;
}
public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) {
- systems.addAll(searchClusters.values());
+ this.searchClusters.addAll(searchClusters.values());
+ initializeDispatchers(searchClusters.values());
initializeSearchChains(searchClusters);
}
+ /** Adds a Dispatcher component to the owning container cluster for each search cluster */
+ private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) {
+ for (AbstractSearchCluster searchCluster : searchClusters) {
+ if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue;
+ owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster));
+ }
+ }
+
// public for testing
public void initializeSearchChains(Map<String, ? extends AbstractSearchCluster> searchClusters) {
getChains().initialize(searchClusters);
QrsCache defaultCacheOptions = getOptions().cacheSettings.get("");
if (defaultCacheOptions != null) {
- for (LocalProvider localProvider: getChains().localProviders()) {
+ for (LocalProvider localProvider : getChains().localProviders()) {
localProvider.setCacheSize(defaultCacheOptions.size);
}
}
- for (LocalProvider localProvider: getChains().localProviders()) {
+ for (LocalProvider localProvider : getChains().localProviders()) {
QrsCache cacheOptions = getOptions().cacheSettings.get(localProvider.getClusterName());
if (cacheOptions != null) {
localProvider.setCacheSize(cacheOptions.size);
@@ -83,39 +95,39 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
@Override
public void getConfig(QueryProfilesConfig.Builder builder) {
- if (queryProfiles!=null) {
+ if (queryProfiles != null) {
queryProfiles.getConfig(builder);
}
}
@Override
public void getConfig(SemanticRulesConfig.Builder builder) {
- if (semanticRules!=null) semanticRules.getConfig(builder);
+ if (semanticRules != null) semanticRules.getConfig(builder);
}
@Override
public void getConfig(PageTemplatesConfig.Builder builder) {
- if (pageTemplates!=null) pageTemplates.getConfig(builder);
+ if (pageTemplates != null) pageTemplates.getConfig(builder);
}
@Override
public void getConfig(IndexInfoConfig.Builder builder) {
- for (AbstractSearchCluster sc : systems) {
+ for (AbstractSearchCluster sc : searchClusters) {
sc.getConfig(builder);
}
}
@Override
public void getConfig(IlscriptsConfig.Builder builder) {
- for (AbstractSearchCluster sc : systems) {
+ for (AbstractSearchCluster sc : searchClusters) {
sc.getConfig(builder);
}
}
@Override
public void getConfig(QrSearchersConfig.Builder builder) {
- for (int i = 0; i < systems.size(); i++) {
- AbstractSearchCluster sys = findClusterWithId(systems, i);
+ for (int i = 0; i < searchClusters.size(); i++) {
+ AbstractSearchCluster sys = findClusterWithId(searchClusters, i);
QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
name(sys.getClusterName());
for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) {
@@ -133,9 +145,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
private static AbstractSearchCluster findClusterWithId(List<AbstractSearchCluster> clusters, int index) {
for (AbstractSearchCluster sys : clusters) {
- if (sys.getClusterIndex() == index) {
+ if (sys.getClusterIndex() == index)
return sys;
- }
}
throw new IllegalArgumentException("No search cluster with index " + index + " exists");
}
@@ -144,10 +155,11 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
return options;
}
- /**
- * Struct that encapsulates qrserver options.
- */
+ /** Encapsulates qrserver options. */
public static class Options {
+
Map<String, QrsCache> cacheSettings = new LinkedHashMap<>();
+
}
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
new file mode 100644
index 00000000000..7f51efcd092
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
@@ -0,0 +1,37 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.xml.BundleMapper;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+
+/**
+ * Represents a dispatcher component - an instance of a dispatcher in a search container cluster
+ * knows how to communicate with one indexed search cluster and owns the connections to it.
+ */
+public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel>
+ implements DispatchConfig.Producer {
+
+ private final IndexedSearchCluster indexedSearchCluster;
+
+ public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) {
+ super(toComponentModel(indexedSearchCluster));
+ this.indexedSearchCluster = indexedSearchCluster;
+ }
+
+ private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) {
+ String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher
+ return new ComponentModel(dispatcherComponentId,
+ "com.yahoo.search.dispatch.Dispatcher",
+ BundleMapper.searchAndDocprocBundle,
+ null);
+ }
+
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
+ indexedSearchCluster.getConfig(builder);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
index 875de1465c6..660ee2628b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
@@ -6,6 +6,7 @@ import com.yahoo.search.searchchain.model.federation.FederationOptions;
/**
* A search chain that is intended to be used for federation (i.e. providers, sources)
+ *
* @author Tony Vaagenes
*/
abstract public class GenericTarget extends SearchChain {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
index aa820d1d898..4d02082217c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
@@ -18,7 +18,11 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import com.yahoo.vespa.model.search.SearchNode;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
/**
* Config producer for search chain responsible for sending queries to a local cluster.
@@ -31,8 +35,7 @@ public class LocalProvider extends Provider implements
AttributesConfig.Producer,
QrMonitorConfig.Producer,
RankProfilesConfig.Producer,
- SearchNodesConfig.Producer,
- DispatchConfig.Producer {
+ SearchNodesConfig.Producer {
private final LocalProviderSpec providerSpec;
private volatile AbstractSearchCluster searchCluster;
@@ -122,7 +125,6 @@ public class LocalProvider extends Provider implements
}
public void setSearchCluster(AbstractSearchCluster searchCluster) {
- assert (this.searchCluster == null);
this.searchCluster = searchCluster;
}
@@ -176,13 +178,4 @@ public class LocalProvider extends Provider implements
return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay;
}
- @Override
- public void getConfig(DispatchConfig.Builder builder) {
- if (!(searchCluster instanceof IndexedSearchCluster)) {
- log.warning("Could not build DispatchConfig: Only supported for IndexedSearchCluster, got "
- + searchCluster.getClass().getCanonicalName());
- return;
- }
- ((IndexedSearchCluster) searchCluster).getConfig(builder);
- }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
index 24ff885f131..1520dffb695 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
@@ -29,7 +29,7 @@ public class SearchChains extends Chains<SearchChain> {
LocalClustersCreator.addDefaultLocalProviders(this, searchClustersByName.keySet());
VespaSearchChainsCreator.addVespaSearchChains(this);
- validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers.
+ validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers
initializeComponents(searchClustersByName);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
index d8e678a5cd3..aa7cb5eb539 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
@@ -28,7 +28,6 @@ public class BundleMapper {
public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars"));
-
public static final String searchAndDocprocBundle = "container-search-and-docproc";
private static final Map<String, String> bundleFromClass;
@@ -111,7 +110,6 @@ public class BundleMapper {
bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle);
bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle);
bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
- bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle);
bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle);
bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index fbbf029d5f1..85663a59556 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -29,51 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
protected int index;
private Double visibilityDelay = 0.0;
private List<String> documentNames = new ArrayList<>();
-
- protected List<SearchDefinitionSpec> localSDS = new LinkedList<>();
-
- public void prepareToDistributeFiles(List<SearchNode> backends) {
- for (SearchDefinitionSpec sds : localSDS)
- sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
- }
-
- public static final class IndexingMode {
-
- public static final IndexingMode REALTIME = new IndexingMode("REALTIME");
- public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
-
- private final String name;
-
- private IndexingMode(String name) {
- this.name = name;
- }
-
- public String getName() { return name; }
-
- public String toString() {
- return "indexingmode: " + name;
- }
- }
-
- public static final class SearchDefinitionSpec {
-
- private final SearchDefinition searchDefinition;
- private final UserConfigRepo userConfigRepo;
-
- public SearchDefinitionSpec(SearchDefinition searchDefinition,
- UserConfigRepo userConfigRepo) {
- this.searchDefinition = searchDefinition;
- this.userConfigRepo = userConfigRepo;
- }
-
- public SearchDefinition getSearchDefinition() {
- return searchDefinition;
- }
-
- public UserConfigRepo getUserConfigs() {
- return userConfigRepo;
- }
- }
+ private List<SearchDefinitionSpec> localSDS = new LinkedList<>();
public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
super(parent, "cluster." + clusterName);
@@ -81,6 +37,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
this.index = index;
}
+ public void prepareToDistributeFiles(List<SearchNode> backends) {
+ for (SearchDefinitionSpec sds : localSDS)
+ sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
+ }
+
public void addDocumentNames(SearchDefinition searchDefinition) {
String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
documentNames.add(dName);
@@ -97,14 +58,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
public final String getIndexingModeName() { return getIndexingMode().getName(); }
public final boolean isRealtime() { return getIndexingMode() == IndexingMode.REALTIME; }
public final boolean isStreaming() { return getIndexingMode() == IndexingMode.STREAMING; }
+
public final AbstractSearchCluster setQueryTimeout(Double to) {
this.queryTimeout=to;
return this;
}
+
public final AbstractSearchCluster setVisibilityDelay(double delay) {
this.visibilityDelay=delay;
return this;
}
+
protected abstract IndexingMode getIndexingMode();
public final Double getVisibilityDelay() { return visibilityDelay; }
public final Double getQueryTimeout() { return queryTimeout; }
@@ -122,4 +86,42 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
public abstract void getConfig(RankProfilesConfig.Builder builder);
public abstract void getConfig(AttributesConfig.Builder builder);
+ public static final class IndexingMode {
+
+ public static final IndexingMode REALTIME = new IndexingMode("REALTIME");
+ public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
+
+ private final String name;
+
+ private IndexingMode(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+
+ public String toString() {
+ return "indexingmode: " + name;
+ }
+ }
+
+ public static final class SearchDefinitionSpec {
+
+ private final SearchDefinition searchDefinition;
+ private final UserConfigRepo userConfigRepo;
+
+ public SearchDefinitionSpec(SearchDefinition searchDefinition,
+ UserConfigRepo userConfigRepo) {
+ this.searchDefinition = searchDefinition;
+ this.userConfigRepo = userConfigRepo;
+ }
+
+ public SearchDefinition getSearchDefinition() {
+ return searchDefinition;
+ }
+
+ public UserConfigRepo getUserConfigs() {
+ return userConfigRepo;
+ }
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 40cc09938e2..3ea36dc800e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -30,53 +30,9 @@ import java.util.List;
public class IndexedSearchCluster extends SearchCluster
implements
DocumentdbInfoConfig.Producer,
- // TODO consider removing, these only produced by UnionConfiguration and DocumentDatabase?
IndexInfoConfig.Producer,
IlscriptsConfig.Producer,
- DispatchConfig.Producer
-{
-
- /**
- * Class used to retrieve combined configuration from multiple document databases.
- * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs,
- * that is handled (by delegating to this) by the {@link IndexedSearchCluster}
- * which is the parent to this. This avoids building the config multiple times.
- */
- public static class UnionConfiguration
- extends AbstractConfigProducer
- implements AttributesConfig.Producer {
- private final List<DocumentDatabase> docDbs;
-
- public void getConfig(IndexInfoConfig.Builder builder) {
- for (DocumentDatabase docDb : docDbs) {
- docDb.getConfig(builder);
- }
- }
-
- public void getConfig(IlscriptsConfig.Builder builder) {
- for (DocumentDatabase docDb : docDbs) {
- docDb.getConfig(builder);
- }
- }
-
- @Override
- public void getConfig(AttributesConfig.Builder builder) {
- for (DocumentDatabase docDb : docDbs) {
- docDb.getConfig(builder);
- }
- }
-
- public void getConfig(RankProfilesConfig.Builder builder) {
- for (DocumentDatabase docDb : docDbs) {
- docDb.getConfig(builder);
- }
- }
-
- private UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) {
- super(parent, "union");
- this.docDbs = docDbs;
- }
- }
+ DispatchConfig.Producer {
private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config.
private String indexingChainName = null;
@@ -388,4 +344,46 @@ public class IndexedSearchCluster extends SearchCluster
@Override
public int getRowBits() { return 8; }
+ /**
+ * Class used to retrieve combined configuration from multiple document databases.
+ * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs,
+ * that is handled (by delegating to this) by the {@link IndexedSearchCluster}
+ * which is the parent to this. This avoids building the config multiple times.
+ */
+ public static class UnionConfiguration
+ extends AbstractConfigProducer
+ implements AttributesConfig.Producer {
+ private final List<DocumentDatabase> docDbs;
+
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ private UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) {
+ super(parent, "union");
+ this.docDbs = docDbs;
+ }
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index 62cdcf5a0c6..ecf8f100288 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -166,8 +166,7 @@ public class IndexedTest extends ContentBaseTest {
}
@Test
- public void requireProtonStreamingOnly()
- {
+ public void requireProtonStreamingOnly() {
VespaModel model = getStreamingVespaModel();
// TODO
// HostResource h = model.getHostSystem().getHosts().get(0);
@@ -185,8 +184,7 @@ public class IndexedTest extends ContentBaseTest {
}
@Test
- public void requireCorrectClusterList()
- {
+ public void requireCorrectClusterList() {
VespaModel model = getStreamingVespaModel();
ContentCluster s = model.getContentClusters().get("test");
assertNotNull(s);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
index 4291c68eff8..54ded910bfa 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
@@ -24,7 +24,8 @@ import static org.junit.Assert.assertTrue;
*/
public class ClusterTest {
- static final double DELTA = 1E-12;
+ private static final double DELTA = 1E-12;
+
@Test
public void requireThatContentSearchIsApplied() {
ContentCluster cluster = newContentCluster(joinLines("<search>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
index 007dc5d7266..1c4e005cb67 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.search.test;
+import com.yahoo.component.ComponentId;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.document.DataType;
import com.yahoo.search.config.ClusterConfig;
@@ -9,20 +10,22 @@ import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.document.Attribute;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.DispatchConfig;
import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.SearchCluster;
import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
import org.junit.Test;
-import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
/**
- *
* Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and
* write _unit_ tests. Thanks.
*
@@ -30,32 +33,19 @@ import static org.junit.Assert.*;
*/
public class SearchClusterTest {
- private String vespaHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
- "<hosts> " +
- "<host name=\"foo\">" +
- "<alias>node0</alias>" +
- "</host>" +
- "<host name=\"bar\">" +
- "<alias>node1</alias>" +
- "</host>" +
- "<host name=\"baz\">" +
- "<alias>node2</alias>" +
- "</host>" +
- "</hosts>";
-
@Test
public void testSdConfigLogical() {
// sd1
- SDDocumentType sdt1=new SDDocumentType("s1");
+ SDDocumentType sdt1 = new SDDocumentType("s1");
Search search1 = new Search("s1", null);
- SDField f1=new SDField("f1", DataType.STRING);
+ SDField f1 = new SDField("f1", DataType.STRING);
f1.addAttribute(new Attribute("f1", DataType.STRING));
f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1"))));
sdt1.addField(f1);
search1.addDocument(sdt1);
// sd2
- SDDocumentType sdt2=new SDDocumentType("s2");
+ SDDocumentType sdt2 = new SDDocumentType("s2");
Search search2 = new Search("s2", null);
SDField f2=new SDField("f2", DataType.STRING);
f2.addAttribute(new Attribute("f2", DataType.STRING));
@@ -71,7 +61,21 @@ public class SearchClusterTest {
@Test
public void search_model_is_connected_to_container_clusters_two_content_clusters() {
- String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
+ String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<hosts>" +
+ " <host name='node0host'>" +
+ " <alias>node0</alias>" +
+ " </host>" +
+ " <host name='node1host'>" +
+ " <alias>node1</alias>" +
+ " </host>" +
+ " <host name='node2host'>" +
+ " <alias>node2</alias>" +
+ " </host>" +
+ "</hosts>";
+
+ String services =
+ "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
"<services version=\"1.0\">" +
" <admin version='2.0'>" +
" <adminserver hostalias='node0' />" +
@@ -105,50 +109,74 @@ public class SearchClusterTest {
" <documents>" +
" <document mode='index' type=\"music\" />" +
" </documents>" +
- " <nodes>" +
- " <node hostalias=\"node0\" distribution-key=\"0\" />" +
- " </nodes>" +
+ " <nodes>" +
+ " <node hostalias=\"node0\" distribution-key=\"0\" />" +
+ " </nodes>" +
" </content>" +
" <content id=\"normal\" version='1.0'>" +
" <redundancy>2</redundancy>" +
" <documents>" +
" <document mode='index' type=\"music\" />" +
" </documents>" +
- " <nodes>" +
- " <node hostalias=\"node2\" distribution-key=\"0\" />" +
- " </nodes>" +
+ " <nodes>" +
+ " <node hostalias=\"node2\" distribution-key=\"0\" />" +
+ " </nodes>" +
" </content>" +
"</services>";
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
- ContainerCluster cluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
- assertFalse(cluster1.getSearch().getChains().localProviders().isEmpty());
+ ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
+ assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty());
- ContainerCluster cluster2 = (ContainerCluster)model.getConfigProducer("j2").get();
- assertFalse(cluster2.getSearch().getChains().localProviders().isEmpty());
+ ContainerCluster containerCluster2 = (ContainerCluster)model.getConfigProducer("j2").get();
+ assertFalse(containerCluster2.getSearch().getChains().localProviders().isEmpty());
QrSearchersConfig.Builder builder = new QrSearchersConfig.Builder();
- cluster1.getConfig(builder);
+ containerCluster1.getConfig(builder);
QrSearchersConfig config = new QrSearchersConfig(builder);
- assertThat(config.searchcluster().size(), is(2));
- int normalId = 0;
- int bulkId = 1;
- assertThat(config.searchcluster().get(normalId).name(), is("normal"));
- assertThat(config.searchcluster().get(bulkId).name(), is("xbulk"));
+ assertEquals(2, config.searchcluster().size());
+ int normalIndex = 0;
+ int xbulkIndex = 1;
+ assertEquals("normal", config.searchcluster().get(normalIndex).name());
+ assertEquals("xbulk", config.searchcluster().get(xbulkIndex).name());
ClusterConfig.Builder clusterConfigBuilder = new ClusterConfig.Builder();
model.getConfig(clusterConfigBuilder, "j1/searchchains/chain/normal/component/com.yahoo.prelude.cluster.ClusterSearcher");
ClusterConfig clusterConfig = new ClusterConfig(clusterConfigBuilder);
- assertThat(clusterConfig.clusterId(), is(normalId));
- assertThat(clusterConfig.clusterName(), is("normal"));
+ assertEquals(normalIndex, clusterConfig.clusterId());
+ assertEquals("normal", clusterConfig.clusterName());
ClusterConfig.Builder clusterConfigBuilder2 = new ClusterConfig.Builder();
model.getConfig(clusterConfigBuilder2, "j2/searchchains/chain/xbulk/component/com.yahoo.prelude.cluster.ClusterSearcher");
ClusterConfig clusterConfig2 = new ClusterConfig(clusterConfigBuilder2);
- assertThat(clusterConfig2.clusterId(), is(bulkId));
- assertThat(clusterConfig2.clusterName(), is("xbulk"));
+ assertEquals(xbulkIndex, clusterConfig2.clusterId());
+ assertEquals("xbulk", clusterConfig2.clusterName());
+
+ AbstractSearchCluster searchCluster1 = model.getSearchClusters().get(normalIndex);
+ assertEquals("normal", searchCluster1.getClusterName());
+ assertEquals("normal/search/cluster.normal", searchCluster1.getConfigId());
+ AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex);
+ assertEquals("xbulk", searchCluster2.getClusterName());
+
+ Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal"));
+ assertNotNull(normalDispatcher);
+ assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue());
+ assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue());
+ assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId());
+ DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder();
+ model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal");
+ assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host());
+
+ Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk"));
+ assertNotNull(xbulkDispatcher);
+ assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue());
+ assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue());
+ assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId());
+ DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder();
+ model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk");
+ assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host());
}
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
index 0c8e564578b..57d093fb1c1 100644
--- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
@@ -1,12 +1,11 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.cluster;
-import com.yahoo.cloud.config.ClusterInfoConfig;
import com.yahoo.component.ComponentId;
import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.handler.VipStatus;
-import com.yahoo.jdisc.Metric;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.fastsearch.ClusterParams;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
@@ -22,7 +21,6 @@ import com.yahoo.search.dispatch.Dispatcher;
import com.yahoo.search.query.ParameterParser;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vespa.config.search.DispatchConfig;
import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher;
import org.apache.commons.lang.StringUtils;
@@ -48,7 +46,7 @@ import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.S
@After("*")
public class ClusterSearcher extends Searcher {
- private final String clusterModelName;
+ private final String searchClusterName;
// The set of document types contained in this search cluster
private final Set<String> documentTypes;
@@ -68,16 +66,14 @@ public class ClusterSearcher extends Searcher {
QrSearchersConfig qrsConfig,
ClusterConfig clusterConfig,
DocumentdbInfoConfig documentDbConfig,
- DispatchConfig dispatchConfig,
- ClusterInfoConfig clusterInfoConfig,
- Metric metric,
+ ComponentRegistry<Dispatcher> dispatchers,
FS4ResourcePool fs4ResourcePool,
VipStatus vipStatus) {
super(id);
int searchClusterIndex = clusterConfig.clusterId();
- clusterModelName = clusterConfig.clusterName();
- QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, clusterModelName);
+ searchClusterName = clusterConfig.clusterName();
+ QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName);
documentTypes = new LinkedHashSet<>();
maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT);
@@ -102,8 +98,8 @@ public class ClusterSearcher extends Searcher {
addBackendSearcher(searcher);
vipStatus.addToRotation(searcher.getName());
} else {
- Dispatcher dispatcher = Dispatcher.create(id.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus, metric);
- FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool.getServerId(), docSumParams, documentDbConfig, dispatcher);
+ FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, fs4ResourcePool.getServerId(),
+ docSumParams, documentDbConfig, dispatchers);
addBackendSearcher(searcher);
}
@@ -126,11 +122,17 @@ public class ClusterSearcher extends Searcher {
}
private static FastSearcher searchDispatch(int searchclusterIndex,
+ String searchClusterName,
String serverId,
SummaryParameters docSumParams,
DocumentdbInfoConfig documentdbInfoConfig,
- Dispatcher dispatcher) {
+ ComponentRegistry<Dispatcher> dispatchers) {
ClusterParams clusterParams = makeClusterParams(searchclusterIndex);
+ ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName);
+ Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId);
+ if (dispatcher == null)
+ throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId +
+ " is configured");
return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig);
}
@@ -154,7 +156,7 @@ public class ClusterSearcher extends Searcher {
/** Do not use, for internal testing purposes only. **/
ClusterSearcher(Set<String> documentTypes) {
this.documentTypes = documentTypes;
- clusterModelName = "testScenario";
+ searchClusterName = "testScenario";
maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT;
maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT;
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index ddd319b7bcb..73e5af50788 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -1,7 +1,10 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.dispatch;
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ClusterInfoConfig;
import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
import com.yahoo.container.handler.VipStatus;
import com.yahoo.jdisc.Metric;
import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
@@ -78,22 +81,38 @@ public class Dispatcher extends AbstractComponent {
public static QueryProfileType getArgumentType() { return argumentType; }
- public static Dispatcher create(String clusterId,
- DispatchConfig dispatchConfig,
- int containerClusterSize,
- VipStatus vipStatus,
- Metric metric) {
- var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus);
- var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster);
-
- return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, rpcFactory, metric);
+ @Inject
+ public Dispatcher(ComponentId clusterId,
+ DispatchConfig dispatchConfig,
+ ClusterInfoConfig clusterInfoConfig,
+ VipStatus vipStatus,
+ Metric metric) {
+ this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus),
+ dispatchConfig,
+ metric);
}
- public Dispatcher(SearchCluster searchCluster,
+ private Dispatcher(SearchCluster searchCluster,
DispatchConfig dispatchConfig,
- InvokerFactory invokerFactory,
- PingFactory pingFactory,
Metric metric) {
+ this(searchCluster,
+ dispatchConfig,
+ new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster),
+ metric);
+ }
+
+ protected Dispatcher(SearchCluster searchCluster,
+ DispatchConfig dispatchConfig,
+ RpcInvokerFactory rcpInvokerFactory,
+ Metric metric) {
+ this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric);
+ }
+
+ protected Dispatcher(SearchCluster searchCluster,
+ DispatchConfig dispatchConfig,
+ InvokerFactory invokerFactory,
+ PingFactory pingFactory,
+ Metric metric) {
this.searchCluster = searchCluster;
this.loadBalancer = new LoadBalancer(searchCluster,
dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 242ba5eb818..6160d3dfa08 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -23,6 +23,7 @@ import java.util.concurrent.Callable;
* @author ollivir
*/
public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
+
/** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */
private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries");
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
index 047cdf3991a..7b9eb8fcff1 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
@@ -16,8 +16,9 @@ import java.util.TreeSet;
public class SourcesTarget extends Target {
- private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources =
- new ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>>() {};
+
+ private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {};
+
private SearchChainInvocationSpec defaultProviderSource;
public SourcesTarget(ComponentId sourceId) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index f35d77de01a..6970093fc9d 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -3,6 +3,7 @@ package com.yahoo.prelude.cluster;
import com.yahoo.cloud.config.ClusterInfoConfig;
import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.container.QrConfig;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.handler.ClustersStatus;
@@ -19,6 +20,7 @@ import com.yahoo.prelude.fastsearch.test.MockMetric;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.config.ClusterConfig;
+import com.yahoo.search.dispatch.Dispatcher;
import com.yahoo.search.result.Hit;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.vespa.config.search.DispatchConfig;
@@ -492,8 +494,7 @@ public class ClusterSearcherTestCase {
}
private static ClusterSearcher createSearcher(String clusterName, Double maxQueryTimeout, Double maxQueryCacheTimeout,
- boolean streamingMode, VipStatus vipStatus)
- {
+ boolean streamingMode, VipStatus vipStatus) {
QrSearchersConfig.Builder qrSearchersConfig = new QrSearchersConfig.Builder();
QrSearchersConfig.Searchcluster.Builder searchClusterConfig = new QrSearchersConfig.Searchcluster.Builder();
searchClusterConfig.name(clusterName);
@@ -512,13 +513,19 @@ public class ClusterSearcherTestCase {
DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder();
documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1"));
+ Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"),
+ new DispatchConfig.Builder().build(),
+ createClusterInfoConfig(),
+ vipStatus,
+ new MockMetric());
+ ComponentRegistry<Dispatcher> dispatchers = new ComponentRegistry<>();
+ dispatchers.register(new ComponentId("dispatcher." + clusterName), dispatcher);
+
return new ClusterSearcher(new ComponentId("test-id"),
qrSearchersConfig.build(),
clusterConfig.build(),
documentDbConfig.build(),
- new DispatchConfig.Builder().build(),
- createClusterInfoConfig(),
- new MockMetric(),
+ dispatchers,
new FS4ResourcePool(new QrConfig.Builder().build()),
vipStatus);
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index 4fbbd9dd936..d1e04b26c15 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.config.search.DispatchConfig;
import java.util.List;
class MockDispatcher extends Dispatcher {
+
public static MockDispatcher create(List<Node> nodes) {
var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes));
@@ -30,7 +31,7 @@ class MockDispatcher extends Dispatcher {
}
private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
- super(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric());
+ super(searchCluster, dispatchConfig, invokerFactory, new MockMetric());
}
private static DispatchConfig toDispatchConfig(List<Node> nodes) {
@@ -46,4 +47,5 @@ class MockDispatcher extends Dispatcher {
}
return new DispatchConfig(dispatchConfigBuilder);
}
+
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
index 3d544f5c114..30f6c5a495d 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -38,14 +38,12 @@ public class DispatcherTest {
@Test
public void requireDispatcherToIgnoreMultilevelConfigurations() {
- SearchCluster cl = new MockSearchCluster("1", 2, 2);
- DispatchConfig.Builder builder = new DispatchConfig.Builder();
- builder.useMultilevelDispatch(true);
- DispatchConfig dc = new DispatchConfig(builder);
+ SearchCluster searchCluster = new MockSearchCluster("1", 2, 2);
+ DispatchConfig dispatchConfig = new DispatchConfig.Builder().useMultilevelDispatch(true).build();
- var invokerFactory = new MockInvokerFactory(cl);
+ var invokerFactory = new MockInvokerFactory(searchCluster);
- Dispatcher disp = new Dispatcher(cl, dc, invokerFactory, invokerFactory, new MockMetric());
+ Dispatcher disp = new Dispatcher(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric());
assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false));
}
@@ -113,6 +111,7 @@ public class DispatcherTest {
}
private static class MockInvokerFactory extends InvokerFactory implements PingFactory {
+
private final FactoryStep[] events;
private int step = 0;