Create one Dispatcher component per search cluster

This avoids creating an excessive number of connections
to search clusters when the application (incorrectly) creates
many local provider chains to the same search cluster.

7 files changed, 70 insertions, 39 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
index 0c8e564578b..57d093fb1c1 100644
--- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
@@ -1,12 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.cluster;
 
-import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.handler.VipStatus;
-import com.yahoo.jdisc.Metric;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.fastsearch.ClusterParams;
 import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
@@ -22,7 +21,6 @@ import com.yahoo.search.dispatch.Dispatcher;
 import com.yahoo.search.query.ParameterParser;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vespa.config.search.DispatchConfig;
 import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher;
 import org.apache.commons.lang.StringUtils;
 
@@ -48,7 +46,7 @@ import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.S
 @After("*")
 public class ClusterSearcher extends Searcher {
 
-    private final String clusterModelName;
+    private final String searchClusterName;
 
     // The set of document types contained in this search cluster
     private final Set<String> documentTypes;
@@ -68,16 +66,14 @@ public class ClusterSearcher extends Searcher {
                            QrSearchersConfig qrsConfig,
                            ClusterConfig clusterConfig,
                            DocumentdbInfoConfig documentDbConfig,
-                           DispatchConfig dispatchConfig,
-                           ClusterInfoConfig clusterInfoConfig,
-                           Metric metric,
+                           ComponentRegistry<Dispatcher> dispatchers,
                            FS4ResourcePool fs4ResourcePool,
                            VipStatus vipStatus) {
         super(id);
 
         int searchClusterIndex = clusterConfig.clusterId();
-        clusterModelName = clusterConfig.clusterName();
-        QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, clusterModelName);
+        searchClusterName = clusterConfig.clusterName();
+        QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName);
         documentTypes = new LinkedHashSet<>();
 
         maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT);
@@ -102,8 +98,8 @@ public class ClusterSearcher extends Searcher {
             addBackendSearcher(searcher);
             vipStatus.addToRotation(searcher.getName());
         } else {
-            Dispatcher dispatcher = Dispatcher.create(id.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus, metric);
-            FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool.getServerId(), docSumParams, documentDbConfig, dispatcher);
+            FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, fs4ResourcePool.getServerId(),
+                                                   docSumParams, documentDbConfig, dispatchers);
             addBackendSearcher(searcher);
 
         }
@@ -126,11 +122,17 @@ public class ClusterSearcher extends Searcher {
     }
 
     private static FastSearcher searchDispatch(int searchclusterIndex,
+                                               String searchClusterName,
                                                String serverId,
                                                SummaryParameters docSumParams,
                                                DocumentdbInfoConfig documentdbInfoConfig,
-                                               Dispatcher dispatcher) {
+                                               ComponentRegistry<Dispatcher> dispatchers) {
         ClusterParams clusterParams = makeClusterParams(searchclusterIndex);
+        ComponentId dispatcherComponentId = new ComponentId("dispatcher." + searchClusterName);
+        Dispatcher dispatcher = dispatchers.getComponent(dispatcherComponentId);
+        if (dispatcher == null)
+            throw new IllegalArgumentException("Configuration error: No dispatcher " + dispatcherComponentId +
+                                               " is configured");
         return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig);
     }
 
@@ -154,7 +156,7 @@ public class ClusterSearcher extends Searcher {
     /** Do not use, for internal testing purposes only. **/
     ClusterSearcher(Set<String> documentTypes) {
         this.documentTypes = documentTypes;
-        clusterModelName = "testScenario";
+        searchClusterName = "testScenario";
         maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT;
         maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT;
     }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index ddd319b7bcb..73e5af50788 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -1,7 +1,10 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch;
 
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
@@ -78,22 +81,38 @@ public class Dispatcher extends AbstractComponent {
 
     public static QueryProfileType getArgumentType() { return argumentType; }
 
-    public static Dispatcher create(String clusterId,
-                                    DispatchConfig dispatchConfig,
-                                    int containerClusterSize,
-                                    VipStatus vipStatus,
-                                    Metric metric) {
-        var searchCluster = new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus);
-        var rpcFactory = new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster);
-
-        return new Dispatcher(searchCluster, dispatchConfig, rpcFactory, rpcFactory, metric);
+    @Inject
+    public Dispatcher(ComponentId clusterId,
+                      DispatchConfig dispatchConfig,
+                      ClusterInfoConfig clusterInfoConfig,
+                      VipStatus vipStatus,
+                      Metric metric) {
+        this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus),
+             dispatchConfig,
+             metric);
     }
 
-    public Dispatcher(SearchCluster searchCluster,
+    private Dispatcher(SearchCluster searchCluster,
                       DispatchConfig dispatchConfig,
-                      InvokerFactory invokerFactory,
-                      PingFactory pingFactory,
                       Metric metric) {
+        this(searchCluster,
+             dispatchConfig,
+             new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster),
+             metric);
+    }
+
+    protected Dispatcher(SearchCluster searchCluster,
+                         DispatchConfig dispatchConfig,
+                         RpcInvokerFactory rcpInvokerFactory,
+                         Metric metric) {
+        this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric);
+    }
+
+    protected Dispatcher(SearchCluster searchCluster,
+                         DispatchConfig dispatchConfig,
+                         InvokerFactory invokerFactory,
+                         PingFactory pingFactory,
+                         Metric metric) {
         this.searchCluster = searchCluster;
         this.loadBalancer = new LoadBalancer(searchCluster,
                                   dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 242ba5eb818..6160d3dfa08 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -23,6 +23,7 @@ import java.util.concurrent.Callable;
  * @author ollivir
  */
 public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
+
     /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */
     private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries");
 
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
index 047cdf3991a..7b9eb8fcff1 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
@@ -16,8 +16,9 @@ import java.util.TreeSet;
 
 
 public class SourcesTarget extends Target {
-    private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources =
-            new ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>>() {};
+
+    private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {};
+
     private SearchChainInvocationSpec defaultProviderSource;
 
     public SourcesTarget(ComponentId sourceId) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index f35d77de01a..6970093fc9d 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -3,6 +3,7 @@ package com.yahoo.prelude.cluster;
 
 import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.QrConfig;
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.handler.ClustersStatus;
@@ -19,6 +20,7 @@ import com.yahoo.prelude.fastsearch.test.MockMetric;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.config.ClusterConfig;
+import com.yahoo.search.dispatch.Dispatcher;
 import com.yahoo.search.result.Hit;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.vespa.config.search.DispatchConfig;
@@ -492,8 +494,7 @@ public class ClusterSearcherTestCase {
     }
 
     private static ClusterSearcher createSearcher(String clusterName, Double maxQueryTimeout, Double maxQueryCacheTimeout,
-                                                  boolean streamingMode, VipStatus vipStatus)
-    {
+                                                  boolean streamingMode, VipStatus vipStatus) {
         QrSearchersConfig.Builder qrSearchersConfig = new QrSearchersConfig.Builder();
         QrSearchersConfig.Searchcluster.Builder searchClusterConfig = new QrSearchersConfig.Searchcluster.Builder();
         searchClusterConfig.name(clusterName);
@@ -512,13 +513,19 @@ public class ClusterSearcherTestCase {
         DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder();
         documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1"));
 
+        Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"),
+                                               new DispatchConfig.Builder().build(),
+                                               createClusterInfoConfig(),
+                                               vipStatus,
+                                               new MockMetric());
+        ComponentRegistry<Dispatcher> dispatchers = new ComponentRegistry<>();
+        dispatchers.register(new ComponentId("dispatcher." + clusterName), dispatcher);
+
         return new ClusterSearcher(new ComponentId("test-id"),
                                    qrSearchersConfig.build(),
                                    clusterConfig.build(),
                                    documentDbConfig.build(),
-                                   new DispatchConfig.Builder().build(),
-                                   createClusterInfoConfig(),
-                                   new MockMetric(),
+                                   dispatchers,
                                    new FS4ResourcePool(new QrConfig.Builder().build()),
                                    vipStatus);
     }
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index 4fbbd9dd936..d1e04b26c15 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.config.search.DispatchConfig;
 import java.util.List;
 
 class MockDispatcher extends Dispatcher {
+
     public static MockDispatcher create(List<Node> nodes) {
         var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes));
 
@@ -30,7 +31,7 @@ class MockDispatcher extends Dispatcher {
     }
 
     private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
-        super(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric());
+        super(searchCluster, dispatchConfig, invokerFactory, new MockMetric());
     }
 
     private static DispatchConfig toDispatchConfig(List<Node> nodes) {
@@ -46,4 +47,5 @@ class MockDispatcher extends Dispatcher {
         }
         return new DispatchConfig(dispatchConfigBuilder);
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
index 3d544f5c114..30f6c5a495d 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -38,14 +38,12 @@ public class DispatcherTest {
 
     @Test
     public void requireDispatcherToIgnoreMultilevelConfigurations() {
-        SearchCluster cl = new MockSearchCluster("1", 2, 2);
-        DispatchConfig.Builder builder = new DispatchConfig.Builder();
-        builder.useMultilevelDispatch(true);
-        DispatchConfig dc = new DispatchConfig(builder);
+        SearchCluster searchCluster = new MockSearchCluster("1", 2, 2);
+        DispatchConfig dispatchConfig = new DispatchConfig.Builder().useMultilevelDispatch(true).build();
 
-        var invokerFactory = new MockInvokerFactory(cl);
+        var invokerFactory = new MockInvokerFactory(searchCluster);
 
-        Dispatcher disp = new Dispatcher(cl, dc, invokerFactory, invokerFactory, new MockMetric());
+        Dispatcher disp = new Dispatcher(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric());
         assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false));
     }
 
@@ -113,6 +111,7 @@ public class DispatcherTest {
     }
 
     private static class MockInvokerFactory extends InvokerFactory implements PingFactory {
+
         private final FactoryStep[] events;
         private int step = 0;