When pooling ClusterMonitors and SearchClusters you will add a cluster every time there is a reconfig.

As nothing will purge them, you both keep stuff alive forever and end up with more clusters that you have.
Hence the magic for not removing vipstatus when there are multiple clusters kick in preventing nodes being taken OOR.
Now it is using the ComponentId for identifying a cluster.

2 files changed, 11 insertions, 17 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 708d80d4969..39fc0d31681 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -49,22 +49,14 @@ public class Dispatcher extends AbstractComponent {
     private final RpcResourcePool rpcResourcePool;
     private final boolean multilevelDispatch;
 
-    public Dispatcher(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
-        this.searchCluster = new SearchCluster(dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus);
+    public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
+        this.searchCluster = new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus);
         this.loadBalancer = new LoadBalancer(searchCluster,
                 dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
         this.rpcResourcePool = new RpcResourcePool(dispatchConfig);
         this.multilevelDispatch = dispatchConfig.useMultilevelDispatch();
     }
 
-    /** For testing */
-    public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) {
-        this.searchCluster = null;
-        this.loadBalancer = new LoadBalancer(searchCluster, true);
-        this.rpcResourcePool = new RpcResourcePool(client, nodeConnections);
-        this.multilevelDispatch = false;
-    }
-
     /** Returns the search cluster this dispatches to */
     public SearchCluster searchCluster() {
         return searchCluster;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index b51620a978b..b8d76906f70 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -41,6 +41,7 @@ public class SearchCluster implements NodeManager<Node> {
     private final double minGroupCoverage;
     private final int maxNodesDownPerGroup;
     private final int size;
+    private final String clusterId;
     private final ImmutableMap<Integer, Group> groups;
     private final ImmutableMultimap<String, Node> nodesByHost;
     private final ImmutableList<Group> orderedGroups;
@@ -60,13 +61,14 @@ public class SearchCluster implements NodeManager<Node> {
     // Only needed until query requests are moved to rpc
     private final FS4ResourcePool fs4ResourcePool;
 
-    public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
-        this(dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(),
+    public SearchCluster(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
+        this(clusterId, dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(),
                 toNodes(dispatchConfig), fs4ResourcePool, containerClusterSize, vipStatus);
     }
 
-    public SearchCluster(double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool,
+    public SearchCluster(String clusterId, double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool,
             int containerClusterSize, VipStatus vipStatus) {
+        this.clusterId = clusterId;
         this.minActivedocsCoveragePercentage = minActivedocsCoverage;
         this.minGroupCoverage = minGroupCoverage;
         this.maxNodesDownPerGroup = maxNodesDownPerGroup;
@@ -194,7 +196,7 @@ public class SearchCluster implements NodeManager<Node> {
         node.setWorking(true);
 
         if (usesDirectDispatchTo(node))
-            vipStatus.addToRotation(this);
+            vipStatus.addToRotation(clusterId);
     }
 
     /** Used by the cluster monitor to manage node status */
@@ -204,16 +206,16 @@ public class SearchCluster implements NodeManager<Node> {
 
         // Take ourselves out if we usually dispatch only to our own host
         if (usesDirectDispatchTo(node))
-            vipStatus.removeFromRotation(this);
+            vipStatus.removeFromRotation(clusterId);
     }
 
     private void updateSufficientCoverage(Group group, boolean sufficientCoverage) {
         // update VIP status if we direct dispatch to this group and coverage status changed
         if (usesDirectDispatchTo(group) && sufficientCoverage != group.hasSufficientCoverage()) {
             if (sufficientCoverage) {
-                vipStatus.addToRotation(this);
+                vipStatus.addToRotation(clusterId);
             } else {
-                vipStatus.removeFromRotation(this);
+                vipStatus.removeFromRotation(clusterId);
             }
         }
         group.setHasSufficientCoverage(sufficientCoverage);