Merge pull request #23617 from vespa-engine/hmusum/cleanup-12

Cluster controller cleanup [run-systemtest]

7 files changed, 112 insertions, 182 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
index f2a4a9736c3..695fecb6314 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java
@@ -1,20 +1,14 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
 package com.yahoo.vespa.clustercontroller.core;
 
 import com.yahoo.vdslib.distribution.ConfiguredNode;
 import com.yahoo.vdslib.distribution.Distribution;
-import com.yahoo.vdslib.distribution.Group;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vdslib.state.Node;
 import com.yahoo.vdslib.state.NodeState;
-import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
 import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener;
-import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable;
-import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer;
 import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest;
-
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -46,61 +40,6 @@ public class ContentCluster {
         setNodes(configuredNodes, new NodeListener() {});
     }
 
-    // TODO move out, this doesn't belong in a domain model class
-    public void writeHtmlState(
-            final VdsClusterHtmlRenderer vdsClusterHtmlRenderer,
-            final StringBuilder sb,
-            final Timer timer,
-            final ClusterStateBundle state,
-            final ClusterStatsAggregator statsAggregator,
-            final Distribution distribution,
-            final FleetControllerOptions options,
-            final EventLog eventLog) {
-
-        final VdsClusterHtmlRenderer.Table table =
-                vdsClusterHtmlRenderer.createNewClusterHtmlTable(clusterName, slobrokGenerationCount);
-
-        if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null
-            table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n");
-            table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n",
-                                          HtmlTable.escape(state.getFeedBlockOrNull().getDescription())));
-        }
-
-        final List<Group> groups = LeafGroups.enumerateFrom(distribution.getRootGroup());
-
-        for (int j=0; j<groups.size(); ++j) {
-            final Group group = groups.get(j);
-            assert(group != null);
-            final String localName = group.getUnixStylePath();
-            assert(localName != null);
-            final TreeMap<Integer, NodeInfo> storageNodeInfoByIndex = new TreeMap<>();
-            final TreeMap<Integer, NodeInfo> distributorNodeInfoByIndex = new TreeMap<>();
-            for (ConfiguredNode configuredNode : group.getNodes()) {
-                storeNodeInfo(configuredNode.index(), NodeType.STORAGE, storageNodeInfoByIndex);
-                storeNodeInfo(configuredNode.index(), NodeType.DISTRIBUTOR, distributorNodeInfoByIndex);
-            }
-            table.renderNodes(
-                    storageNodeInfoByIndex,
-                    distributorNodeInfoByIndex,
-                    timer,
-                    state,
-                    statsAggregator,
-                    options.minMergeCompletionRatio,
-                    options.maxPrematureCrashes,
-                    options.clusterFeedBlockLimit,
-                    eventLog,
-                    clusterName,
-                    localName);
-        }
-        table.addTable(sb, options.stableStateTimePeriod);
-    }
-
-    private void storeNodeInfo(int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) {
-        NodeInfo nodeInfo = getNodeInfo(new Node(nodeType, nodeIndex));
-        if (nodeInfo == null) return;
-        nodeInfoByIndex.put(nodeIndex, nodeInfo);
-    }
-
     public Distribution getDistribution() { return distribution; }
 
     public void setDistribution(Distribution distribution) {
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
index 7f385c6077c..8b335e877cd 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
@@ -12,8 +12,8 @@ import com.yahoo.vdslib.state.State;
 import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler;
 import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo;
-import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener;
 import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener;
+import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener;
 import com.yahoo.vespa.clustercontroller.core.listeners.SystemStateListener;
 import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator;
 import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer;
@@ -27,7 +27,6 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageRespon
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServerInterface;
 import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter;
-
 import java.io.FileNotFoundException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -45,7 +44,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class FleetController implements NodeListener, SlobrokListener, SystemStateListener,
                                         Runnable, RemoteClusterControllerTaskScheduler {
@@ -155,10 +153,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta
                 new ClusterStateRequestHandler(stateVersionTracker));
         this.statusRequestRouter.addHandler(
                 "^/$",
-                new LegacyIndexPageRequestHandler(
-                    timer, options.showLocalSystemStatesInEventLog, cluster,
-                    masterElectionHandler, stateVersionTracker,
-                    eventLog, timer.getCurrentTimeInMillis(), dataExtractor));
+                new LegacyIndexPageRequestHandler(timer, cluster, masterElectionHandler, stateVersionTracker, eventLog,
+                                                  timer.getCurrentTimeInMillis(), dataExtractor));
 
         propagateOptions();
     }
@@ -505,9 +501,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta
             cluster.setSlobrokGenerationCount(0);
         }
 
-        configuredBucketSpaces = Collections.unmodifiableSet(
-                Stream.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace())
-                        .collect(Collectors.toSet()));
+        configuredBucketSpaces = Set.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace());
         stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio);
 
         communicator.propagateOptions(options);
@@ -634,7 +628,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta
             didWork |= metricUpdater.forWork("processAnyPendingStatusPageRequest", this::processAnyPendingStatusPageRequest);
             if ( ! isRunning()) { return; }
             if (rpcServer != null) {
-                didWork |= metricUpdater.forWork("handleRpcRequests", () -> rpcServer.handleRpcRequests(cluster, consolidatedClusterState(), this, this));
+                didWork |= metricUpdater.forWork("handleRpcRequests", () -> rpcServer.handleRpcRequests(cluster, consolidatedClusterState(), this));
             }
 
             if ( ! isRunning()) { return; }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java
index a5a29b8d7f1..502fc37dead 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java
@@ -11,10 +11,9 @@ import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
 import com.yahoo.jrt.Transport;
 import com.yahoo.jrt.Values;
-import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.ClusterState;
+import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.State;
-import java.util.logging.Level;
 import com.yahoo.vespa.clustercontroller.core.ActivateClusterStateVersionRequest;
 import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle;
 import com.yahoo.vespa.clustercontroller.core.Communicator;
@@ -23,7 +22,7 @@ import com.yahoo.vespa.clustercontroller.core.GetNodeStateRequest;
 import com.yahoo.vespa.clustercontroller.core.NodeInfo;
 import com.yahoo.vespa.clustercontroller.core.SetClusterStateRequest;
 import com.yahoo.vespa.clustercontroller.core.Timer;
-
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -71,7 +70,7 @@ public class RPCCommunicator implements Communicator {
         checkArgument(nodeStateRequestTimeoutIntervalStartPercentage >= 0);
         checkArgument(nodeStateRequestTimeoutIntervalStartPercentage <= 100);
         checkArgument(nodeStateRequestTimeoutIntervalStopPercentage >= nodeStateRequestTimeoutIntervalStartPercentage);
-        checkArgument(nodeStateRequestTimeoutIntervalStartPercentage <= 100);
+        checkArgument(nodeStateRequestTimeoutIntervalStopPercentage <= 100);
         checkArgument(nodeStateRequestRoundTripTimeMaxSeconds >= 0);
         this.nodeStateRequestTimeoutIntervalMaxSeconds = nodeStateRequestTimeoutIntervalMaxMs / 1000D;
         this.nodeStateRequestTimeoutIntervalStartPercentage = nodeStateRequestTimeoutIntervalStartPercentage;
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java
index 6e416ce4906..dc21693dcdb 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java
@@ -15,8 +15,6 @@ import com.yahoo.jrt.Transport;
 import com.yahoo.jrt.slobrok.api.BackOffPolicy;
 import com.yahoo.jrt.slobrok.api.Register;
 import com.yahoo.jrt.slobrok.api.SlobrokList;
-import java.util.logging.Level;
-
 import com.yahoo.net.HostName;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vdslib.state.Node;
@@ -27,34 +25,34 @@ import com.yahoo.vespa.clustercontroller.core.ContentCluster;
 import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler;
 import com.yahoo.vespa.clustercontroller.core.NodeInfo;
 import com.yahoo.vespa.clustercontroller.core.Timer;
-import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener;
 import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener;
-
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 public class RpcServer {
 
-    private static Logger log = Logger.getLogger(RpcServer.class.getName());
+    private static final Logger log = Logger.getLogger(RpcServer.class.getName());
 
     private final Timer timer;
     private final Object monitor;
     private final String clusterName;
     private final int fleetControllerIndex;
-    private String slobrokConnectionSpecs[];
+    private String[] slobrokConnectionSpecs;
     private int port = 0;
     private Supervisor supervisor;
     private Acceptor acceptor;
     private Register register;
     private final List<Request> rpcRequests = new LinkedList<>();
     private MasterElectionHandler masterHandler;
-    private BackOffPolicy slobrokBackOffPolicy;
+    private final BackOffPolicy slobrokBackOffPolicy;
     private long lastConnectErrorTime = 0;
     private String lastConnectError = "";
 
@@ -81,10 +79,10 @@ public class RpcServer {
         return "storage/cluster." + clusterName + "/fleetcontroller/" + fleetControllerIndex;
     }
 
-    public void setSlobrokConnectionSpecs(String slobrokConnectionSpecs[], int port) throws ListenFailedException, UnknownHostException {
-        if (this.slobrokConnectionSpecs == null || !this.slobrokConnectionSpecs.equals(slobrokConnectionSpecs) // TODO: <-- probably a bug
-            || this.port != port)
-        {
+    public void setSlobrokConnectionSpecs(String[] slobrokConnectionSpecs, int port) throws ListenFailedException, UnknownHostException {
+        if (this.slobrokConnectionSpecs == null
+                || !Arrays.equals(this.slobrokConnectionSpecs, slobrokConnectionSpecs)
+                || this.port != port) {
             this.slobrokConnectionSpecs = slobrokConnectionSpecs;
             this.port = port;
             disconnect();
@@ -105,7 +103,7 @@ public class RpcServer {
         log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port);
         acceptor = supervisor.listen(new Spec(port));
         log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port());
-        StringBuffer slobroks = new StringBuffer("(");
+        StringBuilder slobroks = new StringBuilder("(");
         for (String s : slobrokConnectionSpecs) {
             slobroks.append(" ").append(s);
         }
@@ -185,10 +183,7 @@ public class RpcServer {
         }
     }
 
-    public boolean handleRpcRequests(ContentCluster cluster, ClusterState systemState,
-                                     NodeListener changeListener,
-                                     SlobrokListener addedListener)
-    {
+    public boolean handleRpcRequests(ContentCluster cluster, ClusterState systemState, NodeListener changeListener) {
         boolean handledAnyRequests = false;
         if (!isConnected()) {
             long time = timer.getCurrentTimeInMillis();
@@ -255,8 +250,6 @@ public class RpcServer {
                     NodeType nodeType = NodeType.get(req.parameters().get(0).asString());
                     int nodeIndex = req.parameters().get(1).asInt32();
                     Node node = new Node(nodeType, nodeIndex);
-                    // First parameter is current state in system state
-                    NodeState ns = systemState.getNodeState(node);
                     req.returnValues().add(new StringValue(systemState.getNodeState(node).serialize()));
                     // Second parameter is state node is reporting
                     NodeInfo nodeInfo = cluster.getNodeInfo(node);
@@ -276,7 +269,7 @@ public class RpcServer {
                         throw new IllegalStateException("Invalid slobrok address '" + slobrokAddress + "'.");
                     }
                     NodeType nodeType = NodeType.get(slobrokAddress.substring(nextButLastSlash + 1, lastSlash));
-                    Integer nodeIndex = Integer.valueOf(slobrokAddress.substring(lastSlash + 1));
+                    int nodeIndex = Integer.parseInt(slobrokAddress.substring(lastSlash + 1));
                     NodeInfo node = cluster.getNodeInfo(new Node(nodeType, nodeIndex));
                     if (node == null)
                         throw new IllegalStateException("Cannot set wanted state of node " + new Node(nodeType, nodeIndex) + ". Index does not correspond to a configured node.");
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
index 378f65f7235..96dc114c734 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java
@@ -1,14 +1,29 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core.status;
 
-import com.yahoo.vdslib.state.ClusterState;
-import com.yahoo.vespa.clustercontroller.core.*;
+import com.yahoo.vdslib.distribution.ConfiguredNode;
+import com.yahoo.vdslib.distribution.Group;
+import com.yahoo.vdslib.state.Node;
+import com.yahoo.vdslib.state.NodeType;
+import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle;
+import com.yahoo.vespa.clustercontroller.core.ClusterStateHistoryEntry;
+import com.yahoo.vespa.clustercontroller.core.ContentCluster;
+import com.yahoo.vespa.clustercontroller.core.EventLog;
+import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions;
+import com.yahoo.vespa.clustercontroller.core.LeafGroups;
+import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler;
+import com.yahoo.vespa.clustercontroller.core.NodeInfo;
+import com.yahoo.vespa.clustercontroller.core.RealTimer;
+import com.yahoo.vespa.clustercontroller.core.StateVersionTracker;
 import com.yahoo.vespa.clustercontroller.core.Timer;
+import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer;
 import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer;
-
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.TreeMap;
 
 /**
 * @author Haakon Humberset
@@ -22,15 +37,15 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
     private final EventLog eventLog;
     private final long startedTime;
     private final RunDataExtractor data;
-    private final boolean showLocalSystemStatesInLog;
 
-    public LegacyIndexPageRequestHandler(Timer timer, boolean showLocalSystemStatesInLog, ContentCluster cluster,
+    public LegacyIndexPageRequestHandler(Timer timer,
+                                         ContentCluster cluster,
                                          MasterElectionHandler masterElectionHandler,
                                          StateVersionTracker stateVersionTracker,
-                                         EventLog eventLog, long startedTime, RunDataExtractor data)
-    {
+                                         EventLog eventLog,
+                                         long startedTime,
+                                         RunDataExtractor data) {
         this.timer = timer;
-        this.showLocalSystemStatesInLog = showLocalSystemStatesInLog;
         this.cluster = cluster;
         this.masterElectionHandler = masterElectionHandler;
         this.stateVersionTracker = stateVersionTracker;
@@ -59,18 +74,9 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         content.append("<tr><td>Cluster controller uptime:</td><td align=\"right\">" + RealTimer.printDuration(currentTime - startedTime) + "</td></tr></table>");
         if (masterElectionHandler.isAmongNthFirst(data.getOptions().stateGatherCount)) {
             // Table overview of all the nodes
-            cluster.writeHtmlState(
-                    new VdsClusterHtmlRenderer(),
-                    content,
-                    timer,
-                    stateVersionTracker.getVersionedClusterStateBundle(),
-                    stateVersionTracker.getAggregatedClusterStats(),
-                    data.getOptions().storageDistribution,
-                    data.getOptions(),
-                    eventLog
-            );
+            writeHtmlState(cluster, content, timer, stateVersionTracker, data.getOptions(), eventLog);
             // Current cluster state and cluster state history
-            writeHtmlState(stateVersionTracker, content, request);
+            writeHtmlState(stateVersionTracker, content);
         } else {
             // Overview of current config
             data.getOptions().writeHtmlState(content);
@@ -87,14 +93,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         return response;
     }
 
-    public void writeHtmlState(StateVersionTracker stateVersionTracker, StringBuilder sb, StatusPageServer.HttpRequest request) {
-        boolean showLocal = showLocalSystemStatesInLog;
-        if (request.hasQueryParameter("showlocal")) {
-            showLocal = true;
-        } else if (request.hasQueryParameter("hidelocal")) {
-            showLocal = false;
-        }
-
+    public void writeHtmlState(StateVersionTracker stateVersionTracker, StringBuilder sb) {
         sb.append("<h2 id=\"clusterstates\">Cluster states</h2>\n");
         writeClusterStates(sb, stateVersionTracker.getVersionedClusterStateBundle());
 
@@ -153,4 +152,53 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa
         sb.append("</td></tr>\n");
     }
 
+    private void writeHtmlState(ContentCluster cluster,
+                                StringBuilder sb,
+                                Timer timer,
+                                StateVersionTracker stateVersionTracker,
+                                FleetControllerOptions options,
+                                EventLog eventLog) {
+        VdsClusterHtmlRenderer renderer = new VdsClusterHtmlRenderer();
+        VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(cluster.getName(), cluster.getSlobrokGenerationCount());
+
+        ClusterStateBundle state = stateVersionTracker.getVersionedClusterStateBundle();
+        if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null
+            table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n");
+            table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n",
+                                          HtmlTable.escape(state.getFeedBlockOrNull().getDescription())));
+        }
+
+        List<Group> groups = LeafGroups.enumerateFrom(options.storageDistribution.getRootGroup());
+
+        for (Group group : groups) {
+            assert (group != null);
+            String localName = group.getUnixStylePath();
+            assert (localName != null);
+            TreeMap<Integer, NodeInfo> storageNodeInfoByIndex = new TreeMap<>();
+            TreeMap<Integer, NodeInfo> distributorNodeInfoByIndex = new TreeMap<>();
+            for (ConfiguredNode configuredNode : group.getNodes()) {
+                storeNodeInfo(cluster, configuredNode.index(), NodeType.STORAGE, storageNodeInfoByIndex);
+                storeNodeInfo(cluster, configuredNode.index(), NodeType.DISTRIBUTOR, distributorNodeInfoByIndex);
+            }
+            table.renderNodes(storageNodeInfoByIndex,
+                              distributorNodeInfoByIndex,
+                              timer,
+                              state,
+                              stateVersionTracker.getAggregatedClusterStats(),
+                              options.minMergeCompletionRatio,
+                              options.maxPrematureCrashes,
+                              options.clusterFeedBlockLimit,
+                              eventLog,
+                              cluster.getName(),
+                              localName);
+        }
+        table.addTable(sb, options.stableStateTimePeriod);
+    }
+
+    private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) {
+        NodeInfo nodeInfo = cluster.getNodeInfo(new Node(nodeType, nodeIndex));
+        if (nodeInfo == null) return;
+        nodeInfoByIndex.put(nodeIndex, nodeInfo);
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java
index c167c82aa90..af067cc394f 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java
@@ -21,7 +21,6 @@ import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
 import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator;
 import com.yahoo.vespa.clustercontroller.core.rpc.RPCUtil;
-
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -39,7 +38,7 @@ import java.util.stream.Collectors;
  */
 public class DummyVdsNode {
 
-    public static Logger log = Logger.getLogger(DummyVdsNode.class.getName());
+    private static final Logger log = Logger.getLogger(DummyVdsNode.class.getName());
 
     private final String[] slobrokConnectionSpecs;
     private final String clusterName;
@@ -55,7 +54,7 @@ public class DummyVdsNode {
     private final Timer timer;
     private boolean failSetSystemStateRequests = false;
     private boolean resetTimestampOnReconnect = false;
-    private final Map<Node, Long> highestStartTimestamps = new TreeMap<Node, Long>();
+    private final Map<Node, Long> highestStartTimestamps = new TreeMap<>();
     int timedOutStateReplies = 0;
     int outdatedStateReplies = 0;
     int immediateStateReplies = 0;
@@ -88,7 +87,7 @@ public class DummyVdsNode {
 
     private final Thread messageResponder = new Thread() {
         public void run() {
-            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread");
+            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": starting message responder thread");
             while (true) {
                 synchronized (timer) {
                     if (isInterrupted()) break;
@@ -96,7 +95,7 @@ public class DummyVdsNode {
                     for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) {
                         Req r = it.next();
                         if (r.timeout <= currentTime) {
-                            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime);
+                            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state request at time " + currentTime);
                             r.request.returnValues().add(new StringValue(nodeState.serialize()));
                             if (r.request.methodName().equals("getnodestate3")) {
                                 r.request.returnValues().add(new StringValue("No host info in dummy implementation"));
@@ -113,7 +112,7 @@ public class DummyVdsNode {
                     }
                 }
             }
-            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread");
+            log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": shut down message responder thread");
         }
     };
 
@@ -170,31 +169,16 @@ public class DummyVdsNode {
         registeredInSlobrok = false;
     }
 
-    void disconnect() { disconnectImmediately(); }
-    void disconnectImmediately() { disconnect(false, 0, false);  }
-    void disconnectBreakConnection() { disconnect(true, FleetControllerTest.timeoutMS, false); }
-    void disconnectAsShutdown() { disconnect(true, FleetControllerTest.timeoutMS, true); }
-    private void disconnect(boolean waitForPendingNodeStateRequest, long timeoutms, boolean setStoppingStateFirst) {
-        log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : ""));
-        if (waitForPendingNodeStateRequest) {
-            this.waitForPendingGetNodeStateRequest(timeoutms);
-        }
-        if (setStoppingStateFirst) {
-            NodeState newState = nodeState.clone();
-            newState.setState(State.STOPPING);
-            // newState.setDescription("Received signal 15 (SIGTERM - Termination signal)");
-            // Altered in storageserver implementation. Updating now to fit
-            newState.setDescription("controlled shutdown");
-            setNodeState(newState);
-            // Sleep a bit in hopes of answer being written before shutting down socket
-            try{ Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ }
-        }
+    void disconnectImmediately() { disconnect();  }
+
+    void disconnect() {
+        log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Breaking connection.");
         if (supervisor == null) return;
         register.shutdown();
         acceptor.shutdown().join();
         supervisor.transport().shutdown().join();
         supervisor = null;
-        log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection.");
+        log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Done breaking connection.");
     }
 
     public String toString() {
@@ -210,11 +194,11 @@ public class DummyVdsNode {
 
     public int getStateCommunicationVersion() { return stateCommunicationVersion; }
 
-    void waitForSystemStateVersion(int version, long timeout) {
+    void waitForSystemStateVersion(int version) {
         try {
             long startTime = System.currentTimeMillis();
             while (getLatestSystemStateVersion().orElse(-1) < version) {
-                if ( (System.currentTimeMillis() - startTime) > timeout)
+                if ( (System.currentTimeMillis() - startTime) > (long) FleetControllerTest.timeoutMS)
                     throw new RuntimeException("Timed out waiting for state version " + version + " in " + this);
                 Thread.sleep(10);
             }
@@ -237,33 +221,6 @@ public class DummyVdsNode {
         }
     }
 
-    private void waitForPendingGetNodeStateRequest(long timeout) {
-        long startTime = System.currentTimeMillis();
-        long endTime = startTime + timeout;
-        log.log(Level.FINE, () -> "Dummy node " + this + " waiting for pending node state request.");
-        while (true) {
-            synchronized(timer) {
-                if (!waitingRequests.isEmpty()) {
-                    log.log(Level.FINE, () -> "Dummy node " + this + " has pending request, returning.");
-                    return;
-                }
-                try {
-                    log.log(Level.FINE, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request.");
-                    timer.wait(endTime - startTime);
-                } catch (InterruptedException e) { /* ignore */ }
-                log.log(Level.FINE, () -> "Dummy node " + this + " woke up to recheck.");
-            }
-            startTime = System.currentTimeMillis();
-            if (startTime >= endTime) {
-                log.log(Level.FINE, () -> "Dummy node " + this + " timeout passed. Don't have pending request.");
-                if (!waitingRequests.isEmpty()) {
-                    log.log(Level.FINE, () -> "Dummy node " + this + ". Non-empty set of waiting requests");
-                }
-                throw new IllegalStateException("Timeout. No pending get node state request pending after waiting " + timeout + " milliseconds.");
-            }
-        }
-    }
-
     void replyToPendingNodeStateRequests() {
         for(Req req : waitingRequests) {
             log.log(Level.FINE, () -> "Dummy node " + this + " answering pending node state request.");
@@ -422,7 +379,7 @@ public class DummyVdsNode {
         for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) {
              Req r = it.next();
              if (r.request.parameters().size() > 2 && r.request.parameters().get(2).asInt32() == index) {
-                 log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one");
+                 log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state reply from controller " + index + " as we received new one");
                  r.request.returnValues().add(new StringValue(nodeState.serialize()));
                  r.request.returnValues().add(new StringValue("No host info from dummy implementation"));
                  r.request.returnRequest();
@@ -448,9 +405,9 @@ public class DummyVdsNode {
                 NodeState givenState = (oldState.equals("unknown") ? null : NodeState.deserialize(type, oldState));
                 if (givenState != null && (givenState.equals(nodeState) || sentReply)) {
                     log.log(Level.FINE, () -> "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. "
-                            + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800l / 1000));
+                            + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800L / 1000));
                     req.detach();
-                    waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800l / 1000));
+                    waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800L / 1000));
                     log.log(Level.FINE, () -> "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty"));
                     timer.notifyAll();
                 } else {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
index 7c61423ac2b..24e65a89d2b 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
@@ -1128,7 +1128,7 @@ public class StateChangeTest extends FleetControllerTest {
 
         // At this time, node taken down should have cluster states with all starting timestamps set. Others node should not.
         for (DummyVdsNode node : nodes) {
-            node.waitForSystemStateVersion(waiter.getCurrentSystemState().getVersion(), timeoutMS);
+            node.waitForSystemStateVersion(waiter.getCurrentSystemState().getVersion());
             List<ClusterState> states = node.getSystemStatesReceived();
             ClusterState lastState = states.get(0);
             StringBuilder stateHistory = new StringBuilder();