Merge pull request #23973 from vespa-engine/hmusum/cleanup-29

Cluster controller cleanup, part 13 [run-systemtest]

10 files changed, 84 insertions, 91 deletions

diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java
index 3e8f60e3a6a..e4360a74885 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java
@@ -140,13 +140,13 @@ public class DatabaseTest extends FleetControllerTest {
 
     private void assertWantedStates(Map<Node, NodeState> wantedStates) {
         for (DummyVdsNode node : nodes) {
-            assertEquals(wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode()), node.getNode().toString());
+            assertEquals(wantedStates.get(node.getNode()), fleetController().getWantedNodeState(node.getNode()), node.getNode().toString());
         }
     }
 
     // Note: different semantics than FleetControllerTest.setWantedState
     private void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) {
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         if (supervisor == null) {
             supervisor = new Supervisor(new Transport());
         }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java
index f7e376c419d..cec23930ea8 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java
@@ -44,12 +44,12 @@ public class DistributionBitCountTest extends FleetControllerTest {
         var options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigIncrease");
         var builder = FleetControllerOptions.Builder.copy(options);
         builder.setDistributionBits(20);
-        fleetController.updateOptions(builder.build());
+        fleetController().updateOptions(builder.build());
         ClusterState currentState = waitForState("version:\\d+ bits:20 distributor:10 storage:10");
 
         int version = currentState.getVersion();
         builder.setDistributionBits(23);
-        fleetController.updateOptions(builder.build());
+        fleetController().updateOptions(builder.build());
         assertEquals(version, currentState.getVersion());
     }
 
@@ -61,7 +61,7 @@ public class DistributionBitCountTest extends FleetControllerTest {
         FleetControllerOptions options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigDecrease");
         var builder = FleetControllerOptions.Builder.copy(options);
         builder.setDistributionBits(12);
-        fleetController.updateOptions(builder.build());
+        fleetController().updateOptions(builder.build());
         waitForState("version:\\d+ bits:12 distributor:10 storage:10");
     }
 
@@ -82,7 +82,7 @@ public class DistributionBitCountTest extends FleetControllerTest {
         ClusterState startState = waitForState("version:\\d+ bits:11 distributor:10 storage:10");
 
         nodes.get(1).setNodeState(new NodeState(NodeType.STORAGE, State.UP).setMinUsedBits(12));
-        assertEquals(startState.getVersion(), fleetController.getSystemState().getVersion(), startState + "->" + fleetController.getSystemState());
+        assertEquals(startState.getVersion(), fleetController().getSystemState().getVersion(), startState + "->" + fleetController().getSystemState());
 
         for (int i = 0; i < 10; ++i) {
             // nodes is array of [distr.0, stor.0, distr.1, stor.1, ...] and we just want the storage nodes
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java
index 2a77b4c7636..cd36bf1eb37 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.clustercontroller.core.testutils.WaitTask;
 import com.yahoo.vespa.clustercontroller.core.testutils.Waiter;
 import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter;
 import org.junit.jupiter.api.AfterEach;
+import java.io.IOException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -58,7 +59,7 @@ public abstract class FleetControllerTest implements Waiter {
     protected Slobrok slobrok;
     protected FleetControllerOptions options;
     ZooKeeperTestServer zooKeeperServer;
-    protected FleetController fleetController;
+    protected List<FleetController> fleetControllers = new ArrayList<>();
     protected List<DummyVdsNode> nodes = new ArrayList<>();
     private String testName;
 
@@ -66,7 +67,7 @@ public abstract class FleetControllerTest implements Waiter {
         @Override
         public Object getMonitor() { return timer; }
         @Override
-        public FleetController getFleetController() { return fleetController; }
+        public FleetController getFleetController() { return fleetController(); }
         @Override
         public List<DummyVdsNode> getDummyNodes() { return nodes; }
         @Override
@@ -149,8 +150,8 @@ public abstract class FleetControllerTest implements Waiter {
     protected FleetControllerOptions setUpFleetController(boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception {
         if (slobrok == null) setUpSystem(builder);
         options = builder.build();
-        if (fleetController == null) {
-            fleetController = createFleetController(useFakeTimer, options);
+        if (fleetControllers.isEmpty()) {
+            fleetControllers.add(createFleetController(useFakeTimer, options));
         } else {
             throw new Exception("called setUpFleetcontroller but it was already setup");
         }
@@ -158,15 +159,19 @@ public abstract class FleetControllerTest implements Waiter {
     }
 
     void stopFleetController() throws Exception {
-        if (fleetController != null) {
-            fleetController.shutdown();
-            fleetController = null;
-        }
+        fleetControllers.forEach(f -> {
+            try {
+                f.shutdown();
+            } catch (InterruptedException | IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        fleetControllers.clear();
     }
 
     void startFleetController(boolean useFakeTimer) throws Exception {
-        if (fleetController == null) {
-            fleetController = createFleetController(useFakeTimer, options);
+        if (fleetControllers.isEmpty()) {
+            fleetControllers.add(createFleetController(useFakeTimer, options));
         } else {
             log.log(Level.WARNING, "already started fleetcontroller, not starting another");
         }
@@ -236,7 +241,7 @@ public abstract class FleetControllerTest implements Waiter {
             @Override
             public Object getMonitor() { return timer; }
             @Override
-            public FleetController getFleetController() { return fleetController; }
+            public FleetController getFleetController() { return fleetController(); }
             @Override
             public List<DummyVdsNode> getDummyNodes() {
                 return nodes.stream()
@@ -249,7 +254,7 @@ public abstract class FleetControllerTest implements Waiter {
         subsetWaiter.waitForState(expectedState);
     }
 
-    protected void tearDownSystem() throws Exception {
+    protected void tearDownSystem() {
         if (testName != null) {
             //log.log(Level.INFO, "STOPPING TEST " + testName);
             System.err.println("STOPPING TEST " + testName);
@@ -258,10 +263,14 @@ public abstract class FleetControllerTest implements Waiter {
         if (supervisor != null) {
             supervisor.transport().shutdown().join();
         }
-        if (fleetController != null) {
-            fleetController.shutdown();
-            fleetController = null;
-        }
+        fleetControllers.forEach(f -> {
+            try {
+                f.shutdown();
+            } catch (InterruptedException | IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        fleetControllers.clear();
         if (nodes != null) for (DummyVdsNode node : nodes) {
             node.shutdown();
             nodes = null;
@@ -291,7 +300,7 @@ public abstract class FleetControllerTest implements Waiter {
     }
 
     void waitForCompleteCycle() {
-        fleetController.waitForCompleteCycle(timeout);
+        fleetController().waitForCompleteCycle(timeout);
     }
 
     public static Set<ConfiguredNode> toNodes(Integer ... indexes) {
@@ -306,7 +315,7 @@ public abstract class FleetControllerTest implements Waiter {
         }
         NodeState ns = new NodeState(node.getType(), state);
         if (reason != null) ns.setDescription(reason);
-        Target connection = supervisor.connect(new Spec("localhost", fleetController.getRpcPort()));
+        Target connection = supervisor.connect(new Spec("localhost", fleetController().getRpcPort()));
         Request req = new Request("setNodeState");
         req.parameters().add(new StringValue(node.getSlobrokName()));
         req.parameters().add(new StringValue(ns.serialize()));
@@ -319,6 +328,8 @@ public abstract class FleetControllerTest implements Waiter {
         }
     }
 
+    protected FleetController fleetController() {return fleetControllers.get(0);}
+
     static String[] getSlobrokConnectionSpecs(Slobrok slobrok) {
         String[] connectionSpecs = new String[1];
         connectionSpecs[0] = "tcp/localhost:" + slobrok.port();
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java
index 036d3edd2a8..a498f26fb7d 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java
@@ -22,7 +22,7 @@ public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest {
     }
 
     private void updateConfigLive(FleetControllerOptions newOptions) {
-        this.fleetController.updateOptions(newOptions);
+        fleetController().updateOptions(newOptions);
     }
 
     private void reconfigureWithMinNodeRatio(FleetControllerOptions options, double minNodeRatio) {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java
index e13fb1512d7..b2d21d0b54f 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java
@@ -33,13 +33,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 public class MasterElectionTest extends FleetControllerTest {
 
     private static final Logger log = Logger.getLogger(MasterElectionTest.class.getName());
+    private static int defaultZkSessionTimeoutInMillis() { return 30_000; }
 
     private Supervisor supervisor;
-    private final List<FleetController> fleetControllers = new ArrayList<>();
-
-    private static int defaultZkSessionTimeoutInMillis() { return 30_000; }
 
-    protected FleetControllerOptions setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception {
+    protected void setUpFleetControllers(int count, boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception {
         if (zooKeeperServer == null) {
             zooKeeperServer = new ZooKeeperTestServer();
         }
@@ -54,7 +52,6 @@ public class MasterElectionTest extends FleetControllerTest {
             b.setIndex(i);
             fleetControllers.add(createFleetController(useFakeTimer, b.build()));
         }
-        return options;
     }
 
     private FleetControllerOptions adjustConfig(FleetControllerOptions options, int fleetControllerIndex, int fleetControllerCount) {
@@ -89,18 +86,6 @@ public class MasterElectionTest extends FleetControllerTest {
         }
     }
 
-    protected void tearDownSystem() throws Exception {
-        for (FleetController fleetController : fleetControllers) {
-            if (fleetController != null) {
-                fleetController.shutdown();
-            }
-        }
-        if (slobrok != null) {
-            slobrok.stop();
-        }
-        super.tearDownSystem();
-    }
-
     public void tearDown() throws Exception {
         if (supervisor != null) {
             supervisor.transport().shutdown().join();
@@ -115,7 +100,7 @@ public class MasterElectionTest extends FleetControllerTest {
         FleetControllerOptions.Builder builder = defaultOptions("mycluster");
         builder.setMasterZooKeeperCooldownPeriod(100);
         boolean usingFakeTimer = false;
-        setUpFleetController(5, usingFakeTimer, builder);
+        setUpFleetControllers(5, usingFakeTimer, builder);
         waitForMaster(0);
         log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0");
         fleetControllers.get(0).shutdown();
@@ -219,11 +204,10 @@ public class MasterElectionTest extends FleetControllerTest {
         startingTest("MasterElectionTest::testClusterStateVersionIncreasesAcrossMasterElections");
         FleetControllerOptions.Builder options = defaultOptions("mycluster");
         options.setMasterZooKeeperCooldownPeriod(1);
-        setUpFleetController(3, false, options);
+        setUpFleetControllers(3, false, options);
         // Currently need to have content nodes present for the cluster controller to even bother
         // attempting to persisting its cluster state version to ZK.
         setUpVdsNodes(false);
-        fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing
         waitForStableSystem();
         waitForMaster(0);
         Stream.of(0, 1, 2).forEach(this::waitForCompleteCycle);
@@ -242,7 +226,7 @@ public class MasterElectionTest extends FleetControllerTest {
         // "Magic" port value is in range allocated to module for testing.
         zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342);
         options.setMasterZooKeeperCooldownPeriod(100);
-        setUpFleetController(2, false, options);
+        setUpFleetControllers(2, false, options);
         waitForMaster(0);
 
         zooKeeperServer.shutdown(true);
@@ -262,7 +246,7 @@ public class MasterElectionTest extends FleetControllerTest {
         FleetControllerOptions.Builder builder = defaultOptions("mycluster")
                 .setMasterZooKeeperCooldownPeriod(100)
                 .setZooKeeperServerAddress("localhost");
-        setUpFleetController(3, false, builder);
+        setUpFleetControllers(3, false, builder);
         waitForMaster(0);
 
         log.log(Level.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress());
@@ -294,7 +278,7 @@ public class MasterElectionTest extends FleetControllerTest {
         startingTest("MasterElectionTest::testMasterZooKeeperCooldown");
         FleetControllerOptions.Builder options = defaultOptions("mycluster");
         options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour
-        setUpFleetController(3, true, options);
+        setUpFleetControllers(3, true, options);
         waitForMaster(0);
         timer.advanceTime(24 * 3600 * 1000); // A day
         waitForCompleteCycle(1);
@@ -342,7 +326,7 @@ public class MasterElectionTest extends FleetControllerTest {
         startingTest("MasterElectionTest::testGetMaster");
         FleetControllerOptions.Builder options = defaultOptions("mycluster");
         options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour
-        setUpFleetController(3, true, options);
+        setUpFleetControllers(3, true, options);
         waitForMaster(0);
 
         supervisor = new Supervisor(new Transport());
@@ -424,7 +408,7 @@ public class MasterElectionTest extends FleetControllerTest {
         startingTest("MasterElectionTest::testReconfigure");
         FleetControllerOptions.Builder options = defaultOptions("mycluster");
         options.setMasterZooKeeperCooldownPeriod(1);
-        setUpFleetController(3, false, options);
+        setUpFleetControllers(3, false, options);
         waitForMaster(0);
 
         FleetControllerOptions newOptions = FleetControllerOptions.Builder.copy(options.build()).build();
@@ -453,9 +437,8 @@ public class MasterElectionTest extends FleetControllerTest {
                 .setMinRatioOfStorageNodesUp(0)
                 .setMinDistributorNodesUp(0)
                 .setMinStorageNodesUp(1);
-        setUpFleetController(3, false, builder);
+        setUpFleetControllers(3, false, builder);
         setUpVdsNodes(false);
-        fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing
         waitForStableSystem();
         waitForMaster(0);
 
@@ -498,9 +481,8 @@ public class MasterElectionTest extends FleetControllerTest {
                 .setMasterZooKeeperCooldownPeriod(1)
                 .setMinTimeBeforeFirstSystemStateBroadcast(100000);
         boolean useFakeTimer = false;
-        setUpFleetController(3, useFakeTimer, builder);
+        setUpFleetControllers(3, useFakeTimer, builder);
         setUpVdsNodes(false);
-        fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing
         waitForMaster(0);
         waitForStableSystem();
         log.info("Waiting for full maintenance mode in default space");
@@ -541,9 +523,8 @@ public class MasterElectionTest extends FleetControllerTest {
                 .setClusterHasGlobalDocumentTypes(false)
                 .setMasterZooKeeperCooldownPeriod(1)
                 .setMinTimeBeforeFirstSystemStateBroadcast(100000);
-        setUpFleetController(3, false, builder);
+        setUpFleetControllers(3, false, builder);
         setUpVdsNodes(false);
-        fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing
         waitForMaster(0);
         waitForStableSystem();
         waitForStateInAllSpaces("version:\\d+ distributor:10 storage:10");
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java
index a760adf3cf2..4ca090b4947 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java
@@ -53,7 +53,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest
         var a = FleetControllerOptions.Builder.copy(options);
         a.setNodes(asConfiguredNodes(nodeIndices));
         options = a.build();
-        fleetController.updateOptions(options);
+        fleetController().updateOptions(options);
         // Need to treat cluster as having 6 nodes due to ideal state algo semantics.
         // Note that we do not use subsetWaiter here since we want node 6 included.
         waitForState("version:\\d+ distributor:7 .4.s:d .5.s:d storage:7 .4.s:d .5.s:d");
@@ -75,7 +75,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest
         builder = FleetControllerOptions.Builder.copy(options);
         builder.setNodes(configuredNodes);
         options = builder.build();
-        fleetController.updateOptions(options);
+        fleetController().updateOptions(options);
 
         waitForState("version:\\d+ distributor:4 storage:4 .0.s:r");
 
@@ -84,7 +84,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest
         builder = FleetControllerOptions.Builder.copy(options);
         builder.setNodes(configuredNodes);
         options = builder.build();
-        fleetController.updateOptions(options);
+        fleetController().updateOptions(options);
 
         // The previously retired node should now be marked as down, as it no longer
         // exists from the point of view of the content cluster. We have to use a subset
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java
index 4b4cb348968..f9eda9040d2 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java
@@ -105,10 +105,10 @@ public class RpcServerTest extends FleetControllerTest {
         log.log(Level.INFO, "Disconnecting distributor 0. Waiting for state to reflect change.");
         nodes.get(0).disconnect();
         nodes.get(19).disconnect();
-        fleetController.waitForNodesInSlobrok(9, 9, timeout());
+        fleetController().waitForNodesInSlobrok(9, 9, timeout());
         timer.advanceTime(options.nodeStateRequestTimeoutMS() + options.maxSlobrokDisconnectGracePeriod());
 
-        wait(new WaitCondition.StateWait(fleetController, fleetController.getMonitor()) {
+        wait(new WaitCondition.StateWait(fleetController(), fleetController().getMonitor()) {
             @Override
             public String isConditionMet() {
                 if (currentState == null) {
@@ -126,7 +126,7 @@ public class RpcServerTest extends FleetControllerTest {
             }
         }, null, timeout());
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
@@ -142,7 +142,7 @@ public class RpcServerTest extends FleetControllerTest {
     }
 
     private void setWantedNodeState(State newState, NodeType nodeType, int nodeIndex) {
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         if (supervisor == null) {
             supervisor = new Supervisor(new Transport());
         }
@@ -189,7 +189,7 @@ public class RpcServerTest extends FleetControllerTest {
         nodes.get(3).connect();
         waitForState("version:\\d+ distributor:10 .0.s:d .2.s:d storage:10 .1.s:i .1.i:0.2 .2.s:d .7.s:m");
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
@@ -321,7 +321,7 @@ public class RpcServerTest extends FleetControllerTest {
                     .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs())
                     .setMaxInitProgressTime(30000)
                     .setStableStateTimePeriod(60000);
-            fleetController.updateOptions(builder.build());
+            fleetController().updateOptions(builder.build());
             waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m .2.s:r .3.s:r .4.s:r");
         }
 
@@ -351,7 +351,7 @@ public class RpcServerTest extends FleetControllerTest {
                     .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs())
                     .setMaxInitProgressTime(30000)
                     .setStableStateTimePeriod(60000);
-            fleetController.updateOptions(builder.build());
+            fleetController().updateOptions(builder.build());
             waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m");
         }
 
@@ -391,7 +391,7 @@ public class RpcServerTest extends FleetControllerTest {
                     .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs())
                     .setMaxInitProgressTime(30000)
                     .setStableStateTimePeriod(60000);
-            fleetController.updateOptions(builder.build());
+            fleetController().updateOptions(builder.build());
             waitForState("version:\\d+ distributor:5 storage:5");
         }
 
@@ -406,7 +406,7 @@ public class RpcServerTest extends FleetControllerTest {
                     .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs())
                     .setMaxInitProgressTime(30000)
                     .setStableStateTimePeriod(60000);
-            fleetController.updateOptions(builder.build());
+            fleetController().updateOptions(builder.build());
             waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r");
         }
 
@@ -420,7 +420,7 @@ public class RpcServerTest extends FleetControllerTest {
                     .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs())
                     .setMaxInitProgressTime(30000)
                     .setStableStateTimePeriod(60000);
-            fleetController.updateOptions(builder.build());
+            fleetController().updateOptions(builder.build());
             waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r");
         }
 
@@ -456,7 +456,7 @@ public class RpcServerTest extends FleetControllerTest {
         setUpVdsNodes(true, false, nodeIndexes);
         waitForState("version:\\d+ distributor:26 .0.s:d .1.s:d .2.s:d .3.s:d .5.s:d .7.s:d .8.s:d .11.s:d .12.s:d .13.s:d .15.s:d .17.s:d .18.s:d .19.s:d .20.s:d .24.s:d storage:26 .0.s:d .1.s:d .2.s:d .3.s:d .5.s:d .7.s:d .8.s:d .11.s:d .12.s:d .13.s:d .15.s:d .17.s:d .18.s:d .19.s:d .20.s:d .24.s:d");
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
@@ -476,7 +476,7 @@ public class RpcServerTest extends FleetControllerTest {
         waitForCompleteCycle();
         timer.advanceTime(100000000);
         waitForCompleteCycle();
-        assertEquals(State.MAINTENANCE, fleetController.getSystemState().getNodeState(new Node(NodeType.STORAGE, 16)).getState());
+        assertEquals(State.MAINTENANCE, fleetController().getSystemState().getNodeState(new Node(NodeType.STORAGE, 16)).getState());
 
         nodes.get(4 * 2 + 1).disconnect();
         waitForState("version:\\d+ distributor:26 .* storage:26 .* .14.s:m.* .16.s:m .*");
@@ -495,7 +495,7 @@ public class RpcServerTest extends FleetControllerTest {
         setUpVdsNodes(true);
         waitForStableSystem();
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
@@ -523,7 +523,7 @@ public class RpcServerTest extends FleetControllerTest {
         setUpVdsNodes(true);
         waitForStableSystem();
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
@@ -548,7 +548,7 @@ public class RpcServerTest extends FleetControllerTest {
         nodes.get(0).disconnect();
         waitForState("version:\\d+ distributor:5 .0.s:d storage:5");
 
-        int rpcPort = fleetController.getRpcPort();
+        int rpcPort = fleetController().getRpcPort();
         supervisor = new Supervisor(new Transport());
         Target connection = supervisor.connect(new Spec("localhost", rpcPort));
         assertTrue(connection.isValid());
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java
index 03ef88c09ee..6d2a4fdeb24 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java
@@ -25,6 +25,7 @@ public class SlobrokTest extends FleetControllerTest {
         setUpVdsNodes(true);
         waitForStableSystem();
 
+        FleetController fleetController = fleetController();
         int version = fleetController.getSystemState().getVersion();
         int slobrokPort = slobrok.port();
 
@@ -79,10 +80,10 @@ public class SlobrokTest extends FleetControllerTest {
         setUpVdsNodes(true);
         waitForStableSystem();
 
-        int version = fleetController.getSystemState().getVersion();
+        int version = fleetController().getSystemState().getVersion();
         nodes.get(0).disconnectSlobrok();
         log.log(Level.INFO, "DISCONNECTED NODE FROM SLOBROK. SHOULD BE IN COOLDOWN PERIOD");
-        fleetController.waitForNodesInSlobrok(9, 10, timeout());
+        fleetController().waitForNodesInSlobrok(9, 10, timeout());
         synchronized(timer) {
             nodes.get(0).sendGetNodeStateReply(0);
         }
@@ -93,7 +94,7 @@ public class SlobrokTest extends FleetControllerTest {
             Thread.sleep(10);
         } catch (InterruptedException e) { /* ignore */
         }
-        assertEquals(version, fleetController.getSystemState().getVersion());
+        assertEquals(version, fleetController().getSystemState().getVersion());
         log.log(Level.INFO, "JUMPING TIME. NODE SHOULD BE MARKED DOWN");
         // At this point the fleetcontroller might not have noticed that the node is out of slobrok yet.
         // Thus we keep advancing time another minute such that it should get down.
@@ -103,7 +104,7 @@ public class SlobrokTest extends FleetControllerTest {
 
     private boolean clusterAvailable() {
         boolean ok = true;
-        ContentCluster cluster = fleetController.getCluster();
+        ContentCluster cluster = fleetController().getCluster();
         for (NodeInfo info : cluster.getNodeInfos()) {
             if (info.getConnectionAttemptCount() > 0) ok = false;
             if (info.getLatestNodeStateRequestTime() == null) ok = false;
@@ -111,7 +112,7 @@ public class SlobrokTest extends FleetControllerTest {
         return ok;
     }
     private void assertClusterAvailable() {
-        ContentCluster cluster = fleetController.getCluster();
+        ContentCluster cluster = fleetController().getCluster();
         for (NodeInfo info : cluster.getNodeInfos()) {
             assertEquals(0, info.getConnectionAttemptCount(), "Node " + info + " connection attempts.");
             assertTrue(info.getLatestNodeStateRequestTime() != 0, "Node " + info + " has no last request time.");
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
index 5256c952d15..af6e2d2b104 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
@@ -968,7 +968,7 @@ public class StateChangeTest extends FleetControllerTest {
         setUpFleetController(useFakeTimer, builder);
 
         StateWaiter waiter = new StateWaiter(timer);
-        fleetController.addSystemStateListener(waiter);
+        fleetController().addSystemStateListener(waiter);
 
         // Ensure all nodes have been seen by fleetcontroller and that it has had enough time to possibly have sent a cluster state
         // Note: this is a candidate state and therefore NOT versioned yet
@@ -986,7 +986,7 @@ public class StateChangeTest extends FleetControllerTest {
         waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:d", timeout());
 
         int version = waiter.getCurrentSystemState().getVersion();
-        fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 19, timeout());
+        fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 19, timeout());
 
         new StateMessageChecker(nodes) {
             @Override
@@ -1018,12 +1018,12 @@ public class StateChangeTest extends FleetControllerTest {
 
         final StateWaiter waiter = new StateWaiter(timer);
 
-        fleetController.addSystemStateListener(waiter);
+        fleetController().addSystemStateListener(waiter);
         waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:i .1.i:1.0", timeout());
         waitForCompleteCycle();
 
         final int version = waiter.getCurrentSystemState().getVersion();
-        fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 20, timeout());
+        fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 20, timeout());
 
         // The last two versions of the cluster state should be seen (all nodes up,
         // zero out timestate)
@@ -1044,7 +1044,7 @@ public class StateChangeTest extends FleetControllerTest {
         waitForStableSystem();
 
         StateWaiter waiter = new StateWaiter(timer);
-        fleetController.addSystemStateListener(waiter);
+        fleetController().addSystemStateListener(waiter);
 
         nodes.get(1).failSetSystemState(true);
         int versionBeforeChange = nodes.get(1).getSystemStatesReceived().get(0).getVersion();
@@ -1052,16 +1052,16 @@ public class StateChangeTest extends FleetControllerTest {
         waiter.waitForState("version:\\d+ distributor:10 .1.s:d storage:10", timeout());
         int versionAfterChange = waiter.getCurrentSystemState().getVersion();
         assertTrue(versionAfterChange > versionBeforeChange);
-        fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(versionAfterChange, 18, timeout());
+        fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(versionAfterChange, 18, timeout());
 
         // Assert that the failed node has not acknowledged the latest version.
         // (The version may still be larger than versionBeforeChange if the fleet controller sends a
         // "stable system" update without timestamps in the meantime
-        assertTrue(fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getClusterStateVersionBundleAcknowledged() < versionAfterChange);
+        assertTrue(fleetController().getCluster().getNodeInfo(nodes.get(1).getNode()).getClusterStateVersionBundleAcknowledged() < versionAfterChange);
 
         // Ensure non-concurrent access to getNewestSystemStateVersionSent
         synchronized(timer) {
-            int sentVersion = fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getNewestSystemStateVersionSent();
+            int sentVersion = fleetController().getCluster().getNodeInfo(nodes.get(1).getNode()).getNewestSystemStateVersionSent();
             assertTrue(sentVersion == -1 || sentVersion == versionAfterChange);
         }
     }
@@ -1122,7 +1122,7 @@ public class StateChangeTest extends FleetControllerTest {
         waitForStableSystem();
 
         StateWaiter waiter = new StateWaiter(timer);
-        fleetController.addSystemStateListener(waiter);
+        fleetController().addSystemStateListener(waiter);
 
         // Simulate netsplit. Take node down without node booting
         assertTrue(nodes.get(0).isDistributor());
@@ -1147,7 +1147,7 @@ public class StateChangeTest extends FleetControllerTest {
                 for (ConfiguredNode i : options.nodes()) {
                     Node nodeId = new Node(NodeType.STORAGE, i.index());
                     long ts = lastState.getNodeState(nodeId).getStartTimestamp();
-                    assertTrue(ts > 0, nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController.getCluster().getNodeInfo(nodeId).getStartTimestamp());
+                    assertTrue(ts > 0, nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController().getCluster().getNodeInfo(nodeId).getStartTimestamp());
                 }
             } else {
                 for (ConfiguredNode i : options.nodes()) {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java
index 86bf7a66458..d697dc40bb2 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java
@@ -33,11 +33,11 @@ public class WantedStateTest extends FleetControllerTest {
 
         setWantedState(nodes.get(1), State.MAINTENANCE, "Foo");
         waitForState("version:\\d+ distributor:10 storage:10 .0.s:m");
-        assertEquals("Foo", fleetController.getWantedNodeState(nodes.get(1).getNode()).getDescription());
+        assertEquals("Foo", fleetController().getWantedNodeState(nodes.get(1).getNode()).getDescription());
 
         setWantedState(nodes.get(1), State.MAINTENANCE, "Bar");
         waitForCompleteCycle();
-        assertEquals("Bar", fleetController.getWantedNodeState(nodes.get(1).getNode()).getDescription());
+        assertEquals("Bar", fleetController().getWantedNodeState(nodes.get(1).getNode()).getDescription());
 
         setWantedState(nodes.get(1), State.UP, null);
         waitForState("version:\\d+ distributor:10 storage:10");