diff options
10 files changed, 84 insertions, 91 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index 3e8f60e3a6a..e4360a74885 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -140,13 +140,13 @@ public class DatabaseTest extends FleetControllerTest { private void assertWantedStates(Map<Node, NodeState> wantedStates) { for (DummyVdsNode node : nodes) { - assertEquals(wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode()), node.getNode().toString()); + assertEquals(wantedStates.get(node.getNode()), fleetController().getWantedNodeState(node.getNode()), node.getNode().toString()); } } // Note: different semantics than FleetControllerTest.setWantedState private void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) { - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); if (supervisor == null) { supervisor = new Supervisor(new Transport()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java index f7e376c419d..cec23930ea8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java @@ -44,12 +44,12 @@ public class DistributionBitCountTest extends FleetControllerTest { var options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigIncrease"); var builder = FleetControllerOptions.Builder.copy(options); builder.setDistributionBits(20); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); ClusterState currentState = waitForState("version:\\d+ bits:20 distributor:10 storage:10"); int version = currentState.getVersion(); builder.setDistributionBits(23); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); assertEquals(version, currentState.getVersion()); } @@ -61,7 +61,7 @@ public class DistributionBitCountTest extends FleetControllerTest { FleetControllerOptions options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigDecrease"); var builder = FleetControllerOptions.Builder.copy(options); builder.setDistributionBits(12); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ bits:12 distributor:10 storage:10"); } @@ -82,7 +82,7 @@ public class DistributionBitCountTest extends FleetControllerTest { ClusterState startState = waitForState("version:\\d+ bits:11 distributor:10 storage:10"); nodes.get(1).setNodeState(new NodeState(NodeType.STORAGE, State.UP).setMinUsedBits(12)); - assertEquals(startState.getVersion(), fleetController.getSystemState().getVersion(), startState + "->" + fleetController.getSystemState()); + assertEquals(startState.getVersion(), fleetController().getSystemState().getVersion(), startState + "->" + fleetController().getSystemState()); for (int i = 0; i < 10; ++i) { // nodes is array of [distr.0, stor.0, distr.1, stor.1, ...] and we just want the storage nodes diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 2a77b4c7636..cd36bf1eb37 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.clustercontroller.core.testutils.WaitTask; import com.yahoo.vespa.clustercontroller.core.testutils.Waiter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.AfterEach; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -58,7 +59,7 @@ public abstract class FleetControllerTest implements Waiter { protected Slobrok slobrok; protected FleetControllerOptions options; ZooKeeperTestServer zooKeeperServer; - protected FleetController fleetController; + protected List<FleetController> fleetControllers = new ArrayList<>(); protected List<DummyVdsNode> nodes = new ArrayList<>(); private String testName; @@ -66,7 +67,7 @@ public abstract class FleetControllerTest implements Waiter { @Override public Object getMonitor() { return timer; } @Override - public FleetController getFleetController() { return fleetController; } + public FleetController getFleetController() { return fleetController(); } @Override public List<DummyVdsNode> getDummyNodes() { return nodes; } @Override @@ -149,8 +150,8 @@ public abstract class FleetControllerTest implements Waiter { protected FleetControllerOptions setUpFleetController(boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception { if (slobrok == null) setUpSystem(builder); options = builder.build(); - if (fleetController == null) { - fleetController = createFleetController(useFakeTimer, options); + if (fleetControllers.isEmpty()) { + fleetControllers.add(createFleetController(useFakeTimer, options)); } else { throw new Exception("called setUpFleetcontroller but it was already setup"); } @@ -158,15 +159,19 @@ public abstract class FleetControllerTest implements Waiter { } void stopFleetController() throws Exception { - if (fleetController != null) { - fleetController.shutdown(); - fleetController = null; - } + fleetControllers.forEach(f -> { + try { + f.shutdown(); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + }); + fleetControllers.clear(); } void startFleetController(boolean useFakeTimer) throws Exception { - if (fleetController == null) { - fleetController = createFleetController(useFakeTimer, options); + if (fleetControllers.isEmpty()) { + fleetControllers.add(createFleetController(useFakeTimer, options)); } else { log.log(Level.WARNING, "already started fleetcontroller, not starting another"); } @@ -236,7 +241,7 @@ public abstract class FleetControllerTest implements Waiter { @Override public Object getMonitor() { return timer; } @Override - public FleetController getFleetController() { return fleetController; } + public FleetController getFleetController() { return fleetController(); } @Override public List<DummyVdsNode> getDummyNodes() { return nodes.stream() @@ -249,7 +254,7 @@ public abstract class FleetControllerTest implements Waiter { subsetWaiter.waitForState(expectedState); } - protected void tearDownSystem() throws Exception { + protected void tearDownSystem() { if (testName != null) { //log.log(Level.INFO, "STOPPING TEST " + testName); System.err.println("STOPPING TEST " + testName); @@ -258,10 +263,14 @@ public abstract class FleetControllerTest implements Waiter { if (supervisor != null) { supervisor.transport().shutdown().join(); } - if (fleetController != null) { - fleetController.shutdown(); - fleetController = null; - } + fleetControllers.forEach(f -> { + try { + f.shutdown(); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + }); + fleetControllers.clear(); if (nodes != null) for (DummyVdsNode node : nodes) { node.shutdown(); nodes = null; @@ -291,7 +300,7 @@ public abstract class FleetControllerTest implements Waiter { } void waitForCompleteCycle() { - fleetController.waitForCompleteCycle(timeout); + fleetController().waitForCompleteCycle(timeout); } public static Set<ConfiguredNode> toNodes(Integer ... indexes) { @@ -306,7 +315,7 @@ public abstract class FleetControllerTest implements Waiter { } NodeState ns = new NodeState(node.getType(), state); if (reason != null) ns.setDescription(reason); - Target connection = supervisor.connect(new Spec("localhost", fleetController.getRpcPort())); + Target connection = supervisor.connect(new Spec("localhost", fleetController().getRpcPort())); Request req = new Request("setNodeState"); req.parameters().add(new StringValue(node.getSlobrokName())); req.parameters().add(new StringValue(ns.serialize())); @@ -319,6 +328,8 @@ public abstract class FleetControllerTest implements Waiter { } } + protected FleetController fleetController() {return fleetControllers.get(0);} + static String[] getSlobrokConnectionSpecs(Slobrok slobrok) { String[] connectionSpecs = new String[1]; connectionSpecs[0] = "tcp/localhost:" + slobrok.port(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java index 036d3edd2a8..a498f26fb7d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java @@ -22,7 +22,7 @@ public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest { } private void updateConfigLive(FleetControllerOptions newOptions) { - this.fleetController.updateOptions(newOptions); + fleetController().updateOptions(newOptions); } private void reconfigureWithMinNodeRatio(FleetControllerOptions options, double minNodeRatio) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index e13fb1512d7..b2d21d0b54f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -33,13 +33,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class MasterElectionTest extends FleetControllerTest { private static final Logger log = Logger.getLogger(MasterElectionTest.class.getName()); + private static int defaultZkSessionTimeoutInMillis() { return 30_000; } private Supervisor supervisor; - private final List<FleetController> fleetControllers = new ArrayList<>(); - - private static int defaultZkSessionTimeoutInMillis() { return 30_000; } - protected FleetControllerOptions setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception { + protected void setUpFleetControllers(int count, boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception { if (zooKeeperServer == null) { zooKeeperServer = new ZooKeeperTestServer(); } @@ -54,7 +52,6 @@ public class MasterElectionTest extends FleetControllerTest { b.setIndex(i); fleetControllers.add(createFleetController(useFakeTimer, b.build())); } - return options; } private FleetControllerOptions adjustConfig(FleetControllerOptions options, int fleetControllerIndex, int fleetControllerCount) { @@ -89,18 +86,6 @@ public class MasterElectionTest extends FleetControllerTest { } } - protected void tearDownSystem() throws Exception { - for (FleetController fleetController : fleetControllers) { - if (fleetController != null) { - fleetController.shutdown(); - } - } - if (slobrok != null) { - slobrok.stop(); - } - super.tearDownSystem(); - } - public void tearDown() throws Exception { if (supervisor != null) { supervisor.transport().shutdown().join(); @@ -115,7 +100,7 @@ public class MasterElectionTest extends FleetControllerTest { FleetControllerOptions.Builder builder = defaultOptions("mycluster"); builder.setMasterZooKeeperCooldownPeriod(100); boolean usingFakeTimer = false; - setUpFleetController(5, usingFakeTimer, builder); + setUpFleetControllers(5, usingFakeTimer, builder); waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); @@ -219,11 +204,10 @@ public class MasterElectionTest extends FleetControllerTest { startingTest("MasterElectionTest::testClusterStateVersionIncreasesAcrossMasterElections"); FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setMasterZooKeeperCooldownPeriod(1); - setUpFleetController(3, false, options); + setUpFleetControllers(3, false, options); // Currently need to have content nodes present for the cluster controller to even bother // attempting to persisting its cluster state version to ZK. setUpVdsNodes(false); - fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); Stream.of(0, 1, 2).forEach(this::waitForCompleteCycle); @@ -242,7 +226,7 @@ public class MasterElectionTest extends FleetControllerTest { // "Magic" port value is in range allocated to module for testing. zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342); options.setMasterZooKeeperCooldownPeriod(100); - setUpFleetController(2, false, options); + setUpFleetControllers(2, false, options); waitForMaster(0); zooKeeperServer.shutdown(true); @@ -262,7 +246,7 @@ public class MasterElectionTest extends FleetControllerTest { FleetControllerOptions.Builder builder = defaultOptions("mycluster") .setMasterZooKeeperCooldownPeriod(100) .setZooKeeperServerAddress("localhost"); - setUpFleetController(3, false, builder); + setUpFleetControllers(3, false, builder); waitForMaster(0); log.log(Level.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); @@ -294,7 +278,7 @@ public class MasterElectionTest extends FleetControllerTest { startingTest("MasterElectionTest::testMasterZooKeeperCooldown"); FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour - setUpFleetController(3, true, options); + setUpFleetControllers(3, true, options); waitForMaster(0); timer.advanceTime(24 * 3600 * 1000); // A day waitForCompleteCycle(1); @@ -342,7 +326,7 @@ public class MasterElectionTest extends FleetControllerTest { startingTest("MasterElectionTest::testGetMaster"); FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour - setUpFleetController(3, true, options); + setUpFleetControllers(3, true, options); waitForMaster(0); supervisor = new Supervisor(new Transport()); @@ -424,7 +408,7 @@ public class MasterElectionTest extends FleetControllerTest { startingTest("MasterElectionTest::testReconfigure"); FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setMasterZooKeeperCooldownPeriod(1); - setUpFleetController(3, false, options); + setUpFleetControllers(3, false, options); waitForMaster(0); FleetControllerOptions newOptions = FleetControllerOptions.Builder.copy(options.build()).build(); @@ -453,9 +437,8 @@ public class MasterElectionTest extends FleetControllerTest { .setMinRatioOfStorageNodesUp(0) .setMinDistributorNodesUp(0) .setMinStorageNodesUp(1); - setUpFleetController(3, false, builder); + setUpFleetControllers(3, false, builder); setUpVdsNodes(false); - fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); @@ -498,9 +481,8 @@ public class MasterElectionTest extends FleetControllerTest { .setMasterZooKeeperCooldownPeriod(1) .setMinTimeBeforeFirstSystemStateBroadcast(100000); boolean useFakeTimer = false; - setUpFleetController(3, useFakeTimer, builder); + setUpFleetControllers(3, useFakeTimer, builder); setUpVdsNodes(false); - fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); log.info("Waiting for full maintenance mode in default space"); @@ -541,9 +523,8 @@ public class MasterElectionTest extends FleetControllerTest { .setClusterHasGlobalDocumentTypes(false) .setMasterZooKeeperCooldownPeriod(1) .setMinTimeBeforeFirstSystemStateBroadcast(100000); - setUpFleetController(3, false, builder); + setUpFleetControllers(3, false, builder); setUpVdsNodes(false); - fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); waitForStateInAllSpaces("version:\\d+ distributor:10 storage:10"); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java index a760adf3cf2..4ca090b4947 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java @@ -53,7 +53,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest var a = FleetControllerOptions.Builder.copy(options); a.setNodes(asConfiguredNodes(nodeIndices)); options = a.build(); - fleetController.updateOptions(options); + fleetController().updateOptions(options); // Need to treat cluster as having 6 nodes due to ideal state algo semantics. // Note that we do not use subsetWaiter here since we want node 6 included. waitForState("version:\\d+ distributor:7 .4.s:d .5.s:d storage:7 .4.s:d .5.s:d"); @@ -75,7 +75,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest builder = FleetControllerOptions.Builder.copy(options); builder.setNodes(configuredNodes); options = builder.build(); - fleetController.updateOptions(options); + fleetController().updateOptions(options); waitForState("version:\\d+ distributor:4 storage:4 .0.s:r"); @@ -84,7 +84,7 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest builder = FleetControllerOptions.Builder.copy(options); builder.setNodes(configuredNodes); options = builder.build(); - fleetController.updateOptions(options); + fleetController().updateOptions(options); // The previously retired node should now be marked as down, as it no longer // exists from the point of view of the content cluster. We have to use a subset diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index 4b4cb348968..f9eda9040d2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -105,10 +105,10 @@ public class RpcServerTest extends FleetControllerTest { log.log(Level.INFO, "Disconnecting distributor 0. Waiting for state to reflect change."); nodes.get(0).disconnect(); nodes.get(19).disconnect(); - fleetController.waitForNodesInSlobrok(9, 9, timeout()); + fleetController().waitForNodesInSlobrok(9, 9, timeout()); timer.advanceTime(options.nodeStateRequestTimeoutMS() + options.maxSlobrokDisconnectGracePeriod()); - wait(new WaitCondition.StateWait(fleetController, fleetController.getMonitor()) { + wait(new WaitCondition.StateWait(fleetController(), fleetController().getMonitor()) { @Override public String isConditionMet() { if (currentState == null) { @@ -126,7 +126,7 @@ public class RpcServerTest extends FleetControllerTest { } }, null, timeout()); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); @@ -142,7 +142,7 @@ public class RpcServerTest extends FleetControllerTest { } private void setWantedNodeState(State newState, NodeType nodeType, int nodeIndex) { - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); if (supervisor == null) { supervisor = new Supervisor(new Transport()); } @@ -189,7 +189,7 @@ public class RpcServerTest extends FleetControllerTest { nodes.get(3).connect(); waitForState("version:\\d+ distributor:10 .0.s:d .2.s:d storage:10 .1.s:i .1.i:0.2 .2.s:d .7.s:m"); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); @@ -321,7 +321,7 @@ public class RpcServerTest extends FleetControllerTest { .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs()) .setMaxInitProgressTime(30000) .setStableStateTimePeriod(60000); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m .2.s:r .3.s:r .4.s:r"); } @@ -351,7 +351,7 @@ public class RpcServerTest extends FleetControllerTest { .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs()) .setMaxInitProgressTime(30000) .setStableStateTimePeriod(60000); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m"); } @@ -391,7 +391,7 @@ public class RpcServerTest extends FleetControllerTest { .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) .setMaxInitProgressTime(30000) .setStableStateTimePeriod(60000); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ distributor:5 storage:5"); } @@ -406,7 +406,7 @@ public class RpcServerTest extends FleetControllerTest { .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) .setMaxInitProgressTime(30000) .setStableStateTimePeriod(60000); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r"); } @@ -420,7 +420,7 @@ public class RpcServerTest extends FleetControllerTest { .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) .setMaxInitProgressTime(30000) .setStableStateTimePeriod(60000); - fleetController.updateOptions(builder.build()); + fleetController().updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r"); } @@ -456,7 +456,7 @@ public class RpcServerTest extends FleetControllerTest { setUpVdsNodes(true, false, nodeIndexes); waitForState("version:\\d+ distributor:26 .0.s:d .1.s:d .2.s:d .3.s:d .5.s:d .7.s:d .8.s:d .11.s:d .12.s:d .13.s:d .15.s:d .17.s:d .18.s:d .19.s:d .20.s:d .24.s:d storage:26 .0.s:d .1.s:d .2.s:d .3.s:d .5.s:d .7.s:d .8.s:d .11.s:d .12.s:d .13.s:d .15.s:d .17.s:d .18.s:d .19.s:d .20.s:d .24.s:d"); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); @@ -476,7 +476,7 @@ public class RpcServerTest extends FleetControllerTest { waitForCompleteCycle(); timer.advanceTime(100000000); waitForCompleteCycle(); - assertEquals(State.MAINTENANCE, fleetController.getSystemState().getNodeState(new Node(NodeType.STORAGE, 16)).getState()); + assertEquals(State.MAINTENANCE, fleetController().getSystemState().getNodeState(new Node(NodeType.STORAGE, 16)).getState()); nodes.get(4 * 2 + 1).disconnect(); waitForState("version:\\d+ distributor:26 .* storage:26 .* .14.s:m.* .16.s:m .*"); @@ -495,7 +495,7 @@ public class RpcServerTest extends FleetControllerTest { setUpVdsNodes(true); waitForStableSystem(); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); @@ -523,7 +523,7 @@ public class RpcServerTest extends FleetControllerTest { setUpVdsNodes(true); waitForStableSystem(); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); @@ -548,7 +548,7 @@ public class RpcServerTest extends FleetControllerTest { nodes.get(0).disconnect(); waitForState("version:\\d+ distributor:5 .0.s:d storage:5"); - int rpcPort = fleetController.getRpcPort(); + int rpcPort = fleetController().getRpcPort(); supervisor = new Supervisor(new Transport()); Target connection = supervisor.connect(new Spec("localhost", rpcPort)); assertTrue(connection.isValid()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index 03ef88c09ee..6d2a4fdeb24 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -25,6 +25,7 @@ public class SlobrokTest extends FleetControllerTest { setUpVdsNodes(true); waitForStableSystem(); + FleetController fleetController = fleetController(); int version = fleetController.getSystemState().getVersion(); int slobrokPort = slobrok.port(); @@ -79,10 +80,10 @@ public class SlobrokTest extends FleetControllerTest { setUpVdsNodes(true); waitForStableSystem(); - int version = fleetController.getSystemState().getVersion(); + int version = fleetController().getSystemState().getVersion(); nodes.get(0).disconnectSlobrok(); log.log(Level.INFO, "DISCONNECTED NODE FROM SLOBROK. SHOULD BE IN COOLDOWN PERIOD"); - fleetController.waitForNodesInSlobrok(9, 10, timeout()); + fleetController().waitForNodesInSlobrok(9, 10, timeout()); synchronized(timer) { nodes.get(0).sendGetNodeStateReply(0); } @@ -93,7 +94,7 @@ public class SlobrokTest extends FleetControllerTest { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ } - assertEquals(version, fleetController.getSystemState().getVersion()); + assertEquals(version, fleetController().getSystemState().getVersion()); log.log(Level.INFO, "JUMPING TIME. NODE SHOULD BE MARKED DOWN"); // At this point the fleetcontroller might not have noticed that the node is out of slobrok yet. // Thus we keep advancing time another minute such that it should get down. @@ -103,7 +104,7 @@ public class SlobrokTest extends FleetControllerTest { private boolean clusterAvailable() { boolean ok = true; - ContentCluster cluster = fleetController.getCluster(); + ContentCluster cluster = fleetController().getCluster(); for (NodeInfo info : cluster.getNodeInfos()) { if (info.getConnectionAttemptCount() > 0) ok = false; if (info.getLatestNodeStateRequestTime() == null) ok = false; @@ -111,7 +112,7 @@ public class SlobrokTest extends FleetControllerTest { return ok; } private void assertClusterAvailable() { - ContentCluster cluster = fleetController.getCluster(); + ContentCluster cluster = fleetController().getCluster(); for (NodeInfo info : cluster.getNodeInfos()) { assertEquals(0, info.getConnectionAttemptCount(), "Node " + info + " connection attempts."); assertTrue(info.getLatestNodeStateRequestTime() != 0, "Node " + info + " has no last request time."); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 5256c952d15..af6e2d2b104 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -968,7 +968,7 @@ public class StateChangeTest extends FleetControllerTest { setUpFleetController(useFakeTimer, builder); StateWaiter waiter = new StateWaiter(timer); - fleetController.addSystemStateListener(waiter); + fleetController().addSystemStateListener(waiter); // Ensure all nodes have been seen by fleetcontroller and that it has had enough time to possibly have sent a cluster state // Note: this is a candidate state and therefore NOT versioned yet @@ -986,7 +986,7 @@ public class StateChangeTest extends FleetControllerTest { waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:d", timeout()); int version = waiter.getCurrentSystemState().getVersion(); - fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 19, timeout()); + fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 19, timeout()); new StateMessageChecker(nodes) { @Override @@ -1018,12 +1018,12 @@ public class StateChangeTest extends FleetControllerTest { final StateWaiter waiter = new StateWaiter(timer); - fleetController.addSystemStateListener(waiter); + fleetController().addSystemStateListener(waiter); waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:i .1.i:1.0", timeout()); waitForCompleteCycle(); final int version = waiter.getCurrentSystemState().getVersion(); - fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 20, timeout()); + fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 20, timeout()); // The last two versions of the cluster state should be seen (all nodes up, // zero out timestate) @@ -1044,7 +1044,7 @@ public class StateChangeTest extends FleetControllerTest { waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); - fleetController.addSystemStateListener(waiter); + fleetController().addSystemStateListener(waiter); nodes.get(1).failSetSystemState(true); int versionBeforeChange = nodes.get(1).getSystemStatesReceived().get(0).getVersion(); @@ -1052,16 +1052,16 @@ public class StateChangeTest extends FleetControllerTest { waiter.waitForState("version:\\d+ distributor:10 .1.s:d storage:10", timeout()); int versionAfterChange = waiter.getCurrentSystemState().getVersion(); assertTrue(versionAfterChange > versionBeforeChange); - fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(versionAfterChange, 18, timeout()); + fleetController().waitForNodesHavingSystemStateVersionEqualToOrAbove(versionAfterChange, 18, timeout()); // Assert that the failed node has not acknowledged the latest version. // (The version may still be larger than versionBeforeChange if the fleet controller sends a // "stable system" update without timestamps in the meantime - assertTrue(fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getClusterStateVersionBundleAcknowledged() < versionAfterChange); + assertTrue(fleetController().getCluster().getNodeInfo(nodes.get(1).getNode()).getClusterStateVersionBundleAcknowledged() < versionAfterChange); // Ensure non-concurrent access to getNewestSystemStateVersionSent synchronized(timer) { - int sentVersion = fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getNewestSystemStateVersionSent(); + int sentVersion = fleetController().getCluster().getNodeInfo(nodes.get(1).getNode()).getNewestSystemStateVersionSent(); assertTrue(sentVersion == -1 || sentVersion == versionAfterChange); } } @@ -1122,7 +1122,7 @@ public class StateChangeTest extends FleetControllerTest { waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); - fleetController.addSystemStateListener(waiter); + fleetController().addSystemStateListener(waiter); // Simulate netsplit. Take node down without node booting assertTrue(nodes.get(0).isDistributor()); @@ -1147,7 +1147,7 @@ public class StateChangeTest extends FleetControllerTest { for (ConfiguredNode i : options.nodes()) { Node nodeId = new Node(NodeType.STORAGE, i.index()); long ts = lastState.getNodeState(nodeId).getStartTimestamp(); - assertTrue(ts > 0, nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController.getCluster().getNodeInfo(nodeId).getStartTimestamp()); + assertTrue(ts > 0, nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController().getCluster().getNodeInfo(nodeId).getStartTimestamp()); } } else { for (ConfiguredNode i : options.nodes()) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java index 86bf7a66458..d697dc40bb2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java @@ -33,11 +33,11 @@ public class WantedStateTest extends FleetControllerTest { setWantedState(nodes.get(1), State.MAINTENANCE, "Foo"); waitForState("version:\\d+ distributor:10 storage:10 .0.s:m"); - assertEquals("Foo", fleetController.getWantedNodeState(nodes.get(1).getNode()).getDescription()); + assertEquals("Foo", fleetController().getWantedNodeState(nodes.get(1).getNode()).getDescription()); setWantedState(nodes.get(1), State.MAINTENANCE, "Bar"); waitForCompleteCycle(); - assertEquals("Bar", fleetController.getWantedNodeState(nodes.get(1).getNode()).getDescription()); + assertEquals("Bar", fleetController().getWantedNodeState(nodes.get(1).getNode()).getDescription()); setWantedState(nodes.get(1), State.UP, null); waitForState("version:\\d+ distributor:10 storage:10"); |