diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-09-01 13:55:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-01 13:55:39 +0200 |
commit | b7df8e7c1f7832cd190526d3915434002d1de6e5 (patch) | |
tree | d3ed3aa1f0b635efb893371494e963269dcc3b6e /clustercontroller-core/src | |
parent | e33251172a1bafde4fb78fe9a3f041498eab4411 (diff) | |
parent | e4a1eb40d1ff0f306fac933beed642c685fd2203 (diff) |
Merge pull request #23889 from vespa-engine/hmusum/cleanup-23
Cluster controller cleanup, part 9 [run-systemtest]
Diffstat (limited to 'clustercontroller-core/src')
11 files changed, 54 insertions, 71 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index a383c225e89..26bb21da3d4 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -31,7 +31,7 @@ public class DatabaseTest extends FleetControllerTest { FleetControllerOptions.Builder builder = defaultOptions("mycluster"); builder.setZooKeeperServerAddress("127.0.0.1"); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); log.info("WAITING FOR STABLE SYSTEM"); waitForStableSystem(); @@ -87,7 +87,7 @@ public class DatabaseTest extends FleetControllerTest { .setMinRatioOfStorageNodesUp(0) .setZooKeeperServerAddress("localhost"); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); // Populate map of wanted states we should have diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index e0c85036bf4..fbaf41fdd47 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -33,6 +33,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static ai.vespa.validation.Validation.requireAtLeast; + /** * * Used to fake a node in VDS, such that we can test the fleetcontroller without dummy interface for talking to @@ -118,14 +120,15 @@ public class DummyVdsNode { } }; - public DummyVdsNode(Timer timer, DummyVdsNodeOptions options, String[] slobrokConnectionSpecs, String clusterName, boolean distributor, int index) throws Exception { + public DummyVdsNode(Timer timer, DummyVdsNodeOptions options, String[] slobrokConnectionSpecs, String clusterName, + NodeType nodeType, int index) throws Exception { this.timer = timer; this.slobrokConnectionSpecs = slobrokConnectionSpecs; this.clusterName = clusterName; - type = distributor ? NodeType.DISTRIBUTOR : NodeType.STORAGE; + type = nodeType; this.index = index; this.nodeState = new NodeState(type, State.UP); - this.stateCommunicationVersion = options.stateCommunicationVersion; + this.stateCommunicationVersion = requireAtLeast(options.stateCommunicationVersion, "state communication version cannot be less than 2", 2); messageResponder.start(); nodeState.setStartTimestamp(timer.getCurrentTimeInMillis() / 1000); } @@ -299,14 +302,6 @@ public class DummyVdsNode { m.returnDesc(0, "returnCode", "Returncode of request. Should be 0 = OK"); supervisor.addMethod(m); - m = new Method("getnodestate", "", "issi", this::rpc_getNodeState); - m.methodDesc("Get nodeState of a node"); - m.returnDesc(0, "returnCode", "Returncode of request. Should be 1 = OK"); - m.returnDesc(1, "returnMessage", "Textual error message if returncode is not ok."); - m.returnDesc(2, "nodeState", "The node state of the given node"); - m.returnDesc(3, "progress", "Progress in percent of node initialization"); - supervisor.addMethod(m); - m = new Method("setsystemstate", "s", "is", this::rpc_setSystemState); m.methodDesc("Set system state of entire system"); m.paramDesc(0, "systemState", "new systemstate"); @@ -362,21 +357,6 @@ public class DummyVdsNode { } } - private void rpc_getNodeState(Request req) { - synchronized(timer) { - if (!negotiatedHandle) { - req.setError(75000, "Connection not bound to a handle"); - return; - } - String stateString = nodeState.serialize(-1, true); - log.log(Level.FINE, () -> "Dummy node " + this + " got old type get node state request, answering: " + stateString); - req.returnValues().add(new Int32Value(1)); - req.returnValues().add(new StringValue("")); - req.returnValues().add(new StringValue(stateString)); - req.returnValues().add(new Int32Value(0)); - } - } - boolean sendGetNodeStateReply(int index) { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java index c391fc27813..d6b5e7fa5d2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java @@ -8,6 +8,11 @@ import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; * over regular RPC. */ public class DummyVdsNodeOptions { - // 0 - 4.1, 1 - 4.2-5.0.10, 2 - 5.0.11+, 3 - 6.220+, 4 - 7.24+ + // Rpc method versions and which Vespa versions supports each version: + // 0 - 4.1 + // 1 - 4.2-5.0.10 + // 2 - 5.0.11+ + // 3 - 6.220+ + // 4 - 7.24+ int stateCommunicationVersion = RPCCommunicator.ACTIVATE_CLUSTER_STATE_VERSION_RPC_VERSION; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index dd723cf3d37..58a076a6176 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -35,15 +35,15 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.vdslib.state.NodeType.DISTRIBUTOR; +import static com.yahoo.vdslib.state.NodeType.STORAGE; import static org.junit.jupiter.api.Assertions.fail; /** @@ -201,8 +201,8 @@ public abstract class FleetControllerTest implements Waiter { } } - protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options) throws Exception { - setUpVdsNodes(useFakeTimer, options, false); + protected void setUpVdsNodes(boolean useFakeTimer) throws Exception { + setUpVdsNodes(useFakeTimer, new DummyVdsNodeOptions(), false); } protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected) throws Exception { setUpVdsNodes(useFakeTimer, options, startDisconnected, DEFAULT_NODE_COUNT); @@ -213,15 +213,23 @@ public abstract class FleetControllerTest implements Waiter { nodeIndexes.add(this.nodes.size()/2 + i); // divide by 2 because there are 2 nodes (storage and distributor) per index setUpVdsNodes(useFakeTimer, options, startDisconnected, nodeIndexes); } - protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected, Set<Integer> nodeIndexes) throws Exception { - String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); + protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions nodeOptions, boolean startDisconnected, Set<Integer> nodeIndexes) throws Exception { for (int nodeIndex : nodeIndexes) { - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName(), true, nodeIndex)); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName(), false, nodeIndex)); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); + nodes.add(createNode(useFakeTimer, nodeOptions, startDisconnected, DISTRIBUTOR, nodeIndex)); + nodes.add(createNode(useFakeTimer, nodeOptions, startDisconnected, STORAGE, nodeIndex)); } } + + private DummyVdsNode createNode(boolean useFakeTimer, DummyVdsNodeOptions nodeOptions, boolean startDisconnected, + NodeType nodeType, int nodeIndex) throws Exception { + String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); + DummyVdsNode node = new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), nodeOptions, connectionSpecs, + options.clusterName(), nodeType, nodeIndex); + if ( ! startDisconnected) + node.connect(); + return node; + } + // TODO: Replace all usages of the above setUp methods with this one, and remove the nodes field /** @@ -230,14 +238,10 @@ public abstract class FleetControllerTest implements Waiter { * the returned list is twice as large as configuredNodes. */ protected List<DummyVdsNode> setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected, List<ConfiguredNode> configuredNodes) throws Exception { - String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); nodes = new ArrayList<>(); - final boolean distributor = true; for (ConfiguredNode configuredNode : configuredNodes) { - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName(), distributor, configuredNode.index())); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName(), !distributor, configuredNode.index())); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); + nodes.add(createNode(useFakeTimer, options, startDisconnected, DISTRIBUTOR, configuredNode.index())); + nodes.add(createNode(useFakeTimer, options, startDisconnected, STORAGE, configuredNode.index())); } return nodes; } @@ -274,13 +278,6 @@ public abstract class FleetControllerTest implements Waiter { subsetWaiter.waitForState(expectedState); } - static Map<NodeType, Integer> transitionTimes(int milliseconds) { - Map<NodeType, Integer> maxTransitionTime = new TreeMap<>(); - maxTransitionTime.put(NodeType.DISTRIBUTOR, milliseconds); - maxTransitionTime.put(NodeType.STORAGE, milliseconds); - return maxTransitionTime; - } - protected void tearDownSystem() throws Exception { if (testName != null) { //log.log(Level.INFO, "STOPPING TEST " + testName); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index 0594fc13ef5..c5d7c520751 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -29,7 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(FleetControllerTest.CleanupZookeeperLogsOnSuccess.class) -@Timeout(20) +@Timeout(120) public class MasterElectionTest extends FleetControllerTest { private static final Logger log = Logger.getLogger(MasterElectionTest.class.getName()); @@ -222,7 +222,7 @@ public class MasterElectionTest extends FleetControllerTest { setUpFleetController(3, false, options); // Currently need to have content nodes present for the cluster controller to even bother // attempting to persisting its cluster state version to ZK. - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); @@ -454,7 +454,7 @@ public class MasterElectionTest extends FleetControllerTest { .setMinDistributorNodesUp(0) .setMinStorageNodesUp(1); setUpFleetController(3, false, builder); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); @@ -499,7 +499,7 @@ public class MasterElectionTest extends FleetControllerTest { .setMinTimeBeforeFirstSystemStateBroadcast(100000); boolean useFakeTimer = false; setUpFleetController(3, useFakeTimer, builder); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); @@ -542,7 +542,7 @@ public class MasterElectionTest extends FleetControllerTest { .setMasterZooKeeperCooldownPeriod(1) .setMinTimeBeforeFirstSystemStateBroadcast(100000); setUpFleetController(3, false, builder); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java index a1f47e32a7d..483c92d4861 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java @@ -13,7 +13,7 @@ public class NoZooKeeperTest extends FleetControllerTest { // Null is the default for zooKeeperServerAddress FleetControllerOptions.Builder builder = defaultOptions("mycluster"); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); assertTrue(nodes.get(0).isDistributor()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index c91bb7b19b7..d36486d2eb6 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -84,7 +84,7 @@ public class RpcServerTest extends FleetControllerTest { try { startingTest("RpcServerTest::testFailOccasionallyAndIgnoreToSeeIfOtherTestsThenWork"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); } catch (Throwable t) { } @@ -96,7 +96,7 @@ public class RpcServerTest extends FleetControllerTest { startingTest("RpcServerTest::testGetSystemState"); FleetControllerOptions.Builder options = defaultOptions("mycluster"); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); assertTrue(nodes.get(0).isDistributor()); @@ -166,7 +166,7 @@ public class RpcServerTest extends FleetControllerTest { builder.setMaxInitProgressTime(30000); builder.setStableStateTimePeriod(60000); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); setWantedNodeState(State.DOWN, NodeType.DISTRIBUTOR, 2); @@ -490,7 +490,7 @@ public class RpcServerTest extends FleetControllerTest { FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(2, 10))); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int rpcPort = fleetController.getRpcPort(); @@ -518,7 +518,7 @@ public class RpcServerTest extends FleetControllerTest { FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(2, 10))); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int rpcPort = fleetController.getRpcPort(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index be3f7f27488..4cf69b778a2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -20,7 +20,7 @@ public class SlobrokTest extends FleetControllerTest { .setNodeStateRequestTimeoutMS(60 * 60 * 1000) .setMaxSlobrokDisconnectGracePeriod(60 * 60 * 1000); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int version = fleetController.getSystemState().getVersion(); @@ -74,7 +74,7 @@ public class SlobrokTest extends FleetControllerTest { .setMaxSlobrokDisconnectGracePeriod(60 * 1000) .setNodeStateRequestTimeoutMS(10000 * 60 * 1000); setUpFleetController(true, builder); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int version = fleetController.getSystemState().getVersion(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 30eae09724e..56fcd4666e8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -1038,7 +1038,7 @@ public class StateChangeTest extends FleetControllerTest { startingTest("StateChangeTest::testDontTagFailingSetSystemStateOk()"); FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); @@ -1116,7 +1116,7 @@ public class StateChangeTest extends FleetControllerTest { startingTest("StateChangeTest::testSetAllTimestampsAfterDowntime"); FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java index 712ed9172d5..6ecfc9ca550 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.NodeType; import org.junit.jupiter.api.Test; import java.time.Instant; import java.util.concurrent.TimeoutException; @@ -35,8 +36,8 @@ public class StateGatherTest extends FleetControllerTest { setUpFleetController(true, builder); String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); DummyVdsNodeOptions dummyOptions = new DummyVdsNodeOptions(); - DummyVdsNode dnode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), true, 0); - DummyVdsNode snode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), false, 0); + DummyVdsNode dnode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), NodeType.DISTRIBUTOR, 0); + DummyVdsNode snode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), NodeType.STORAGE, 0); dnode.connect(); snode.connect(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java index 7bbae78dbab..e89f5fc7e58 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java @@ -12,7 +12,7 @@ public class WantedStateTest extends FleetControllerTest { void testSettingStorageNodeMaintenanceAndBack() throws Exception { startingTest("WantedStateTest::testSettingStorageNodeMaintenanceAndBack()"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); setWantedState(nodes.get(1), State.MAINTENANCE, null); @@ -26,7 +26,7 @@ public class WantedStateTest extends FleetControllerTest { void testOverridingWantedStateOtherReason() throws Exception { startingTest("WantedStateTest::testOverridingWantedStateOtherReason()"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); setWantedState(nodes.get(1), State.MAINTENANCE, "Foo"); |