From a2074370aab00d3de6387e0397e8dd7aaf94ac20 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 15 May 2023 10:04:27 +0200 Subject: Simplify and minor cleanup --- .../clustercontroller/core/FleetController.java | 2 + .../clustercontroller/core/StateChangeTest.java | 258 +++++++++++---------- 2 files changed, 135 insertions(+), 125 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 05d8c06c76a..5f2a6daf39e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -370,6 +370,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } } + public EventLog getEventLog() { return eventLog; } + private boolean maybePublishOldMetrics() { verifyInControllerThread(); if (isMaster() && cycleCount > 300 + lastMetricUpdateCycleCount) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index dfef6e5a9d0..c0e116ef5fe 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -7,12 +7,10 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; -import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter; -import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; + import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -32,7 +30,6 @@ public class StateChangeTest extends FleetControllerTest { private FleetController ctrl; private DummyCommunicator communicator; - private EventLog eventLog; private void initialize(FleetControllerOptions options) throws Exception { List nodes = new ArrayList<>(); @@ -43,16 +40,8 @@ public class StateChangeTest extends FleetControllerTest { var context = new TestFleetControllerContext(options); communicator = new DummyCommunicator(nodes, timer); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); - eventLog = new EventLog(timer, metricUpdater); - var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); - var stateGatherer = new NodeStateGatherer(timer, timer, eventLog); - var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress(), timer); - var stateGenerator = new StateChangeHandler(context, timer, eventLog); - var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); - var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex(), options.fleetControllerCount(), timer, timer); - ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, null, communicator, database, - stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + boolean start = false; + ctrl = createFleetController(timer, options, context, communicator, communicator, null, start); ctrl.tick(); if (options.fleetControllerCount() == 1) { @@ -71,20 +60,13 @@ public class StateChangeTest extends FleetControllerTest { private void verifyNodeEvents(Node n, String correct) { String actual = ""; - for (NodeEvent e : eventLog.getNodeEvents(n)) { + for (NodeEvent e : ctrl.getEventLog().getNodeEvents(n)) { actual += e.toString() + "\n"; } assertEquals(correct, actual); } - private static List createNodes(int count) { - List nodes = new ArrayList<>(); - for (int i = 0; i < count; i++) - nodes.add(new ConfiguredNode(i, false)); - return nodes; - } - @Test void testNormalStartup() throws Exception { FleetControllerOptions.Builder options = defaultOptions(); @@ -139,22 +121,26 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:8 distributor:10 storage:10", ctrl.getSystemState().toString()); verifyNodeEvents(new Node(NodeType.DISTRIBUTOR, 0), - "Event: distributor.0: Now reporting state U\n" + - "Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: distributor.0: Now reporting state I, i 0.00\n" + - "Event: distributor.0: Altered node state in cluster state from 'U' to 'I, i 0.00'\n" + - "Event: distributor.0: Now reporting state U\n" + - "Event: distributor.0: Altered node state in cluster state from 'I, i 0.00' to 'U'\n"); + """ + Event: distributor.0: Now reporting state U + Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: distributor.0: Now reporting state I, i 0.00 + Event: distributor.0: Altered node state in cluster state from 'U' to 'I, i 0.00' + Event: distributor.0: Now reporting state U + Event: distributor.0: Altered node state in cluster state from 'I, i 0.00' to 'U' + """); verifyNodeEvents(new Node(NodeType.STORAGE, 0), - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.0: Now reporting state I, i 0.00 (ls)\n" + - "Event: storage.0: Altered node state in cluster state from 'U' to 'D'\n" + - "Event: storage.0: Now reporting state I, i 0.100 (read)\n" + - "Event: storage.0: Altered node state in cluster state from 'D' to 'I, i 0.100 (read)'\n" + - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'I, i 0.100 (read)' to 'U'\n"); + """ + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.0: Now reporting state I, i 0.00 (ls) + Event: storage.0: Altered node state in cluster state from 'U' to 'D' + Event: storage.0: Now reporting state I, i 0.100 (read) + Event: storage.0: Altered node state in cluster state from 'D' to 'I, i 0.100 (read)' + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'I, i 0.100 (read)' to 'U' + """); } @Test @@ -217,25 +203,29 @@ public class StateChangeTest extends FleetControllerTest { assert(!ctrl.getReportedNodeState(new Node(NodeType.STORAGE, 0)).hasDescription()); verifyNodeEvents(new Node(NodeType.DISTRIBUTOR, 0), - "Event: distributor.0: Now reporting state U\n" + - "Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: distributor.0: Failed to get node state: D: Closed at other end\n" + - "Event: distributor.0: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: distributor.0: Altered node state in cluster state from 'U' to 'D: Closed at other end'\n" + - "Event: distributor.0: Now reporting state U, t 12345678\n" + - "Event: distributor.0: Altered node state in cluster state from 'D: Closed at other end' to 'U, t 12345678'\n" + - "Event: distributor.0: Altered node state in cluster state from 'U, t 12345678' to 'U'\n"); + """ + Event: distributor.0: Now reporting state U + Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: distributor.0: Failed to get node state: D: Closed at other end + Event: distributor.0: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1. + Event: distributor.0: Altered node state in cluster state from 'U' to 'D: Closed at other end' + Event: distributor.0: Now reporting state U, t 12345678 + Event: distributor.0: Altered node state in cluster state from 'D: Closed at other end' to 'U, t 12345678' + Event: distributor.0: Altered node state in cluster state from 'U, t 12345678' to 'U' + """); verifyNodeEvents(new Node(NodeType.STORAGE, 0), - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.0: Failed to get node state: D: Closed at other end\n" + - "Event: storage.0: Stopped or possibly crashed after 1000 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: storage.0: Altered node state in cluster state from 'U' to 'M: Closed at other end'\n" + - "Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + - "Event: storage.0: Altered node state in cluster state from 'M: Closed at other end' to 'D: Closed at other end'\n" + - "Event: storage.0: Now reporting state U, t 12345679\n" + - "Event: storage.0: Altered node state in cluster state from 'D: Closed at other end' to 'U, t 12345679'\n"); + """ + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.0: Failed to get node state: D: Closed at other end + Event: storage.0: Stopped or possibly crashed after 1000 ms, which is before stable state time period. Premature crash count is now 1. + Event: storage.0: Altered node state in cluster state from 'U' to 'M: Closed at other end' + Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down. + Event: storage.0: Altered node state in cluster state from 'M: Closed at other end' to 'D: Closed at other end' + Event: storage.0: Now reporting state U, t 12345679 + Event: storage.0: Altered node state in cluster state from 'D: Closed at other end' to 'U, t 12345679' + """); assertEquals(1, ctrl.getCluster().getNodeInfo(new Node(NodeType.DISTRIBUTOR, 0)).getPrematureCrashCount()); assertEquals(1, ctrl.getCluster().getNodeInfo(new Node(NodeType.STORAGE, 0)).getPrematureCrashCount()); @@ -299,22 +289,26 @@ public class StateChangeTest extends FleetControllerTest { assertEquals(0, ctrl.getCluster().getNodeInfo(new Node(NodeType.STORAGE, 0)).getPrematureCrashCount()); verifyNodeEvents(new Node(NodeType.DISTRIBUTOR, 0), - "Event: distributor.0: Now reporting state U\n" + - "Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: distributor.0: Failed to get node state: D: controlled shutdown\n" + - "Event: distributor.0: Altered node state in cluster state from 'U' to 'D: controlled shutdown'\n" + - "Event: distributor.0: Now reporting state U\n" + - "Event: distributor.0: Altered node state in cluster state from 'D: controlled shutdown' to 'U'\n"); + """ + Event: distributor.0: Now reporting state U + Event: distributor.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: distributor.0: Failed to get node state: D: controlled shutdown + Event: distributor.0: Altered node state in cluster state from 'U' to 'D: controlled shutdown' + Event: distributor.0: Now reporting state U + Event: distributor.0: Altered node state in cluster state from 'D: controlled shutdown' to 'U' + """); verifyNodeEvents(new Node(NodeType.STORAGE, 0), - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.0: Failed to get node state: D: controlled shutdown\n" + - "Event: storage.0: Altered node state in cluster state from 'U' to 'M: controlled shutdown'\n" + - "Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + - "Event: storage.0: Altered node state in cluster state from 'M: controlled shutdown' to 'D: controlled shutdown'\n" + - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'D: controlled shutdown' to 'U'\n"); + """ + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.0: Failed to get node state: D: controlled shutdown + Event: storage.0: Altered node state in cluster state from 'U' to 'M: controlled shutdown' + Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down. + Event: storage.0: Altered node state in cluster state from 'M: controlled shutdown' to 'D: controlled shutdown' + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'D: controlled shutdown' to 'U' + """); } @@ -354,9 +348,11 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:3 distributor:10 storage:10", ctrl.getSystemState().toString()); verifyNodeEvents(new Node(NodeType.STORAGE, 0), - "Event: storage.0: Now reporting state U\n" + - "Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.0: Node is no longer in slobrok, but we still have a pending state request.\n"); + """ + Event: storage.0: Now reporting state U + Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.0: Node is no longer in slobrok, but we still have a pending state request. + """); } @Test @@ -401,16 +397,18 @@ public class StateChangeTest extends FleetControllerTest { assert(!ctrl.getReportedNodeState(new Node(NodeType.STORAGE, 6)).hasDescription()); verifyNodeEvents(new Node(NodeType.STORAGE, 6), - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + - "Event: storage.6: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: Now reporting state I, i 0.00 (ls)\n" + - "Event: storage.6: Now reporting state I, i 0.600 (read)\n" + - "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'I, i 0.600 (read)'\n" + - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'I, i 0.600 (read)' to 'U'\n"); + """ + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.6: Failed to get node state: D: Connection error: Closed at other end + Event: storage.6: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1. + Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end' + Event: storage.6: Now reporting state I, i 0.00 (ls) + Event: storage.6: Now reporting state I, i 0.600 (read) + Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'I, i 0.600 (read)' + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'I, i 0.600 (read)' to 'U' + """); } @Test @@ -461,15 +459,17 @@ public class StateChangeTest extends FleetControllerTest { assert(!ctrl.getReportedNodeState(new Node(NodeType.STORAGE, 6)).hasDescription()); verifyNodeEvents(new Node(NodeType.STORAGE, 6), - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'R'\n" + - "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + - "Event: storage.6: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: storage.6: Altered node state in cluster state from 'R' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: Now reporting state I, i 0.00 (ls)\n" + - "Event: storage.6: Now reporting state I, i 0.600 (read)\n" + - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'R'\n"); + """ + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'R' + Event: storage.6: Failed to get node state: D: Connection error: Closed at other end + Event: storage.6: Stopped or possibly crashed after 0 ms, which is before stable state time period. Premature crash count is now 1. + Event: storage.6: Altered node state in cluster state from 'R' to 'M: Connection error: Closed at other end' + Event: storage.6: Now reporting state I, i 0.00 (ls) + Event: storage.6: Now reporting state I, i 0.600 (read) + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'R' + """); } @Test @@ -549,17 +549,19 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:7 distributor:10 storage:10", ctrl.getSystemState().toString()); verifyNodeEvents(new Node(NodeType.STORAGE, 6), - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + - "Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + - "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end'\n" + - "Event: storage.6: Now reporting state I, i 0.00100 (ls)\n" + - "Event: storage.6: Now reporting state I, i 0.100 (read)\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Connection error: Closed at other end' to 'I, i 0.100 (read)'\n" + - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'I, i 0.100 (read)' to 'U'\n"); + """ + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.6: Failed to get node state: D: Connection error: Closed at other end + Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end' + Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down. + Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end' + Event: storage.6: Now reporting state I, i 0.00100 (ls) + Event: storage.6: Now reporting state I, i 0.100 (read) + Event: storage.6: Altered node state in cluster state from 'D: Connection error: Closed at other end' to 'I, i 0.100 (read)' + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'I, i 0.100 (read)' to 'U' + """); } @Test @@ -630,21 +632,23 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:8 distributor:10 storage:10", ctrl.getSystemState().toString()); verifyNodeEvents(new Node(NodeType.STORAGE, 6), - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + - "Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + - "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end'\n" + - "Event: storage.6: Now reporting state I, i 0.100 (read)\n" + - "Event: storage.6: Altered node state in cluster state from 'D: Connection error: Closed at other end' to 'I, i 0.100 (read)'\n" + - "Event: storage.6: 5001 milliseconds without initialize progress. Marking node down. Premature crash count is now 1.\n" + - "Event: storage.6: Altered node state in cluster state from 'I, i 0.100 (read)' to 'D'\n" + - "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + - "Event: storage.6: Now reporting state I, i 0.00 (ls)\n" + - "Event: storage.6: Now reporting state I, i 0.100 (read)\n" + - "Event: storage.6: Now reporting state U\n" + - "Event: storage.6: Altered node state in cluster state from 'D' to 'U'\n"); + """ + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.6: Failed to get node state: D: Connection error: Closed at other end + Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end' + Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down. + Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end' + Event: storage.6: Now reporting state I, i 0.100 (read) + Event: storage.6: Altered node state in cluster state from 'D: Connection error: Closed at other end' to 'I, i 0.100 (read)' + Event: storage.6: 5001 milliseconds without initialize progress. Marking node down. Premature crash count is now 1. + Event: storage.6: Altered node state in cluster state from 'I, i 0.100 (read)' to 'D' + Event: storage.6: Failed to get node state: D: Connection error: Closed at other end + Event: storage.6: Now reporting state I, i 0.00 (ls) + Event: storage.6: Now reporting state I, i 0.100 (read) + Event: storage.6: Now reporting state U + Event: storage.6: Altered node state in cluster state from 'D' to 'U' + """); } @@ -1188,11 +1192,13 @@ public class StateChangeTest extends FleetControllerTest { } verifyNodeEvents(new Node(NodeType.STORAGE, 2), - "Event: storage.2: Now reporting state U\n" + - "Event: storage.2: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.2: Failed to get node state: D: foo\n" + - "Event: storage.2: Stopped or possibly crashed after 500 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: storage.2: Altered node state in cluster state from 'U' to 'M: foo'\n"); + """ + Event: storage.2: Now reporting state U + Event: storage.2: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.2: Failed to get node state: D: foo + Event: storage.2: Stopped or possibly crashed after 500 ms, which is before stable state time period. Premature crash count is now 1. + Event: storage.2: Altered node state in cluster state from 'U' to 'M: foo' + """); // Note: even though max transition time has passed, events are now emitted only on cluster state // publish edges. These are currently suppressed when the cluster state is down, as all cluster down // states are considered similar to other cluster down states. This is not necessarily optimal, but @@ -1232,11 +1238,13 @@ public class StateChangeTest extends FleetControllerTest { // We should only get "Altered min distribution bit count" event once, not 9 times. verifyNodeEvents(new Node(NodeType.STORAGE, 2), - "Event: storage.2: Now reporting state U\n" + - "Event: storage.2: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + - "Event: storage.2: Now reporting state I, i 0.100 (read)\n" + - "Event: storage.2: Altered node state in cluster state from 'U' to 'I, i 0.100 (read)'\n" + - "Event: storage.2: Altered min distribution bit count from 16 to 17\n"); + """ + Event: storage.2: Now reporting state U + Event: storage.2: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U' + Event: storage.2: Now reporting state I, i 0.100 (read) + Event: storage.2: Altered node state in cluster state from 'U' to 'I, i 0.100 (read)' + Event: storage.2: Altered min distribution bit count from 16 to 17 + """); } -- cgit v1.2.3