ZooKeeper-persist and load published cluster state bundles

Store synchronously upon each new versioned state, load whenever controller
is elected master. Effectively carries over visible node states from one
controller's lifetime to the next. This removes the edge case where default
bucket space content nodes are marked as in Maintainence until their global
merge status is known.

To avoid controller tripping over its own feet, state bundles are now _not_
versioned at all until the initial send time period has passed. This prevents
overwriting the state persisted from a previous controller with a transient
state where all nodes are down due to not having Slobrok contact yet.

A new cluster state recompute+send edge has been added when the master passes
its initial state send time period.

1 files changed, 5 insertions, 3 deletions

diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
index b10a8101c37..32de3591f2d 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java
@@ -5,6 +5,7 @@ import com.yahoo.jrt.*;
 import com.yahoo.vdslib.distribution.ConfiguredNode;
 import com.yahoo.vdslib.state.*;
 import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler;
+import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory;
 import com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter;
 import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter;
 import org.junit.Before;
@@ -46,7 +47,7 @@ public class StateChangeTest extends FleetControllerTest {
         ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution,
                                                     options.minStorageNodesUp, options.minRatioOfStorageNodesUp);
         NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog);
-        DatabaseHandler database = new DatabaseHandler(timer, options.zooKeeperServerAddress, options.fleetControllerIndex, timer);
+        DatabaseHandler database = new DatabaseHandler(new ZooKeeperDatabaseFactory(), timer, options.zooKeeperServerAddress, options.fleetControllerIndex, timer);
         StateChangeHandler stateGenerator = new StateChangeHandler(timer, eventLog, metricUpdater);
         SystemStateBroadcaster stateBroadcaster = new SystemStateBroadcaster(timer, timer);
         MasterElectionHandler masterElectionHandler = new MasterElectionHandler(options.fleetControllerIndex, options.fleetControllerCount, timer, timer);
@@ -955,8 +956,9 @@ public class StateChangeTest extends FleetControllerTest {
         StateWaiter waiter = new StateWaiter(timer);
         fleetController.addSystemStateListener(waiter);
 
-            // Ensure all nodes have been seen by fleetcontroller and that it has had enough time to possibly have sent a cluster state
-        waiter.waitForState("version:\\d+ distributor:10 (\\.\\d+\\.t:\\d+ )*storage:10 (\\.\\d+\\.t:\\d+ )*.1.s:d( \\.\\d+\\.t:\\d+)*", timeoutMS);
+        // Ensure all nodes have been seen by fleetcontroller and that it has had enough time to possibly have sent a cluster state
+        // Note: this is a candidate state and therefore NOT versioned yet
+        waiter.waitForState("^distributor:10 (\\.\\d+\\.t:\\d+ )*storage:10 (\\.\\d+\\.t:\\d+ )*.1.s:d( \\.\\d+\\.t:\\d+)*", timeoutMS);
         waitForCompleteCycle();
         new StateMessageChecker(nodes) {
             @Override int expectedMessageCount(final DummyVdsNode node) { return 0; }