Shut down ClusterController from last configurer, thus before ZK

2 files changed, 40 insertions, 11 deletions

diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java
index 1eca2d18493..5f606c68c1e 100644
--- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java
+++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java
@@ -17,6 +17,8 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 
 /**
@@ -30,6 +32,8 @@ public class ClusterController extends AbstractComponent
     private final JDiscMetricWrapper metricWrapper;
     private final Map<String, FleetController> controllers = new TreeMap<>();
     private final Map<String, StatusHandler.ContainerStatusPageServer> status = new TreeMap<>();
+    private final AtomicInteger referents = new AtomicInteger();
+    private final AtomicBoolean shutdown = new AtomicBoolean();
 
     /**
      * Dependency injection constructor for controller. A {@link VespaZooKeeperServer} argument is required
@@ -43,6 +47,7 @@ public class ClusterController extends AbstractComponent
     }
 
     public void setOptions(FleetControllerOptions options, Metric metricImpl) throws Exception {
+        referents.incrementAndGet();
         metricWrapper.updateMetricImplementation(metricImpl);
         verifyThatZooKeeperWorks(options);
         synchronized (controllers) {
@@ -60,16 +65,32 @@ public class ClusterController extends AbstractComponent
 
     @Override
     public void deconstruct() {
-        synchronized (controllers) {
-            for (FleetController controller : controllers.values()) {
-                try{
-                    shutdownController(controller);
-                } catch (Exception e) {
-                    log.warning("Failed to shut down fleet controller: " + e.getMessage());
+        shutdown();
+    }
+
+    /**
+     * Since we hack around injecting a running ZK here by providing one through the configurer instead,
+     * we must also let the last configurer shut down this controller, to ensure this is shut down
+     * before the ZK server it had injected from the configurers.
+     */
+    void countdown() {
+        if (referents.decrementAndGet() == 0)
+            shutdown();
+    }
+
+    void shutdown() {
+        if (shutdown.compareAndSet(false, true)) {
+            synchronized (controllers) {
+                for (FleetController controller : controllers.values()) {
+                    try {
+                        shutdownController(controller);
+                    }
+                    catch (Exception e) {
+                        log.warning("Failed to shut down fleet controller: " + e.getMessage());
+                    }
                 }
             }
         }
-        super.deconstruct();
     }
 
     @Override
@@ -79,6 +100,8 @@ public class ClusterController extends AbstractComponent
         }
     }
 
+    FleetController getController(String name) { return controllers.get(name); }
+
     @Override
     public StatusHandler.ContainerStatusPageServer get(String cluster) {
         return status.get(cluster);
diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
index 75e838bae3e..a0e290de172 100644
--- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
+++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.clustercontroller.apps.clustercontroller;
 
 import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vdslib.distribution.Distribution;
 import com.yahoo.vdslib.state.NodeType;
@@ -19,9 +20,10 @@ import java.util.Map;
  * When the cluster controller is reconfigured, a new instance of this is created, which will propagate configured
  * options to receivers such as the fleet controller.
  */
-public class ClusterControllerClusterConfigurer {
+public class ClusterControllerClusterConfigurer extends AbstractComponent {
 
     private final FleetControllerOptions options;
+    private final ClusterController controller;
 
     /**
      * The {@link VespaZooKeeperServer} argument is required by the injected {@link ClusterController},
@@ -37,9 +39,13 @@ public class ClusterControllerClusterConfigurer {
                                               Metric metricImpl,
                                               VespaZooKeeperServer started) throws Exception {
         this.options = configure(distributionConfig, fleetcontrollerConfig, slobroksConfig, zookeepersConfig);
-        if (controller != null) {
-            controller.setOptions(options, metricImpl);
-        }
+        this.controller = controller;
+        if (controller != null) controller.setOptions(options, metricImpl);
+    }
+
+    @Override
+    public void deconstruct() {
+        if (controller != null) controller.countdown();
     }
 
     FleetControllerOptions getOptions() { return options; }