Merge pull request #16856 from vespa-engine/vekterli/immediately-exit-cc-if-node-index-changed-live

Immediately exit cluster controller if node index config is changed live

1 files changed, 20 insertions, 3 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
index bafd7168f03..5d5ffb917d2 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
@@ -70,6 +70,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
     private final AtomicBoolean running = new AtomicBoolean(true);
     private FleetControllerOptions options;
     private FleetControllerOptions nextOptions;
+    private final int configuredIndex;
     private final List<SystemStateListener> systemStateListeners = new CopyOnWriteArrayList<>();
     private boolean processingCycle = false;
     private boolean wantedStateChanged = false;
@@ -125,6 +126,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
                            MetricUpdater metricUpdater,
                            FleetControllerOptions options) {
         log.info("Starting up cluster controller " + options.fleetControllerIndex + " for cluster " + cluster.getName());
+        this.configuredIndex = options.fleetControllerIndex;
         this.timer = timer;
         this.monitor = timer;
         this.eventLog = eventLog;
@@ -487,6 +489,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
     /** This is called when the options field has been set to a new set of options */
     private void propagateOptions() {
         verifyInControllerThread();
+        selfTerminateIfConfiguredNodeIndexHasChanged();
 
         if (changesConfiguredNodeSet(options.nodes)) {
             // Force slobrok node re-fetch in case of changes to the set of configured nodes
@@ -544,6 +547,16 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
         nextConfigGeneration = -1;
     }
 
+    private void selfTerminateIfConfiguredNodeIndexHasChanged() {
+        if (options.fleetControllerIndex != configuredIndex) {
+            log.warning(String.format("Got new configuration where CC index has changed from %d to %d. We do not support "+
+                                      "doing this live; immediately exiting now to force new configuration",
+                                      configuredIndex, options.fleetControllerIndex));
+            prepareShutdownEdge();
+            System.exit(1);
+        }
+    }
+
     public StatusPageResponse fetchStatusPage(StatusPageServer.HttpRequest httpRequest) {
         verifyInControllerThread();
         StatusPageResponse.ResponseCode responseCode;
@@ -1103,12 +1116,16 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
             synchronized (monitor) { running.set(false); }
             System.exit(1);
         } finally {
-            running.set(false);
-            failAllVersionDependentTasks();
-            synchronized (monitor) { monitor.notifyAll(); }
+            prepareShutdownEdge();
         }
     }
 
+    private void prepareShutdownEdge() {
+        running.set(false);
+        failAllVersionDependentTasks();
+        synchronized (monitor) { monitor.notifyAll(); }
+    }
+
     public DatabaseHandler.Context databaseContext = new DatabaseHandler.Context() {
         @Override
         public ContentCluster getCluster() { return cluster; }