Merge pull request #24621 from vespa-engine/jonmv/flag-to-stop-deployment-jobs

Allow turning off jobs in a zone, optionally per application

3 files changed, 11 insertions, 1 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index d5a31a07408..169cde8437a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -378,6 +378,7 @@ public class DeploymentTrigger {
             DeploymentStatus.Job job = jobsList.get(0);
             if (     job.readyAt().isPresent()
                 && ! clock.instant().isBefore(job.readyAt().get())
+                && ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type()))
                 && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))
                 &&   abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters.
                 jobs.add(deploymentJob(status.application().require(jobId.application().instance()),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index edddaa5dee7..08cf8d2e1c4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -10,6 +10,9 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Tags;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.flags.FetchVector.Dimension;
+import com.yahoo.vespa.flags.ListFlag;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -118,6 +121,7 @@ public class JobController {
     private final BufferedLogStore logs;
     private final TesterCloud cloud;
     private final JobMetrics metric;
+    private final ListFlag<String> disabledZones;
 
     private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { });
 
@@ -128,6 +132,7 @@ public class JobController {
         this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore());
         this.cloud = controller.serviceRegistry().testerCloud();
         this.metric = new JobMetrics(controller.metric());
+        this.disabledZones = PermanentFlags.DISABLED_DEPLOYMENT_ZONES.bindTo(controller.flagSource());
     }
 
     public TesterCloud cloud() { return cloud; }
@@ -144,6 +149,10 @@ public class JobController {
             }
     }
 
+    public boolean isDisabled(JobId id) {
+        return disabledZones.with(Dimension.APPLICATION_ID, id.application().serializedForm()).value().contains(id.type().zone().value());
+    }
+
     /** Returns all entries currently logged for the given run. */
     public Optional<RunLog> details(RunId id) {
         return details(id, -1);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index cd48d6839f3..de1d3ef955d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -75,7 +75,7 @@ public class JobRunner extends ControllerMaintainer {
     }
 
     public void advance(Run run) {
-        advance(run.id());
+        if ( ! jobs.isDisabled(run.id().job())) advance(run.id());
     }
 
     /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */