diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2022-10-28 15:06:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-28 15:06:33 +0200 |
commit | 44595d372748d960535e72ab5086fc1412db867c (patch) | |
tree | bbca6b459aece588b8e04f364edbb9e5f0c0f0dc | |
parent | 8bd3f408ffdf1dff8bcbef381bd96285b58ed9ae (diff) | |
parent | 6e8c3cf510c1690009f80f7a5efb26097884e9fd (diff) |
Merge pull request #24621 from vespa-engine/jonmv/flag-to-stop-deployment-jobs
Allow turning off jobs in a zone, optionally per application
4 files changed, 18 insertions, 1 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d5a31a07408..169cde8437a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -378,6 +378,7 @@ public class DeploymentTrigger { DeploymentStatus.Job job = jobsList.get(0); if ( job.readyAt().isPresent() && ! clock.instant().isBefore(job.readyAt().get()) + && ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type())) && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId)) && abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters. jobs.add(deploymentJob(status.application().require(jobId.application().instance()), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index edddaa5dee7..08cf8d2e1c4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -10,6 +10,9 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.FetchVector.Dimension; +import com.yahoo.vespa.flags.ListFlag; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -118,6 +121,7 @@ public class JobController { private final BufferedLogStore logs; private final TesterCloud cloud; private final JobMetrics metric; + private final ListFlag<String> disabledZones; private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); @@ -128,6 +132,7 @@ public class JobController { this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); this.cloud = controller.serviceRegistry().testerCloud(); this.metric = new JobMetrics(controller.metric()); + this.disabledZones = PermanentFlags.DISABLED_DEPLOYMENT_ZONES.bindTo(controller.flagSource()); } public TesterCloud cloud() { return cloud; } @@ -144,6 +149,10 @@ public class JobController { } } + public boolean isDisabled(JobId id) { + return disabledZones.with(Dimension.APPLICATION_ID, id.application().serializedForm()).value().contains(id.type().zone().value()); + } + /** Returns all entries currently logged for the given run. */ public Optional<RunLog> details(RunId id) { return details(id, -1); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index cd48d6839f3..de1d3ef955d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -75,7 +75,7 @@ public class JobRunner extends ControllerMaintainer { } public void advance(Run run) { - advance(run.id()); + if ( ! jobs.isDisabled(run.id().job())) advance(run.id()); } /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index f68f6816ec8..4ea67ccdc73 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -296,6 +296,13 @@ public class PermanentFlags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundListFlag<String> DISABLED_DEPLOYMENT_ZONES = defineListFlag( + "disabled-deployment-zones", List.of(), String.class, + "The zones, e.g., prod.norway-71, where deployments jobs are currently disabled", + "Takes effect immediately", + APPLICATION_ID + ); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( |