diff options
author | Martin Polden <mpolden@mpolden.no> | 2022-03-01 08:45:00 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-03-01 08:45:00 +0100 |
commit | d17415c26842855b9a7c71f31ea27ca9abe2caaf (patch) | |
tree | 3751a1436ff9b4cd7cf5cc07de145fd7a4c4448e /controller-server | |
parent | 1b836d788a2f71ca26210b24a34fbd1faebd700b (diff) | |
parent | 72b94c670761848d79b397101970ba04cf587e2f (diff) |
Merge pull request #21463 from vespa-engine/jonmv/deployment-orch-adjustments
Jonmv/deployment orch adjustments
Diffstat (limited to 'controller-server')
21 files changed, 151 insertions, 93 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 20fa539c820..ed5df62ca5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -39,6 +39,7 @@ import java.util.stream.Collectors; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; /** * Responsible for scheduling deployment jobs in a build system and keeping @@ -166,18 +167,24 @@ public class DeploymentTrigger { return triggeredJobs; } + + /** Attempts to trigger the given job. */ + private void trigger(Job job) { + trigger(job, null); + } + /** Attempts to trigger the given job. */ - public void trigger(Job job) { + private void trigger(Job job, String reason) { log.log(Level.FINE, () -> "Triggering " + job); applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> { - jobs.start(job.applicationId(), job.jobType, job.versions); + jobs.start(job.applicationId(), job.jobType, job.versions, false, Optional.ofNullable(reason)); applications().store(application.with(job.applicationId().instance(), instance -> instance.withJobPause(job.jobType, OptionalLong.empty()))); }); } /** Force triggering of a job for given instance, with same versions as last run. */ - public JobId reTrigger(ApplicationId applicationId, JobType jobType) { + public JobId reTrigger(ApplicationId applicationId, JobType jobType, String reason) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); JobId job = new JobId(instance.id(), jobType); @@ -185,42 +192,51 @@ public class DeploymentTrigger { Versions versions = jobStatus.lastTriggered() .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")) .versions(); - trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant())); + trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant()), reason); return job; } /** Force triggering of a job for given instance. */ - public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String user, boolean requireTests) { + public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String reason, boolean requireTests, + boolean upgradeRevision, boolean upgradePlatform) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); JobId job = new JobId(instance.id(), jobType); if (job.type().environment().isManuallyDeployed()) - return forceTriggerManualJob(job); + return forceTriggerManualJob(job, reason); DeploymentStatus status = jobs.deploymentStatus(application); - Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion()); + Change change = instance.change(); + if ( ! upgradeRevision && change.application().isPresent()) change = change.withoutApplication(); + if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); + Versions versions = Versions.from(change, application, status.deploymentFor(job), controller.readSystemVersion()); DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change()); - Map<JobId, List<DeploymentStatus.Job>> jobs = status.testJobs(Map.of(job, List.of(toTrigger))); - if (jobs.isEmpty() || ! requireTests) - jobs = Map.of(job, List.of(toTrigger)); + Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger))); + + Map<JobId, List<DeploymentStatus.Job>> jobs = testJobs.isEmpty() || ! requireTests + ? Map.of(job, List.of(toTrigger)) + : testJobs.entrySet().stream() + .filter(entry -> controller.jobController().last(entry.getKey()).map(Run::hasEnded).orElse(true)) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + jobs.forEach((jobId, versionsList) -> { - trigger(deploymentJob(instance, versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant())); + trigger(deploymentJob(instance, versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant()), reason); }); return List.copyOf(jobs.keySet()); } - private List<JobId> forceTriggerManualJob(JobId job) { + private List<JobId> forceTriggerManualJob(JobId job, String reason) { Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); Versions target = new Versions(controller.readSystemVersion(), last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); - jobs.start(job.application(), job.type(), target, true); + jobs.start(job.application(), job.type(), target, true, Optional.of(reason)); return List.of(job); } /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ - public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment) { + public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { JobType jobType = JobType.from(controller.system(), deployment.zoneId()) .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() @@ -241,10 +257,10 @@ public class DeploymentTrigger { .collect(toList()); controller.curator().writeRetriggerEntries(newList); } - controller.jobController().abort(run.id()); + controller.jobController().abort(run.id(), "force re-triggered"); return Optional.empty(); } else { - return Optional.of(reTrigger(deployment.applicationId(), jobType)); + return Optional.of(reTrigger(deployment.applicationId(), jobType, reason)); } } @@ -352,12 +368,15 @@ public class DeploymentTrigger { private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { status.jobs().get(job) .flatMap(JobStatus::lastTriggered) - .filter(last -> ! last.hasEnded()) + .filter(last -> ! last.hasEnded() && last.reason().isEmpty()) .ifPresent(last -> { if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) && versions.versions().sourcesMatchIfPresent(last.versions()))) { log.log(Level.INFO, "Aborting outdated run " + last); - controller.jobController().abort(last.id()); + controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + + jobs.get(job).stream() + .map(scheduled -> scheduled.versions().toString()) + .collect(Collectors.joining(", "))); } }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 60339a3233c..70640278f4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -61,6 +61,7 @@ import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; @@ -68,6 +69,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.util.Comparator.naturalOrder; import static java.util.function.Predicate.not; +import static java.util.logging.Level.INFO; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; @@ -440,8 +442,13 @@ public class JobController { } /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */ - public void abort(RunId id) { - locked(id, run -> run.aborted()); + public void abort(RunId id, String reason) { + locked(id, run -> { + run.stepStatuses().entrySet().stream() + .filter(entry -> entry.getValue() == unfinished) + .forEach(entry -> log(id, entry.getKey(), INFO, "Aborting run: " + reason)); + return run.aborted(); + }); } /** Accepts and stores a new application package and test jar pair under a generated application version key. */ @@ -488,24 +495,19 @@ public class JobController { } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions) { - start(id, type, versions, false); - } - - /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment) { - start(id, type, versions, isRedeployment, JobProfile.of(type)); + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, Optional<String> reason) { + start(id, type, versions, isRedeployment, JobProfile.of(type), reason); } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile) { + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile, Optional<String> reason) { locked(id, type, __ -> { Optional<Run> last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) throw new IllegalArgumentException("Cannot start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile)); + curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile, reason)); metric.jobStarted(newId.job()); }); } @@ -550,7 +552,8 @@ public class JobController { lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetApplication())), false, - dryRun ? JobProfile.developmentDryRun : JobProfile.development); + dryRun ? JobProfile.developmentDryRun : JobProfile.development, + Optional.empty()); }); locked(id, type, __ -> { @@ -560,7 +563,7 @@ public class JobController { /** Aborts a run and waits for it complete. */ private void abortAndWait(RunId id) { - abort(id); + abort(id, "replaced by new deployment"); runner.get().accept(last(id.application(), id.type()).get()); while ( ! last(id.application(), id.type()).get().hasEnded()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index c12448ab269..e73d3f52e1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -40,11 +40,12 @@ public class Run { private final Optional<ConvergenceSummary> convergenceSummary; private final Optional<X509Certificate> testerCertificate; private final boolean dryRun; + private final Optional<String> reason; // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, - Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun) { + Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -59,13 +60,14 @@ public class Run { this.convergenceSummary = convergenceSummary; this.testerCertificate = testerCertificate; this.dryRun = dryRun; + this.reason = reason; } - public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { + public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile, Optional<String> triggeredBy) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), Optional.empty(), running, - -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun); + -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); } /** Returns a new Run with the status of the given completed step set accordingly. */ @@ -79,7 +81,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } /** Returns a new Run with a new start time*/ @@ -94,19 +96,19 @@ public class Run { steps.put(step.get(), stepInfo.with(startTime)); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } public Run finished(Instant now) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), sleepUntil, status == running ? success : status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun, reason); } public Run aborted() { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, aborted, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } public Run reset() { @@ -114,43 +116,43 @@ public class Run { Map<Step, StepInfo> reset = new EnumMap<>(steps); reset.replaceAll((step, __) -> StepInfo.initial(step)); return new Run(id, reset, versions, isRedeployment, start, end, sleepUntil, running, -1, lastVespaLogTimestamp, - Optional.empty(), Optional.empty(), testerCertificate, dryRun); + Optional.empty(), Optional.empty(), testerCertificate, dryRun, reason); } public Run with(long lastTestRecord) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun); + Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun, reason); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun); + noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun, reason); } public Run with(X509Certificate testerCertificate) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun); + noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun, reason); } public Run sleepingUntil(Instant instant) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, Optional.of(instant), status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); } /** Returns the id of this run. */ @@ -254,6 +256,11 @@ public class Run { /** Whether this is a dry run deployment. */ public boolean isDryRun() { return dryRun; } + /** The specific reason for triggering this run, if any. This should be empty for jobs triggered bvy deployment orchestration. */ + public Optional<String> reason() { + return reason; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 73ee8527492..bf3b9b90f66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -49,7 +49,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { if ( ! isLikelyNightFor(job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); - controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true); + controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 3f54958fc45..783f34ec9ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -107,7 +107,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { JobType job = JobType.from(controller().system(), zone).orElseThrow(); - deploymentTrigger.reTrigger(applicationId, job); + deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 7a547dbc150..369699eb3a3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -80,7 +80,7 @@ public class JobRunner extends ControllerMaintainer { if ( ! run.hasFailed() && controller().clock().instant().isAfter(run.sleepUntil().orElse(run.start()).plus(jobTimeout))) executors.execute(() -> { - jobs.abort(run.id()); + jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached"); advance(jobs.run(run.id()).get()); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 73d28a1eeae..4c72f6747d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -31,7 +31,8 @@ public class RetriggerMaintainer extends ControllerMaintainer { retriggerEntries.stream() .filter(this::needsTrigger) .filter(entry -> readyToTrigger(entry.jobId())) - .forEach(entry -> controller().applications().deploymentTrigger().reTrigger(entry.jobId().application(), entry.jobId().type())); + .forEach(entry -> controller().applications().deploymentTrigger().reTrigger(entry.jobId().application(), entry.jobId().type(), + "re-triggered by RetriggerMaintainer")); // Remove all jobs that has succeeded with the required job run and persist the list List<RetriggerEntry> remaining = retriggerEntries.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 9a301f50b1a..c3d81b8dcd5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -105,6 +105,7 @@ class RunSerializer { private static final String convergenceSummaryField = "convergenceSummaryV2"; private static final String testerCertificateField = "testerCertificate"; private static final String isDryRunField = "isDryRun"; + private static final String reasonField = "reason"; Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); @@ -150,7 +151,8 @@ class RunSerializer { Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())), - runObject.field(isDryRunField).valid() && runObject.field(isDryRunField).asBool()); + runObject.field(isDryRunField).valid() && runObject.field(isDryRunField).asBool(), + SlimeUtils.optionalString(runObject.field(reasonField))); } private Versions versionsFromSlime(Inspector versionsObject) { @@ -257,6 +259,7 @@ class RunSerializer { versionsObject.setObject(sourceField)); }); runObject.setBool(isDryRunField, run.isDryRun()); + run.reason().ifPresent(reason -> runObject.setString(reasonField, reason)); } private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 72e5a0dfc64..217bb9e1444 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -352,7 +352,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), request, appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return resume(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return disableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -1126,7 +1126,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { DeploymentId deployment = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Principal principal = requireUserPrincipal(request); SupportAccess disallowed = controller.supportAccess().disallow(deployment, principal.getName()); - controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment); + controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment, "re-triggered to disallow support access, by " + request.getJDiscRequest().getUserPrincipal().getName()); return new SlimeJsonResponse(SupportAccessSerializer.serializeCurrentState(disallowed, controller.clock().instant())); } @@ -1159,14 +1159,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Inspector requestObject = toSlime(request.getData()).get(); boolean requireTests = ! requestObject.field("skipTests").asBool(); boolean reTrigger = requestObject.field("reTrigger").asBool(); + boolean upgradeRevision = ! requestObject.field("skipRevision").asBool(); + boolean upgradePlatform = ! requestObject.field("skipUpgrade").asBool(); String triggered = reTrigger ? controller.applications().deploymentTrigger() - .reTrigger(id, type).type().jobName() + .reTrigger(id, type, "re-triggered by " + request.getJDiscRequest().getUserPrincipal().getName()).type().jobName() : controller.applications().deploymentTrigger() - .forceTrigger(id, type, request.getJDiscRequest().getUserPrincipal().getName(), requireTests) + .forceTrigger(id, type, "triggered by " + request.getJDiscRequest().getUserPrincipal().getName(), requireTests, upgradeRevision, upgradePlatform) .stream().map(job -> job.type().jobName()).collect(joining(", ")); + String suppressedUpgrades = ( ! upgradeRevision || ! upgradePlatform ? ", without " : "") + + (upgradeRevision ? "" : "revision") + + ( ! upgradeRevision && ! upgradePlatform ? " and " : "") + + (upgradePlatform ? "" : "platform") + + ( ! upgradeRevision || ! upgradePlatform ? " upgrade" : ""); return new MessageResponse(triggered.isEmpty() ? "Job " + type.jobName() + " for " + id + " not triggered" - : "Triggered " + triggered + " for " + id); + : "Triggered " + triggered + " for " + id + suppressedUpgrades); } private HttpResponse pause(ApplicationId id, JobType type) { @@ -1839,7 +1846,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ private HttpResponse deployPlatform(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { - String versionString = readToString(request.getData()); ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 648d03a54d7..31d9ea29923 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.ChangeBlocker; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.SlimeJsonResponse; @@ -202,12 +203,12 @@ class JobControllerApiHandlerHelper { } /** Aborts any job of the given type. */ - static HttpResponse abortJobResponse(JobController jobs, ApplicationId id, JobType type) { + static HttpResponse abortJobResponse(JobController jobs, HttpRequest request, ApplicationId id, JobType type) { Slime slime = new Slime(); Cursor responseObject = slime.setObject(); Optional<Run> run = jobs.last(id, type).flatMap(last -> jobs.active(last.id())); if (run.isPresent()) { - jobs.abort(run.get().id()); + jobs.abort(run.get().id(), "aborted by " + request.getJDiscRequest().getUserPrincipal()); responseObject.setString("message", "Aborting " + run.get().id()); } else diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 951a22c5ee7..346e61c907c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -124,7 +124,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { SupportAccess supportAccess = controller.supportAccess().registerGrant(deployment, principal.getName(), certificate); // Trigger deployment to include operator cert - Optional<JobId> jobId = controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment); + Optional<JobId> jobId = controller.applications().deploymentTrigger().reTriggerOrAddToQueue(deployment, "re-triggered to grant access, by " + request.getJDiscRequest().getUserPrincipal().getName()); return new MessageResponse( jobId.map(id -> Text.format("Operator %s granted access and job %s triggered", principal.getName(), id.type().jobName())) .orElseGet(() -> Text.format("Operator %s granted access and job trigger queued", principal.getName()))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 08cd88cf23f..21cc69369d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -410,7 +410,7 @@ public class DeploymentContext { public DeploymentContext abortJob(JobType type) { var job = jobId(type); assertNotSame(RunStatus.aborted, currentRun(job).status()); - jobs.abort(currentRun(job).id()); + jobs.abort(currentRun(job).id(), "DeploymentContext.abortJob"); jobAborted(type); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 01e1301c8cf..39c5dd2eae0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -135,7 +135,7 @@ public class DeploymentTester { public void abortAll() { triggerJobs(); for (Run run : jobs.active()) { - jobs.abort(run.id()); + jobs.abort(run.id(), "DeploymentTester.abortAll"); runner.advance(jobs.run(run.id()).get()); assertTrue(jobs.run(run.id()).get().hasEnded()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 805d727d355..b95d34f5414 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -183,6 +183,14 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest); // outdated run is aborted when otherwise blocking a new run tester.triggerJobs(); app.jobAborted(productionUsCentral1); + Versions outdated = tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions(); + + // Flesh bag re-triggers job, and _that_ is not aborted + tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsCentral1, "flesh bag"); + tester.triggerJobs(); + app.runJob(productionUsCentral1); + Versions reTriggered = tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions(); + assertEquals(outdated, reTriggered); app.runJob(productionUsCentral1).runJob(productionUsWest1).runJob(productionUsEast3); assertEquals(Change.empty(), app.instance().change()); @@ -552,9 +560,11 @@ public class DeploymentTriggerTest { // us-east-3 does not automatically trigger when paused, but does when forced. tester.triggerJobs(); app.assertNotRunning(productionUsEast3); - tester.deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "mrTrigger", true); + tester.deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "mrTrigger", true, true, false); app.assertRunning(productionUsEast3); assertFalse(app.instance().jobPause(productionUsEast3).isPresent()); + assertEquals(app.deployment(productionUsEast3.zone(tester.controller().system())).version(), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); } @Test @@ -1823,7 +1833,7 @@ public class DeploymentTriggerTest { app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); app.runJob(productionCdUsEast1) .abortJob(stagingTest) // Complete failing run. .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. @@ -1835,7 +1845,7 @@ public class DeploymentTriggerTest { tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // System and staging tests both require unknown versions, and are broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); app.runJob(productionCdUsEast1) .triggerJobs() .jobAborted(systemTest) @@ -1849,7 +1859,7 @@ public class DeploymentTriggerTest { app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); app.runJob(productionCdUsEast1) .jobAborted(stagingTest) .runJob(stagingTest) @@ -1988,9 +1998,9 @@ public class DeploymentTriggerTest { app.submit(); tester.triggerJobs(); - tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsEast3); - tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3"))); - tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3"))); + tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsEast3, null); + tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3")), null); + tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3")), null); List<RetriggerEntry> retriggerEntries = tester.controller().curator().readRetriggerEntries(); Assert.assertEquals(1, retriggerEntries.size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index d55b5cea4ee..5f443759048 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -72,7 +72,7 @@ public class DeploymentExpirerTest { // Dev application expires when enough time has passed since most recent attempt // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); - tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true); + tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true, Optional.of("upgrade")); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 6b380981e15..71a6fcb1d84 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -95,13 +96,13 @@ public class JobRunnerTest { ApplicationId id = appId.defaultInstance(); jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); try { - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); fail("Job is already running, so this should not be allowed!"); } catch (IllegalArgumentException ignored) { } - jobs.start(id, stagingTest, versions); + start(jobs, id, stagingTest); assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); assertFalse(jobs.last(id, systemTest).get().hasEnded()); @@ -127,7 +128,7 @@ public class JobRunnerTest { jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); RunId first = run.get().id(); Map<Step, Status> steps = run.get().stepStatuses(); @@ -174,7 +175,7 @@ public class JobRunnerTest { assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester, deactivateReal); // Abortion does nothing, as the run has already failed. - jobs.abort(run.get().id()); + jobs.abort(run.get().id(), "abort"); runner.maintain(); assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester, deactivateReal); @@ -188,7 +189,7 @@ public class JobRunnerTest { assertSame(aborted, run.get().status()); // A new run is attempted. - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); assertEquals(first.number() + 1, run.get().id().number()); // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. @@ -206,7 +207,7 @@ public class JobRunnerTest { assertEquals(2, jobs.runs(id, systemTest).size()); // Start a third run, then unregister and wait for data to be deleted. - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); tester.applications().deleteInstance(id); runner.maintain(); assertFalse(jobs.last(id, systemTest).isPresent()); @@ -234,7 +235,7 @@ public class JobRunnerTest { jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); runner.maintain(); barrier.await(); try { @@ -272,14 +273,14 @@ public class JobRunnerTest { assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); runner.run(); } assertEquals(64, jobs.runs(jobId).size()); assertTrue(jobs.details(new RunId(instanceId, systemTest, 1)).isPresent()); - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); runner.run(); assertEquals(64, jobs.runs(jobId).size()); @@ -291,7 +292,7 @@ public class JobRunnerTest { // Make all but the oldest of the 54 jobs a failure. for (int i = 0; i < jobs.historyLength() - 1; i++) { - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); failureRunner.run(); } assertEquals(64, jobs.runs(jobId).size()); @@ -300,7 +301,7 @@ public class JobRunnerTest { assertEquals(66, jobs.firstFailing(jobId).get().id().number()); // Oldest success is kept even though it would normally overflow. - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); failureRunner.run(); assertEquals(65, jobs.runs(jobId).size()); assertEquals(65, jobs.runs(jobId).keySet().iterator().next().number()); @@ -308,7 +309,7 @@ public class JobRunnerTest { assertEquals(66, jobs.firstFailing(jobId).get().id().number()); // First failure after the last success is also kept. - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); failureRunner.run(); assertEquals(66, jobs.runs(jobId).size()); assertEquals(65, jobs.runs(jobId).keySet().iterator().next().number()); @@ -317,7 +318,7 @@ public class JobRunnerTest { assertEquals(66, jobs.firstFailing(jobId).get().id().number()); // No other jobs are kept with repeated failures. - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); failureRunner.run(); assertEquals(66, jobs.runs(jobId).size()); assertEquals(65, jobs.runs(jobId).keySet().iterator().next().number()); @@ -327,7 +328,7 @@ public class JobRunnerTest { assertEquals(66, jobs.firstFailing(jobId).get().id().number()); // history length returns to 256 when a new success is recorded. - jobs.start(instanceId, systemTest, versions); + start(jobs, instanceId, systemTest); runner.run(); assertEquals(64, jobs.runs(jobId).size()); assertEquals(69, jobs.runs(jobId).keySet().iterator().next().number()); @@ -365,7 +366,7 @@ public class JobRunnerTest { ApplicationId id = appId.defaultInstance(); jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); runner.run(); assertSame(aborted, jobs.last(id, systemTest).get().status()); @@ -388,7 +389,7 @@ public class JobRunnerTest { for (RunStatus status : RunStatus.values()) { if (status == success || status == reset) continue; // Status not used for steps. outcomes.put(deployTester, status); - jobs.start(id, systemTest, versions); + start(jobs, id, systemTest); runner.run(); jobs.finish(jobs.last(id, systemTest).get().id()); } @@ -410,6 +411,10 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } + private void start(JobController jobs, ApplicationId id, JobType type) { + jobs.start(id, type, versions, false, Optional.empty()); + } + public static ExecutorService inThreadExecutor() { return new AbstractExecutorService() { final AtomicBoolean shutDown = new AtomicBoolean(false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java index 0c1b3b39b65..ccb2b6ebb74 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java @@ -38,10 +38,10 @@ public class RetriggerMaintainerTest { devApp.completeRollout(); // Trigger a run (to simulate a running job) - tester.deploymentTrigger().reTrigger(applicationId, JobType.devUsEast1); + tester.deploymentTrigger().reTrigger(applicationId, JobType.devUsEast1, null); // Add a job to the queue - tester.deploymentTrigger().reTriggerOrAddToQueue(devApp.deploymentIdIn(ZoneId.from("dev", "us-east-1"))); + tester.deploymentTrigger().reTriggerOrAddToQueue(devApp.deploymentIdIn(ZoneId.from("dev", "us-east-1")), null); // Should be 1 entry in the queue: List<RetriggerEntry> retriggerEntries = tester.controller().curator().readRetriggerEntries(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index f4f52b20325..237d54db20c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -99,6 +99,7 @@ public class RunSerializerTest { assertEquals(applicationVersion.compileVersion(), run.versions().targetApplication().compileVersion()); assertEquals("f00bad", run.versions().targetApplication().commit().get()); assertEquals("https://github.com/user/repo/tree/f00bad", run.versions().targetApplication().sourceUrl().get()); + assertEquals("because", run.reason().get()); assertEquals(new Version(1, 2, 2), run.versions().sourcePlatform().get()); assertEquals(ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", "master", @@ -153,8 +154,9 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); assertEquals(run.isDryRun(), phoenix.isDryRun()); + assertEquals(run.reason(), phoenix.reason()); - Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production); + Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production, Optional.empty()); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 54cde2bacef..85881fbfdbc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -53,6 +53,7 @@ "build": 122, "deployedDirectly": false } - } + }, + "reason": "because" } ]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index b4230cb1fc1..d9192598452 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -366,9 +366,9 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a triggering to force a production job to start without successful tests tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/job/production-us-west-1", POST) - .data("{\"skipTests\":true}") + .data("{ \"skipTests\": true, \"skipRevision\": true, \"skipUpgrade\": true }") .userIdentity(USER_ID), - "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1\"}"); + "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1, without revision and platform upgrade\"}"); app2.runJob(JobType.productionUsWest1); // POST a re-triggering to force a production job to start with previous parameters diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index b830a4ce512..cf6453235d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -49,7 +49,7 @@ public class BadgeApiTest extends ControllerContainerTest { for (int i = 0; i < 31; i++) application.failDeployment(JobType.productionUsWest1); application.triggerJobs(); - tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.testEuWest1); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.testEuWest1, "reason"); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), Files.readString(Paths.get(responseFiles + "overview.svg")), 200); |