diff options
author | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2018-03-26 15:48:05 +0200 |
---|---|---|
committer | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2018-03-26 15:48:05 +0200 |
commit | bec8af2f0dbe2f82d63acafb5c2b13d60d78c75c (patch) | |
tree | 3dffb4ef1eac1b50174067fa32ae0eb5b06a6759 /controller-server | |
parent | b1511cf1c5f4237a67d72151b10b5a0b9649c662 (diff) |
Out of capacity retrying through triggerReadyJobs
Diffstat (limited to 'controller-server')
2 files changed, 10 insertions, 6 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index cf4071b9035..6c954191075 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -113,8 +113,10 @@ public class DeploymentTrigger { for (JobType job : jobs) application = trigger(new Triggering(application, job, false, report.jobType().jobName() + " completed"), jobs, false); } - else if (retryBecauseOutOfCapacity(application, report.jobType())) - application = trigger(new Triggering(application, report.jobType(), true, "Retrying on out of capacity"), Collections.emptySet(), false); + else if (retryBecauseOutOfCapacity(application, report.jobType())) { + triggerReadyJobs(application); + return; // Don't overwrite below. + } else if (retryBecauseNewFailure(application, report.jobType())) { triggerReadyJobs(application); return; // Don't overwrite below. @@ -166,8 +168,10 @@ public class DeploymentTrigger { List<JobType> nextJobs = order.nextAfter(jobType, application); for (JobType nextJobType : nextJobs) { JobStatus nextStatus = application.deploymentJobs().jobStatus().get(nextJobType); - if (changesAvailable(application, jobStatus, nextStatus) || nextStatus.isHanging(jobTimeoutLimit())) - application = trigger(new Triggering(application, nextJobType, false, "Available change in " + jobType.jobName()), nextJobs, false); + if (changesAvailable(application, jobStatus, nextStatus) || nextStatus.isHanging(jobTimeoutLimit())) { + boolean isRetry = nextStatus != null && nextStatus.jobError().filter(JobError.outOfCapacity::equals).isPresent(); + application = trigger(new Triggering(application, nextJobType, isRetry, isRetry ? "Retrying on out of capacity" : "Available change in " + jobType.jobName()), nextJobs, false); + } } applications().store(application); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 594a9c0da44..50bf20c4405 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -470,14 +470,14 @@ public class ControllerTest { assertEquals(2, deploymentQueue.jobs().size()); // app1: 4 hours pass in total, staging-test job for app1 is re-queued by periodic trigger mechanism and added at the - // back of the queue + // front of the queue tester.clock().advance(Duration.ofHours(3)); tester.clock().advance(Duration.ofMinutes(50)); tester.readyJobTrigger().maintain(); + assertEquals(Collections.singletonList(new BuildService.BuildJob(project1, stagingTest.jobName())), deploymentQueue.takeJobsToRun()); assertEquals(Collections.singletonList(new BuildService.BuildJob(project2, stagingTest.jobName())), deploymentQueue.takeJobsToRun()); assertEquals(Collections.singletonList(new BuildService.BuildJob(project3, stagingTest.jobName())), deploymentQueue.takeJobsToRun()); - assertEquals(Collections.singletonList(new BuildService.BuildJob(project1, stagingTest.jobName())), deploymentQueue.takeJobsToRun()); assertEquals(Collections.emptyList(), deploymentQueue.takeJobsToRun()); } |