diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2019-11-18 12:30:53 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2019-11-18 12:30:53 +0100 |
commit | bc648228c6628c53f68f0e7d312cc9938f7659ad (patch) | |
tree | e383346744dda302c7fc99ab45147a7d2a11d005 /controller-server | |
parent | fc1f3704120dc6fc67d1c4ff27a3405df2b2958a (diff) |
Retry deleting deployments when jobs end
Diffstat (limited to 'controller-server')
2 files changed, 27 insertions, 10 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index b9c6fd8c555..ee9704d3842 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -35,8 +35,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.yolean.Exceptions; @@ -534,9 +532,11 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return Optional.of(running); + return retrying(10, () -> { + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + return running; + }); } catch (RuntimeException e) { logger.log(WARNING, "Failed deleting application " + id.application(), e); @@ -546,9 +546,11 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return Optional.of(running); + return retrying(10, () -> { + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().deactivateTester(id.tester(), id.type()); + return running; + }); } catch (RuntimeException e) { logger.log(WARNING, "Failed deleting tester of " + id.application(), e); @@ -556,6 +558,22 @@ public class InternalStepRunner implements StepRunner { } } + private static Optional<RunStatus> retrying(int retries, Supplier<RunStatus> task) { + RuntimeException exception = null; + do { + try { + return Optional.of(task.get()); + } + catch (RuntimeException e) { + if (exception == null) + exception = e; + else + exception.addSuppressed(e); + } + } while (--retries >= 0); + throw exception; + } + private Optional<RunStatus> report(RunId id, DualLogger logger) { try { controller.jobController().active(id).ifPresent(run -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index ffe90b8c44d..6d215bd7fa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -95,8 +95,7 @@ public class JobRunner extends Maintainer { : DeploymentJobs.JobError.unknown)); controller().applications().deploymentTrigger().notifyOfCompletion(report); }); - - } + } catch (Exception e) { log.log(LogLevel.WARNING, "Exception finishing " + id, e); } |