diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2022-01-19 22:59:12 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-19 22:59:12 +0100 |
commit | 15040f5d9099587e2a766032e73ff100c934ba8f (patch) | |
tree | 8f185a532ee61fa4f3506abb50c23b77802c9419 /controller-server/src/main/java/com/yahoo | |
parent | b76f5147a898570ff5944b29b18d9bc18a831bd4 (diff) | |
parent | 0db0da43c891e1869247f6c2738bbbd4fcee2835 (diff) |
Merge pull request #20878 from vespa-engine/jonmv/deployment-orchestration-for-long-pipelinesv7.531.17
Let deployment run for some time before giving up nodes which are down
Diffstat (limited to 'controller-server/src/main/java/com/yahoo')
2 files changed, 7 insertions, 9 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 28f48bd1cab..7b1cc1fd84d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -350,14 +350,12 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList - .isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) - .and(nodeList - .not().isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) - ); - if ( ! suspendedTooLong.isEmpty()) { + NodeList suspendedTooLong = nodeList.isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) + .and(nodeList.not().isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) + ); + if ( ! suspendedTooLong.isEmpty() && deployment.get().at().plus(timeouts.statelessNodesDown()).isBefore(controller.clock().instant())) { failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 2b9e3dd0733..4b38306f905 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -210,7 +210,7 @@ public class Run { return lastVespaLogTimestamp; } - /** Returns the timestamp of the last time no nodes were allowed to be down. */ + /** Returns since when no nodes have been allowed to be down. */ public Optional<Instant> noNodesDownSince() { return noNodesDownSince; } |