From 0db0da43c891e1869247f6c2738bbbd4fcee2835 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 19 Jan 2022 22:44:36 +0100 Subject: Let deployment run for some time before giving up nodes which are down --- .../java/com/yahoo/vespa/documentmodel/DocumentModel.java | 10 +++++----- .../hosted/controller/deployment/InternalStepRunner.java | 14 ++++++-------- .../com/yahoo/vespa/hosted/controller/deployment/Run.java | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index bea7c95d412..112385e1801 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -9,12 +9,12 @@ import com.yahoo.documentmodel.DocumentTypeRepo; * It contains a search manager managing all specified search definitions. * It contains a storage manager managing all specified storage definitions. * - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class DocumentModel { - private DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private SearchManager searchMan = new SearchManager(); + + private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); + private final SearchManager searchMan = new SearchManager(); /** * @@ -26,6 +26,6 @@ public class DocumentModel { * * @return Returns the SearchManager */ - public SearchManager getSearchManager() { return searchMan; } + public SearchManager getSearchManager() { return searchMan; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 225634634b2..a6d1800bf71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -349,14 +349,12 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList - .isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) - .and(nodeList - .not().isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) - ); - if ( ! suspendedTooLong.isEmpty()) { + NodeList suspendedTooLong = nodeList.isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) + .and(nodeList.not().isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) + ); + if ( ! suspendedTooLong.isEmpty() && deployment.get().at().plus(timeouts.statelessNodesDown()).isBefore(controller.clock().instant())) { failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 2b9e3dd0733..4b38306f905 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -210,7 +210,7 @@ public class Run { return lastVespaLogTimestamp; } - /** Returns the timestamp of the last time no nodes were allowed to be down. */ + /** Returns since when no nodes have been allowed to be down. */ public Optional noNodesDownSince() { return noNodesDownSince; } -- cgit v1.2.3