diff options
author | jonmv <venstad@gmail.com> | 2022-12-02 12:46:28 +0100 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2022-12-02 12:46:28 +0100 |
commit | 01d67f9d2ceed1c138842cc5df8b9f188ae2b7d0 (patch) | |
tree | ed7c133fcc3a7b07019e7fef19675740ab10a46b /zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java | |
parent | 6e26f23c8b7ad77ab12d50ac41472bc3eb5b706d (diff) |
Improve timeout and retry handling when failing to reconfigure ZK
Diffstat (limited to 'zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java')
-rw-r--r-- | zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java | 22 |
1 files changed, 12 insertions, 10 deletions
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index 0f090f83c0b..3e3ae08ff64 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -29,7 +29,7 @@ public class Reconfigurer extends AbstractComponent { private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName()); - private static final Duration TIMEOUT = Duration.ofMinutes(3); + private static final Duration TIMEOUT = Duration.ofMinutes(15); private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)); private final VespaZooKeeperAdmin vespaZooKeeperAdmin; @@ -97,7 +97,7 @@ public class Reconfigurer extends AbstractComponent { Duration reconfigTimeout = reconfigTimeout(); Instant end = now.plus(reconfigTimeout); // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed - for (int attempt = 1; now.isBefore(end); attempt++) { + for (int attempt = 1; ; attempt++) { try { Instant reconfigStarted = Instant.now(); vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers); @@ -110,17 +110,19 @@ public class Reconfigurer extends AbstractComponent { return; } catch (ReconfigException e) { Duration delay = backoff.delay(attempt); - log.log(Level.WARNING, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay + - ", time left " + Duration.between(now, end) + ": " + - Exceptions.toMessageString(e)); - sleeper.sleep(delay); - } finally { now = Instant.now(); + if (now.isBefore(end)) { + log.log(Level.WARNING, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay + + ", time left " + Duration.between(now, end) + ": " + Exceptions.toMessageString(e)); + sleeper.sleep(delay); + } + else { + log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " + + reconfigTimeout + "; giving up now: " + Exceptions.toMessageString(e)); + shutdownAndDie(reconfigTimeout); + } } } - - // Reconfiguration failed - shutdownAndDie(reconfigTimeout); } private void shutdownAndDie(Duration reconfigTimeout) { |