Improve timeout and retry handling when failing to reconfigure ZK

1 files changed, 12 insertions, 10 deletions

diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
index 0f090f83c0b..3e3ae08ff64 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
@@ -29,7 +29,7 @@ public class Reconfigurer extends AbstractComponent {
 
     private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName());
 
-    private static final Duration TIMEOUT = Duration.ofMinutes(3);
+    private static final Duration TIMEOUT = Duration.ofMinutes(15);
 
     private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10));
     private final VespaZooKeeperAdmin vespaZooKeeperAdmin;
@@ -97,7 +97,7 @@ public class Reconfigurer extends AbstractComponent {
         Duration reconfigTimeout = reconfigTimeout();
         Instant end = now.plus(reconfigTimeout);
         // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed
-        for (int attempt = 1; now.isBefore(end); attempt++) {
+        for (int attempt = 1; ; attempt++) {
             try {
                 Instant reconfigStarted = Instant.now();
                 vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers);
@@ -110,17 +110,19 @@ public class Reconfigurer extends AbstractComponent {
                 return;
             } catch (ReconfigException e) {
                 Duration delay = backoff.delay(attempt);
-                log.log(Level.WARNING, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay +
-                                       ", time left " + Duration.between(now, end) + ": " +
-                                       Exceptions.toMessageString(e));
-                sleeper.sleep(delay);
-            } finally {
                 now = Instant.now();
+                if (now.isBefore(end)) {
+                    log.log(Level.WARNING, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay +
+                                           ", time left " + Duration.between(now, end) + ": " + Exceptions.toMessageString(e));
+                    sleeper.sleep(delay);
+                }
+                else {
+                    log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " +
+                                          reconfigTimeout + "; giving up now: " + Exceptions.toMessageString(e));
+                    shutdownAndDie(reconfigTimeout);
+                }
             }
         }
-
-        // Reconfiguration failed
-        shutdownAndDie(reconfigTimeout);
     }
 
     private void shutdownAndDie(Duration reconfigTimeout) {