Always force shutdown of non-reconfigurable ZooKeeper server

This basically reverts to the original behaviour, where the container is forced
to exit when a new VespaZooKeeperServer component is setup.

8 files changed, 52 insertions, 6 deletions

diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
index f8ca8987a70..d614aecbad2 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
@@ -31,4 +31,9 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem
         peer.start(configFilePath);
     }
 
+    @Override
+    public boolean reconfigurable() {
+        return true;
+    }
+
 }
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index a7f30ab95d8..4dfcbeea444 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -38,4 +38,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Vespa
         peer.start(configFilePath);
     }
 
+    @Override
+    public boolean reconfigurable() {
+        return false;
+    }
+
 }
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
index f8ca8987a70..d614aecbad2 100644
--- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
@@ -31,4 +31,9 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem
         peer.start(configFilePath);
     }
 
+    @Override
+    public boolean reconfigurable() {
+        return true;
+    }
+
 }
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index e93bae1e438..2a66bebe048 100644
--- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -38,4 +38,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Vespa
         peer.start(configFilePath);
     }
 
+    @Override
+    public boolean reconfigurable() {
+        return false;
+    }
+
 }
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java
index ba53ea04b6e..c0b5b7c035f 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java
@@ -21,4 +21,9 @@ public class DummyVespaZooKeeperServer extends AbstractComponent implements Vesp
     @Override
     public void start(Path configFilePath) {}
 
+    @Override
+    public boolean reconfigurable() {
+        return false;
+    }
+
 }
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java
index ea8c6d2bbee..bb64fcc16ce 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServer.java
@@ -6,12 +6,17 @@ import java.nio.file.Path;
 /**
  * Interface for a component that starts/stops a ZooKeeper server.
  *
- * @author Harald Musum
+ * @author hmusum
  */
 public interface VespaZooKeeperServer {
 
+    /** Shut down the server. Blocks until shutdown has completed */
     void shutdown();
 
+    /** Start the server with the given config file */
     void start(Path configFilePath);
 
+    /** Whether this server support dynamic reconfiguration */
+    boolean reconfigurable();
+
 }
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java
index bf2dd588c93..812c974f9f8 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.zookeeper;
 
 import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.protect.Process;
 import com.yahoo.security.tls.TransportSecurityUtils;
 
 import java.nio.file.Path;
@@ -72,16 +73,26 @@ public class ZooKeeperRunner implements Runnable {
                 startServer(path); // Will block in a real implementation of VespaZooKeeperServer
                 return;
             } catch (RuntimeException e) {
-                Duration delay = backoff.delay(attempt);
-                log.log(Level.WARNING, "Starting ZooKeeper server failed on attempt " + attempt +
-                                       ". Retrying in " + delay + ", time left " + Duration.between(now, end), e);
-                sleeper.sleep(delay);
+                String messagePart = "Starting " + serverDescription() + " failed on attempt " +
+                                     attempt;
+                if (server.reconfigurable()) {
+                    Duration delay = backoff.delay(attempt);
+                    log.log(Level.WARNING, messagePart + ". Retrying in " + delay + ", time left " +
+                                           Duration.between(now, end), e);
+                    sleeper.sleep(delay);
+                } else {
+                    Process.logAndDie(messagePart + ". Forcing shutdown", e);
+                }
             } finally {
                 now = Instant.now();
             }
         }
     }
 
+    private String serverDescription() {
+        return (server.reconfigurable() ? "" : "non-") + "reconfigurable ZooKeeper server";
+    }
+
     private void startServer(Path path) {
         // Note: Hack to make this work in ZooKeeper 3.6, where metrics provider class is
         // loaded by using Thread.currentThread().getContextClassLoader() which does not work
@@ -91,7 +102,7 @@ public class ZooKeeperRunner implements Runnable {
         try {
             server.start(path);
         } catch (Throwable e) {
-            throw new RuntimeException("Starting ZooKeeper server failed", e);
+            throw new RuntimeException("Starting " + serverDescription() + " failed", e);
         } finally {
             Thread.currentThread().setContextClassLoader(tccl);
         }
diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
index 61ddc5996a4..a1b98a23bd0 100644
--- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
+++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
@@ -165,6 +165,11 @@ public class ReconfigurerTest {
         @Override
         public void start(Path configFilePath) { }
 
+        @Override
+        public boolean reconfigurable() {
+            return true;
+        }
+
     }
 
     private static class TestableVespaZooKeeperAdmin implements VespaZooKeeperAdmin {