Revert "Revert "Create dummy node after starting ZooKeeper server""

4 files changed, 50 insertions, 16 deletions

diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
index 0a25d432766..eb84b13d4d6 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
@@ -1,13 +1,11 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.zookeeper;
 
-import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.component.AbstractComponent;
-import com.yahoo.net.HostName;
+import com.yahoo.component.annotation.Inject;
 import com.yahoo.protect.Process;
 import com.yahoo.yolean.Exceptions;
-
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
@@ -92,7 +90,7 @@ public class Reconfigurer extends AbstractComponent {
         log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." +
                             "\nServers in active config:" + servers(activeConfig) +
                             "\nServers in new config:" + servers(newConfig));
-        String connectionSpec = localConnectionSpec(activeConfig);
+        String connectionSpec = vespaZooKeeperAdmin.localConnectionSpec(activeConfig);
         Instant now = Instant.now();
         Duration reconfigTimeout = reconfigTimeout();
         Instant end = now.plus(reconfigTimeout);
@@ -136,10 +134,6 @@ public class Reconfigurer extends AbstractComponent {
         return TIMEOUT;
     }
 
-    private static String localConnectionSpec(ZookeeperServerConfig config) {
-        return HostName.getLocalhost() + ":" + config.clientPort();
-    }
-
     private static List<String> servers(ZookeeperServerConfig config) {
         return config.server().stream()
                      .filter(server -> ! server.retired())
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java
index 59c9628bcab..a9096125cba 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.zookeeper;
 
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.net.HostName;
 import java.time.Duration;
 
 /**
@@ -15,4 +17,6 @@ public interface VespaZooKeeperAdmin {
     /* Timeout for connecting to ZooKeeper */
     default Duration sessionTimeout() { return Duration.ofSeconds(30); }
 
+    default String localConnectionSpec(ZookeeperServerConfig config) { return HostName.getLocalhost() + ":" + config.clientPort(); }
+
 }
diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java
index ae7bf8d84f5..c6d1a4f310f 100644
--- a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java
+++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java
@@ -1,19 +1,23 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.zookeeper;
 
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.net.HostName;
 import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.admin.ZooKeeperAdmin;
 import org.apache.zookeeper.data.ACL;
-
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import static com.yahoo.yolean.Exceptions.uncheck;
+
 /**
  * @author hmusum
  */
@@ -43,14 +47,46 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin {
                 | KeeperException.NewConfigNoQuorum  e) {
             throw new ReconfigException(e);
         }
-        catch (KeeperException | IOException | InterruptedException e) {
+        catch (KeeperException | InterruptedException e) {
             throw new RuntimeException(e);
         }
     }
 
-    private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException {
-        return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(),
-                                  (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig());
+    private ZooKeeperAdmin createAdmin(String connectionSpec) {
+        return uncheck(() -> new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(),
+                                                (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()));
+    }
+
+    /** Creates a node in zookeeper, with hostname as part of node name, this ensures that server is up and working before returning */
+    void createDummyNode(ZookeeperServerConfig zookeeperServerConfig) {
+        int sleepTime = 2_000;
+        try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(localConnectionSpec(zookeeperServerConfig))) {
+            Instant end = Instant.now().plus(Duration.ofMinutes(5));
+            Exception exception = null;
+            do {
+                try {
+                    zooKeeperAdmin.create("/dummy-node-" + HostName.getLocalhost(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                    return;
+                } catch (KeeperException e) {
+                    if (e instanceof KeeperException.NodeExistsException) {
+                        try {
+                            zooKeeperAdmin.setData("/dummy-node-" + HostName.getLocalhost(), new byte[0], 0);
+                            return;
+                        } catch (KeeperException ex) {
+                            log.log(Level.INFO, e.getMessage());
+                            Thread.sleep(sleepTime);
+                            continue;
+                        }
+                    }
+                    log.log(Level.INFO, e.getMessage());
+                    exception = e;
+                    Thread.sleep(sleepTime);
+                }
+            } while (Instant.now().isBefore(end));
+            throw new RuntimeException("Unable to create dummy node: ", exception);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
     }
 
 }
diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index e5bfa60f47e..8adabeedb1b 100644
--- a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -2,10 +2,9 @@
 package com.yahoo.vespa.zookeeper;
 
 import ai.vespa.validation.Validation;
-import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.component.AbstractComponent;
-
+import com.yahoo.component.annotation.Inject;
 import java.nio.file.Path;
 import java.time.Duration;
 
@@ -25,6 +24,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Vespa
                            "dynamicReconfiguration must be false");
         this.peer = new VespaQuorumPeer();
         this.runner = new ZooKeeperRunner(zookeeperServerConfig, this);
+        new VespaZooKeeperAdminImpl().createDummyNode(zookeeperServerConfig);
     }
 
     @Override