Revert "Revert "Hmusum/upgrade to zookeeper 3.5""

5 files changed, 54 insertions, 18 deletions

diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml
index 3b1cb16aa96..9ac7c93134c 100644
--- a/clustercontroller-core/pom.xml
+++ b/clustercontroller-core/pom.xml
@@ -66,6 +66,13 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
+            <!-- Not used by this module, but compilation fails without it because zookeeper uses these annotations.
+                 Provided scoped here to avoid dependents getting it transitively. -->
+            <groupId>com.github.spotbugs</groupId>
+            <artifactId>spotbugs-annotations</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
             <scope>provided</scope>
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java
index 880f2cdaf19..adba17040f0 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java
@@ -1,25 +1,33 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core.database;
 
+import com.yahoo.log.LogLevel;
+import com.yahoo.vdslib.state.Node;
+import com.yahoo.vdslib.state.NodeState;
+import com.yahoo.vdslib.state.State;
 import com.yahoo.vespa.clustercontroller.core.AnnotatedClusterState;
 import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle;
 import com.yahoo.vespa.clustercontroller.core.ContentCluster;
 import com.yahoo.vespa.clustercontroller.core.rpc.EnvelopedClusterStateBundleCodec;
 import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec;
-import org.apache.zookeeper.*;
-import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vdslib.state.NodeState;
-import com.yahoo.vdslib.state.State;
-import com.yahoo.vdslib.state.Node;
+import org.apache.zookeeper.data.Stat;
 
-import java.util.logging.Logger;
-import java.util.*;
 import java.io.IOException;
-import java.io.StringWriter;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.logging.Logger;
 
 public class ZooKeeperDatabase extends Database {
 
diff --git a/parent/pom.xml b/parent/pom.xml
index 7efe36e73f7..ade0bc8725e 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -505,14 +505,17 @@
                 <artifactId>commons-pool</artifactId>
                 <version>1.5.6</version>
             </dependency>
-            <!-- Explicitly included to get Zookeeper version 3.4.14,
+
+            <!-- BEGIN zookeeper -->
+
+            <!-- Explicitly force Zookeeper version,
                  can be excluded if you want the Zookeeper version
                  used by curator by default
             -->
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>
                 <artifactId>zookeeper</artifactId>
-                <version>3.4.14</version>
+                <version>${zookeeper.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
@@ -525,6 +528,17 @@
                 <version>${curator.version}</version>
             </dependency>
             <dependency>
+                <!-- Needed by zookeeper, which has an optional dependency.
+                     Version must be the same as the one zk uses - check the zk pom file.  -->
+                <groupId>com.github.spotbugs</groupId>
+                <artifactId>spotbugs-annotations</artifactId>
+                <!-- No version property, as we don't want maven-dependency-plugin to alert about newer versions. -->
+                <version>3.1.9</version>
+            </dependency>
+
+            <!-- END zookeeper -->
+
+            <dependency>
                 <groupId>org.junit.jupiter</groupId>
                 <artifactId>junit-jupiter-engine</artifactId>
                 <version>${junit.version}</version>
@@ -789,6 +803,7 @@
         <junit.version>5.4.2</junit.version>
         <prometheus.client.version>0.6.0</prometheus.client.version>
         <protobuf.version>3.7.0</protobuf.version>
+        <zookeeper.version>3.5.5</zookeeper.version>
     </properties>
 
 </project>
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
index 5b52f757dad..6ad927ef44d 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
@@ -66,10 +66,12 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable {
         sb.append("clientPort=").append(config.clientPort()).append("\n");
         sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
         sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
-        // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands
-        // Includes all available commands in 3.4, except 'wchc' and 'wchp'
-        // Mandatory when using ZooKeeper 3.5
-        sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
+        // See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands
+        // Includes all available commands in 3.5, except 'wchc' and 'wchp'
+        sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
+        sb.append("admin.enableServer=false").append("\n");
+        // Need NettyServerCnxnFactory to be able to use TLS for communication
+        sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n");
         ensureThisServerIsRepresented(config.myid(), config.server());
         config.server().forEach(server -> addServerToCfg(sb, server));
         return sb.toString();
@@ -113,7 +115,7 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable {
     public void run() {
         System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
         String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())};
-        log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config: " + args[0]);
+        log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]);
         log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
         org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args);
     }
diff --git a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java b/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java
index 3f33892fd45..920b1c03752 100644
--- a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java
+++ b/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java
@@ -110,7 +110,9 @@ public class ZooKeeperServerTest {
             "clientPort=2181\n" +
             "autopurge.purgeInterval=1\n" +
             "autopurge.snapRetainCount=15\n" +
-            "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
+            "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
+            "admin.enableServer=false\n" +
+            "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" +
             "server.0=foo:321:123\n";
         validateConfigFile(cfgFile, expected);
     }
@@ -126,7 +128,9 @@ public class ZooKeeperServerTest {
                         "clientPort=2181\n" +
                         "autopurge.purgeInterval=1\n" +
                         "autopurge.snapRetainCount=15\n" +
-                        "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
+                        "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
+                        "admin.enableServer=false\n" +
+                        "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" +
                         "server.0=foo:321:123\n" +
                         "server.1=bar:432:234\n" +
                         "server.2=baz:543:345\n";