Merge pull request #15735 from vespa-engine/hmusum/add-support-for-zookeeper-server-3.6.2

Add support for ZooKeeper server 3.6.2

10 files changed, 228 insertions, 3 deletions

diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt
index 2dd7a56ce82..2fa0caeb5fb 100644
--- a/zookeeper-server/CMakeLists.txt
+++ b/zookeeper-server/CMakeLists.txt
@@ -1,3 +1,4 @@
-# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 add_subdirectory(zookeeper-server-common)
 add_subdirectory(zookeeper-server-3.5.6)
+add_subdirectory(zookeeper-server-3.6.2)
diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml
index 60065e9fff3..9a3425230a5 100644
--- a/zookeeper-server/pom.xml
+++ b/zookeeper-server/pom.xml
@@ -14,6 +14,7 @@
   <modules>
     <module>zookeeper-server-common</module>
     <module>zookeeper-server-3.5.6</module>
+    <module>zookeeper-server-3.6.2</module>
   </modules>
   <dependencies>
     <dependency>
diff --git a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt
index b68994d32e0..0de588e0ad3 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt
+++ b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt
@@ -1,3 +1,4 @@
-# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 install_fat_java_artifact(zookeeper-server-3.5.6)
+# Included when this is the wanted default version (and symlinks for other versions need to be removed)
 install_symlink(lib/jars/zookeeper-server-3.5.6-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar)
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index 4a9a5bb4f65..c168e475ad6 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -11,7 +11,7 @@ import java.nio.file.Path;
  * Main component controlling startup and stop of zookeeper server
  *
  * @author Ulf Lilleengen
- * @author Harald Musum
+ * @author hmusum
  */
 public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer {
 
diff --git a/zookeeper-server/zookeeper-server-3.6.2/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.6.2/CMakeLists.txt
new file mode 100644
index 00000000000..e540353f113
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/CMakeLists.txt
@@ -0,0 +1,4 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_fat_java_artifact(zookeeper-server-3.6.2)
+# TODO: Needs to be included when this is the wanted default version (and symlinks for other versions need to be removed)
+# install_symlink(lib/jars/zookeeper-server-3.6.2-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar)
diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml
new file mode 100644
index 00000000000..c479c84ac86
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>zookeeper-server</artifactId>
+    <version>7-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>zookeeper-server-3.6.2</artifactId>
+  <packaging>container-plugin</packaging>
+  <version>7-SNAPSHOT</version>
+  <dependencies>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>zookeeper-server-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>3.6.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-jdk14</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.5</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <compilerArgs>
+            <arg>-Xlint:all</arg>
+            <arg>-Werror</arg>
+          </compilerArgs>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile>
+          <forkMode>once</forkMode>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-install-plugin</artifactId>
+        <configuration>
+          <updateReleaseInfo>true</updateReleaseInfo>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
+        <extensions>true</extensions>
+        <configuration>
+          <importPackage>com.sun.management</importPackage>
+          <bundleSymbolicName>zookeeper-server</bundleSymbolicName>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
new file mode 100644
index 00000000000..027bb212f09
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java
@@ -0,0 +1,26 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.zookeeper;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.AbstractComponent;
+
+import java.nio.file.Path;
+
+/**
+ * Starts or reconfigures zookeeper cluster
+ *
+ * @author hmusum
+ */
+public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer {
+
+    @Inject
+    public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) {
+        reconfigurer.startOrReconfigure(zookeeperServerConfig, this);
+    }
+
+    public void start(Path configFilePath) {
+        new ZooKeeperServer().start(configFilePath);
+    }
+
+}
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java
new file mode 100644
index 00000000000..57ae62c0ebc
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java
@@ -0,0 +1,47 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.zookeeper;
+
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.admin.ZooKeeperAdmin;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author hmusum
+ */
+@SuppressWarnings("unused") // Created by injection
+public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin {
+
+    private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName());
+
+    @Override
+    public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException {
+        try {
+            ZooKeeperAdmin zooKeeperAdmin = new ZooKeeperAdmin(connectionSpec,
+                                                               (int) sessionTimeout().toMillis(),
+                                                               (event) -> log.log(Level.INFO, event.toString()));
+            long fromConfig = -1;
+            // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0)
+            byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null);
+            log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8));
+        } catch (KeeperException e) {
+            if (retryOn(e))
+                throw new ReconfigException(e);
+            else
+                throw new RuntimeException(e);
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static boolean retryOn(KeeperException e) {
+        return e instanceof KeeperException.ReconfigInProgress ||
+               e instanceof KeeperException.ConnectionLossException ||
+               e instanceof KeeperException.NewConfigNoQuorum;
+    }
+
+}
+
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
new file mode 100644
index 00000000000..4a9a5bb4f65
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -0,0 +1,35 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.zookeeper;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.AbstractComponent;
+
+import java.nio.file.Path;
+
+/**
+ * Main component controlling startup and stop of zookeeper server
+ *
+ * @author Ulf Lilleengen
+ * @author Harald Musum
+ */
+public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer {
+
+    private final ZooKeeperRunner zooKeeperRunner;
+
+    @Inject
+    public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) {
+        this.zooKeeperRunner = new ZooKeeperRunner(zookeeperServerConfig, this);
+    }
+
+    @Override
+    public void deconstruct() {
+        zooKeeperRunner.shutdown();
+        super.deconstruct();
+    }
+
+    public void start(Path configFilePath) {
+        new ZooKeeperServer().start(configFilePath);
+    }
+
+}
diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
new file mode 100644
index 00000000000..ceb036049da
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
@@ -0,0 +1,32 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.zookeeper;
+
+import org.apache.zookeeper.server.admin.AdminServer;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+/**
+ * Class to start zookeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps
+ * exceptions so it can be used by code that does not depend on zookeeper.
+ *
+ * @author hmusum
+ */
+class ZooKeeperServer extends QuorumPeerMain {
+
+    public void start(Path path) {
+        initializeAndRun(new String[]{ path.toFile().getAbsolutePath()});
+    }
+
+    @Override
+    protected void initializeAndRun(String[] args) {
+        try {
+            super.initializeAndRun(args);
+        } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) {
+            throw new RuntimeException("Exception when initializing or running ZooKeeper server", e);
+        }
+    }
+
+}