From 89670c122f53ea97f763aa4011c59e6baa3bc5df Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 19 Jan 2022 12:35:21 +0100 Subject: Remove ZK 3.6.3, which is no longer in use --- zookeeper-server/CMakeLists.txt | 1 - zookeeper-server/pom.xml | 1 - .../zookeeper-server-3.6.3/CMakeLists.txt | 4 - zookeeper-server/zookeeper-server-3.6.3/pom.xml | 125 ---------- .../ReconfigurableVespaZooKeeperServer.java | 43 ---- .../zookeeper/VespaMtlsAuthenticationProvider.java | 41 ---- .../com/yahoo/vespa/zookeeper/VespaQuorumPeer.java | 60 ----- .../vespa/zookeeper/VespaZooKeeperAdminImpl.java | 57 ----- .../vespa/zookeeper/VespaZooKeeperServerImpl.java | 47 ---- .../java/org/apache/zookeeper/common/NetUtils.java | 94 -------- .../server/VespaNettyServerCnxnFactory.java | 37 --- .../yahoo/vespa/zookeper/VespaZooKeeperTest.java | 258 --------------------- 12 files changed, 768 deletions(-) delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/pom.xml delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java delete mode 100644 zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 6582b4602af..a4e4ddaeb2d 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.6.3) add_subdirectory(zookeeper-server-3.7.0) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index e482282388f..3efdde2cd81 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,6 @@ 7-SNAPSHOT zookeeper-server-common - zookeeper-server-3.6.3 zookeeper-server-3.7.0 diff --git a/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt deleted file mode 100644 index b7871cfbde1..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.6.3) -# Needs to be included when this is the wanted default version (and symlinks for other versions need to be removed) -#install_symlink(lib/jars/zookeeper-server-3.6.3-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml deleted file mode 100644 index a8ad183de4e..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - - - 4.0.0 - - com.yahoo.vespa - zookeeper-server - 7-SNAPSHOT - ../pom.xml - - zookeeper-server-3.6.3 - container-plugin - 7-SNAPSHOT - - 3.6.3 - - - - com.yahoo.vespa - zookeeper-server-common - ${project.version} - - - com.yahoo.vespa - zookeeper-client-common - ${project.version} - - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - io.dropwizard.metrics - metrics-core - compile - 3.2.5 - - - org.slf4j - slf4j-api - - - - - org.xerial.snappy - snappy-java - compile - 1.1.7 - - - junit - junit - test - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - -Xlint:all - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${test.hide} - once - - ${zookeeper.version} - - - - - org.apache.maven.plugins - maven-install-plugin - - true - - - - com.yahoo.vespa - bundle-plugin - true - - com.sun.management - zookeeper-server - - - - - diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java deleted file mode 100644 index 246911fdfc7..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Starts or reconfigures zookeeper cluster. - * The QuorumPeer conditionally created here is owned by the Reconfigurer; - * when it already has a peer, that peer is used here in case start or shutdown is required. - * - * @author hmusum - */ -public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private QuorumPeer peer; - - @Inject - public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return true; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java deleted file mode 100644 index 66742b0e05b..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.auth.AuthenticationProvider; -import org.apache.zookeeper.server.auth.X509AuthenticationProvider; - -import java.security.cert.X509Certificate; -import java.util.logging.Logger; - -/** - * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS - * - * @author bjorncs - */ -public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { - - private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); - - public VespaMtlsAuthenticationProvider() throws X509Exception { super(null, null);} - - @Override - public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { - // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. - // The client is authorized once the SSL handshake has completed. - X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); - if (certificateChain == null || certificateChain.length == 0) { - log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); - return KeeperException.Code.AUTHFAILED; - } - X509Certificate certificate = certificateChain[0]; - cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); - return KeeperException.Code.OK; - } - - @Override public String getScheme() { return "x509"; } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java deleted file mode 100644 index 47ec03367c1..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.protect.Process; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps - * exceptions so it can be used by code that does not depend on ZooKeeper. - * - * @author hmusum - */ -class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); - - @Override - public void start(Path path) { - initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); - } - - @Override - public void shutdown(Duration timeout) { - if (quorumPeer != null) { - log.log(Level.FINE, "Shutting down ZooKeeper server"); - try { - quorumPeer.shutdown(); - quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete - if (quorumPeer.isAlive()) - throw new IllegalStateException("Peer still alive after " + timeout); - } catch (RuntimeException | InterruptedException e) { - // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. - // - // When a VespaZooKeeperServer component receives a new config, the container will try to start a new - // server with the new config, this will fail until the old server is deconstructed. If the old server - // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is - // the better option. - Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); - } - } - } - - @Override - protected void initializeAndRun(String[] args) { - try { - super.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { - throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java deleted file mode 100644 index f0a95b70e96..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -@SuppressWarnings("unused") // Created by injection -public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); - - @Override - public void reconfigure(String connectionSpec, String servers) throws ReconfigException { - try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { - long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - log.log(Level.INFO, "Applying ZooKeeper config: " + servers); - byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); - log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - - // Verify by issuing a write operation; this is only accepted once new quorum is obtained. - List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); - zooKeeperAdmin.delete(node, -1); - - log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } - catch ( KeeperException.ReconfigInProgress - | KeeperException.ConnectionLossException - | KeeperException.NewConfigNoQuorum e) { - throw new ReconfigException(e); - } - catch (KeeperException | IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { - return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); - } - -} - diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 8f3a5a91a43..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; - -/** - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - private final ZooKeeperRunner runner; - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this.peer = new VespaQuorumPeer(); - this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); - } - - @Override - public void deconstruct() { - runner.shutdown(); - super.deconstruct(); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return false; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java deleted file mode 100644 index 33ec9b1303a..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.common; - -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; - -/** - * This class contains common utilities for netstuff. Like printing IPv6 literals correctly - */ -public class NetUtils { - - // Note: Changed from original to use hostname from InetSocketAddress if there exists one - public static String formatInetAddr(InetSocketAddress addr) { - String hostName = addr.getHostName(); - if (hostName != null) { - return String.format("%s:%s", hostName, addr.getPort()); - } - - InetAddress ia = addr.getAddress(); - - if (ia == null) { - return String.format("%s:%s", addr.getHostString(), addr.getPort()); - } - if (ia instanceof Inet6Address) { - return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); - } else { - return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); - } - } - - /** - * Separates host and port from given host port string if host port string is enclosed - * within square bracket. - * - * @param hostPort host port string - * @return String[]{host, port} if host port string is host:port - * or String[] {host, port:port} if host port string is host:port:port - * or String[] {host} if host port string is host - * or String[]{} if not a ipv6 host port string. - */ - public static String[] getIPV6HostAndPort(String hostPort) { - if (hostPort.startsWith("[")) { - int i = hostPort.lastIndexOf(']'); - if (i < 0) { - throw new IllegalArgumentException( - hostPort + " starts with '[' but has no matching ']'"); - } - String host = hostPort.substring(1, i); - if (host.isEmpty()) { - throw new IllegalArgumentException(host + " is empty."); - } - if (hostPort.length() > i + 1) { - return getHostPort(hostPort, i, host); - } - return new String[] { host }; - } else { - //Not an IPV6 host port string - return new String[] {}; - } - } - - private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { - // [127::1]:2181 , check separator : exits - if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { - throw new IllegalArgumentException(hostPort + " does not have : after ]"); - } - // [127::1]: scenario - if (indexOfClosingBracket + 2 == hostPort.length()) { - throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); - } - //do not include - String port = hostPort.substring(indexOfClosingBracket + 2); - return new String[] { host, port }; - } -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java deleted file mode 100644 index fdfe0fe8467..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.apache.zookeeper.server; - -import com.yahoo.vespa.zookeeper.Configurator; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.logging.Logger; - -/** - * Overrides secure setting with value from {@link Configurator}. - * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 - * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. - * - * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. - * - * @author bjorncs - */ -public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { - - private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); - - private final boolean isSecure; - - public VespaNettyServerCnxnFactory() { - super(); - this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; - boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); - log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); - } - - @Override - public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { - log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); - super.configure(addr, maxClientCnxns, backlog, isSecure); - } -} diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java deleted file mode 100644 index 922c389f94a..00000000000 --- a/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.net.HostName; -import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; -import com.yahoo.vespa.zookeeper.Reconfigurer; -import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.ServerSocket; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.IntStream; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; - -public class VespaZooKeeperTest { - - static final Path tempDirRoot = getTmpDir(); - static final List ports = new ArrayList<>(); - - /** - * Performs dynamic reconfiguration of ZooKeeper servers. - * - * First, a cluster of 3 servers is set up, and some data is written to it. - * Then, 3 new servers are added, and the first 3 marked for retirement; - * this should force the quorum to move the 3 new servers, but not disconnect the old ones. - * Next, the old servers are removed. - * Then, the cluster is reduced to size 1. - * Finally, the cluster grows to size 3 again. - * - * Throughout all of this, quorum should remain, and the data should remain the same. - */ - @Test(timeout = 120_000) - @Ignore // Unstable, some ZK server keeps resetting connections sometimes. - public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { - List keepers = new ArrayList<>(); - for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); - for (int i = 0; i < 8; i++) keepers.get(i).run(); - - // Start the first three servers. - List configs = getConfigs(0, 0, 3, 0); - for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for all servers to be up and running. - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Write data to verify later. - String path = writeData(configs.get(0)); - - // Let three new servers join, causing the three older ones to retire and leave the ensemble. - configs = getConfigs(0, 3, 3, 3); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // The existing servers can't reconfigure and leave before the joiners are up. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for new quorum to be established. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - // Old servers are removed. - configs = getConfigs(3, 0, 3, 0); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // Old servers shut down, while the newer servers remain. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Ensure old servers shut down properly. - for (int i = 0; i < 3; i++) keepers.get(i).await(); - // Ensure new servers have reconfigured. - for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - - // Cluster shrinks to a single server. - configs = getConfigs(5, 0, 1, 0); - for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // We let the remaining server reconfigure the others out before they die. - for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 3; i < 5; i++) keepers.get(i).await(); - verifyData(path, configs.get(5)); - - // Cluster grows to 3 servers again. - configs = getConfigs(5, 0, 3, 2); - for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Wait for the joiners. - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - verifyData(path, configs.get(7)); - - // Let the remaining servers terminate. - for (int i = 5; i < 8; i++) keepers.get(i).config = null; - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 8; i++) keepers.get(i).await(); - } - - static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - ZooKeeperAdmin admin = createAdmin(config); - List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); - String read = new String(admin.getData(node, false, new Stat()), UTF_8); - assertEquals("hi", read); - return node; - } - - static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - for (int i = 0; i < 10; i++) { - try { - assertEquals("hi", new String(createAdmin(config).getData(path, false, new Stat()), UTF_8)); - return; - } - catch (KeeperException.ConnectionLossException e) { - e.printStackTrace(); - Thread.sleep(10 << i); - } - } - } - - static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { - return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), - 10_000, - System.err::println, - new ZkClientConfigBuilder().toConfig()); - } - - static class ZooKeeper { - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - final Phaser phaser = new Phaser(2); - final AtomicReference> future = new AtomicReference<>(); - ZookeeperServerConfig config; - - void run() { - future.set(executor.submit(() -> { - Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); - phaser.arriveAndAwaitAdvance(); - while (config != null) { - new ReconfigurableVespaZooKeeperServer(reconfigurer, config); - phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here - phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff - } - reconfigurer.deconstruct(); - })); - } - - void await() throws ExecutionException, InterruptedException, TimeoutException { - future.get().get(30, SECONDS); - } - } - - static List getConfigs(int removed, int retired, int active, int joining) { - return IntStream.rangeClosed(1, removed + retired + active) - .mapToObj(id -> getConfig(removed, retired, active, joining, id)) - .collect(toList()); - } - - // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. - static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { - if (id <= removed) - return null; - - Path tempDir = tempDirRoot.resolve("zookeeper-" + id); - return new ZookeeperServerConfig.Builder() - .clientPort(getPorts(id).get(0)) - .dataDir(tempDir.toString()) - .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) - .myid(id) - .myidFile(tempDir.resolve("myid").toString()) - .dynamicReconfiguration(true) - .server(IntStream.rangeClosed(removed + 1, removed + retired + active) - .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() - .id(i) - .clientPort(getPorts(i).get(0)) - .electionPort(getPorts(i).get(1)) - .quorumPort(getPorts(i).get(2)) - .hostname("localhost") - .joining(i - removed > retired + active - joining) - .retired(i - removed <= retired)) - .collect(toList())) - .build(); - } - - static List getPorts(int id) { - if (ports.size() < id * 3) { - int previousPort; - if (ports.isEmpty()) { - String[] version = System.getProperty("zk-version").split("\\."); - int versionPortOffset = 0; - for (String part : version) - versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); - previousPort = 20000 + versionPortOffset % 30000; - } - else - previousPort = ports.get(ports.size() - 1); - - for (int i = 0; i < 3; i++) - ports.add(previousPort = nextPort(previousPort)); - } - return ports.subList(id * 3 - 3, id * 3); - } - - static int nextPort(int previousPort) { - for (int j = 1; j <= 30000; j++) { - int port = (previousPort + j); - while (port > 50000) - port -= 30000; - - try (ServerSocket socket = new ServerSocket(port)) { - return socket.getLocalPort(); - } - catch (IOException e) { - System.err.println("Could not bind port " + port + ": " + e); - } - } - throw new RuntimeException("No free ports"); - } - - static Path getTmpDir() { - try { - Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); - tempDir.toFile().deleteOnExit(); - return tempDir.toAbsolutePath(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} -- cgit v1.2.3