diff options
author | Harald Musum <musum@yahooinc.com> | 2022-05-11 09:31:37 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2022-05-11 09:31:37 +0200 |
commit | 162abbd04eb3062451415bab1117be11b19c5bf6 (patch) | |
tree | c975707e9c5eb4bc48a60947d716a4d2fd08f84b /zookeeper-server | |
parent | 138f2a38192801522e5c1241839f9a9e6a431982 (diff) |
Add support for zookeeper server version 3.7.1
Diffstat (limited to 'zookeeper-server')
13 files changed, 766 insertions, 1 deletions
diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index e88493c1a1b..42ecff1cc96 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,3 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) add_subdirectory(zookeeper-server) +add_subdirectory(zookeeper-server-3.7.1) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 1685a075955..34f4c0e9191 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -14,6 +14,7 @@ <modules> <module>zookeeper-server-common</module> <module>zookeeper-server</module> + <module>zookeeper-server-3.7.1</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt new file mode 100644 index 00000000000..3f90b81966b --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zookeeper-server-3.7.1) diff --git a/zookeeper-server/zookeeper-server-3.7.1/pom.xml b/zookeeper-server/zookeeper-server-3.7.1/pom.xml new file mode 100644 index 00000000000..ed677ef8601 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/pom.xml @@ -0,0 +1,123 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>zookeeper-server-3.7.1</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <properties> + <zookeeper.version>3.7.1</zookeeper.version> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Don't use ZK version from zookeeper-client-common --> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- snappy-java and metrics-core are included here + to be able to work with ZooKeeper 3.7.0 due to + class loading issues --> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <forkMode>once</forkMode> + <systemPropertyVariables> + <zk-version>${zookeeper.version}</zk-version> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <importPackage>com.sun.management</importPackage> + <bundleSymbolicName>zookeeper-server</bundleSymbolicName> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java new file mode 100644 index 00000000000..e94110af2fb --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Starts or reconfigures zookeeper cluster. + * The QuorumPeer conditionally created here is owned by the Reconfigurer; + * when it already has a peer, that peer is used here in case start or shutdown is required. + * + * @author hmusum + */ +public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { + + private QuorumPeer peer; + + @Inject + public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return true; + } + +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java new file mode 100644 index 00000000000..66742b0e05b --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.auth.AuthenticationProvider; +import org.apache.zookeeper.server.auth.X509AuthenticationProvider; + +import java.security.cert.X509Certificate; +import java.util.logging.Logger; + +/** + * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS + * + * @author bjorncs + */ +public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { + + private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); + + public VespaMtlsAuthenticationProvider() throws X509Exception { super(null, null);} + + @Override + public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { + // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. + // The client is authorized once the SSL handshake has completed. + X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); + if (certificateChain == null || certificateChain.length == 0) { + log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); + return KeeperException.Code.AUTHFAILED; + } + X509Certificate certificate = certificateChain[0]; + cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); + return KeeperException.Code.OK; + } + + @Override public String getScheme() { return "x509"; } + +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java new file mode 100644 index 00000000000..47ec03367c1 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.protect.Process; +import org.apache.zookeeper.server.admin.AdminServer; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps + * exceptions so it can be used by code that does not depend on ZooKeeper. + * + * @author hmusum + */ +class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); + + @Override + public void start(Path path) { + initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); + } + + @Override + public void shutdown(Duration timeout) { + if (quorumPeer != null) { + log.log(Level.FINE, "Shutting down ZooKeeper server"); + try { + quorumPeer.shutdown(); + quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete + if (quorumPeer.isAlive()) + throw new IllegalStateException("Peer still alive after " + timeout); + } catch (RuntimeException | InterruptedException e) { + // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. + // + // When a VespaZooKeeperServer component receives a new config, the container will try to start a new + // server with the new config, this will fail until the old server is deconstructed. If the old server + // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is + // the better option. + Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); + } + } + } + + @Override + protected void initializeAndRun(String[] args) { + try { + super.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { + throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java new file mode 100644 index 00000000000..ae7bf8d84f5 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author hmusum + */ +@SuppressWarnings("unused") // Created by injection +public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); + + @Override + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { + long fromConfig = -1; + // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); + log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { + return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), + (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); + } + +} + diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..48f95d28910 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; + +import java.nio.file.Path; +import java.time.Duration; + +/** + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { + + private final VespaQuorumPeer peer; + private final ZooKeeperRunner runner; + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + this.peer = new VespaQuorumPeer(); + this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); + } + + @Override + public void deconstruct() { + runner.shutdown(); + super.deconstruct(); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return false; + } + +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java new file mode 100644 index 00000000000..33ec9b1303a --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.common; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +/** + * This class contains common utilities for netstuff. Like printing IPv6 literals correctly + */ +public class NetUtils { + + // Note: Changed from original to use hostname from InetSocketAddress if there exists one + public static String formatInetAddr(InetSocketAddress addr) { + String hostName = addr.getHostName(); + if (hostName != null) { + return String.format("%s:%s", hostName, addr.getPort()); + } + + InetAddress ia = addr.getAddress(); + + if (ia == null) { + return String.format("%s:%s", addr.getHostString(), addr.getPort()); + } + if (ia instanceof Inet6Address) { + return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); + } else { + return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); + } + } + + /** + * Separates host and port from given host port string if host port string is enclosed + * within square bracket. + * + * @param hostPort host port string + * @return String[]{host, port} if host port string is host:port + * or String[] {host, port:port} if host port string is host:port:port + * or String[] {host} if host port string is host + * or String[]{} if not a ipv6 host port string. + */ + public static String[] getIPV6HostAndPort(String hostPort) { + if (hostPort.startsWith("[")) { + int i = hostPort.lastIndexOf(']'); + if (i < 0) { + throw new IllegalArgumentException( + hostPort + " starts with '[' but has no matching ']'"); + } + String host = hostPort.substring(1, i); + if (host.isEmpty()) { + throw new IllegalArgumentException(host + " is empty."); + } + if (hostPort.length() > i + 1) { + return getHostPort(hostPort, i, host); + } + return new String[] { host }; + } else { + //Not an IPV6 host port string + return new String[] {}; + } + } + + private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { + // [127::1]:2181 , check separator : exits + if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { + throw new IllegalArgumentException(hostPort + " does not have : after ]"); + } + // [127::1]: scenario + if (indexOfClosingBracket + 2 == hostPort.length()) { + throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); + } + //do not include + String port = hostPort.substring(indexOfClosingBracket + 2); + return new String[] { host, port }; + } +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java new file mode 100644 index 00000000000..fdfe0fe8467 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package org.apache.zookeeper.server; + +import com.yahoo.vespa.zookeeper.Configurator; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.logging.Logger; + +/** + * Overrides secure setting with value from {@link Configurator}. + * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 + * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. + * + * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. + * + * @author bjorncs + */ +public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { + + private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); + + private final boolean isSecure; + + public VespaNettyServerCnxnFactory() { + super(); + this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; + boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); + log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); + } + + @Override + public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { + log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); + super.configure(addr, maxClientCnxns, backlog, isSecure); + } +} diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java new file mode 100644 index 00000000000..db643d76e0d --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java @@ -0,0 +1,259 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.Reconfigurer; +import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +public class VespaZooKeeperTest { + + static final Path tempDirRoot = getTmpDir(); + static final List<Integer> ports = new ArrayList<>(); + + /** + * Performs dynamic reconfiguration of ZooKeeper servers. + * + * First, a cluster of 3 servers is set up, and some data is written to it. + * Then, 3 new servers are added, and the first 3 marked for retirement; + * this should force the quorum to move the 3 new servers, but not disconnect the old ones. + * Next, the old servers are removed. + * Then, the cluster is reduced to size 1. + * Finally, the cluster grows to size 3 again. + * + * Throughout all of this, quorum should remain, and the data should remain the same. + */ + @Test(timeout = 120_000) + @Ignore // Unstable, some ZK server keeps resetting connections sometimes. + public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { + List<ZooKeeper> keepers = new ArrayList<>(); + for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); + for (int i = 0; i < 8; i++) keepers.get(i).run(); + + // Start the first three servers. + List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); + for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for all servers to be up and running. + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Write data to verify later. + String path = writeData(configs.get(0)); + + // Let three new servers join, causing the three older ones to retire and leave the ensemble. + configs = getConfigs(0, 3, 3, 3); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // The existing servers can't reconfigure and leave before the joiners are up. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for new quorum to be established. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + // Old servers are removed. + configs = getConfigs(3, 0, 3, 0); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // Old servers shut down, while the newer servers remain. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Ensure old servers shut down properly. + for (int i = 0; i < 3; i++) keepers.get(i).await(); + // Ensure new servers have reconfigured. + for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + + // Cluster shrinks to a single server. + configs = getConfigs(5, 0, 1, 0); + for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // We let the remaining server reconfigure the others out before they die. + for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 3; i < 5; i++) keepers.get(i).await(); + verifyData(path, configs.get(5)); + + // Cluster grows to 3 servers again. + configs = getConfigs(5, 0, 3, 2); + for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Wait for the joiners. + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + verifyData(path, configs.get(7)); + + // Let the remaining servers terminate. + for (int i = 5; i < 8; i++) keepers.get(i).config = null; + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 8; i++) keepers.get(i).await(); + } + + static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + try (ZooKeeperAdmin admin = createAdmin(config)) { + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); + String read = new String(admin.getData(node, false, new Stat()), UTF_8); + assertEquals("hi", read); + return node; + } + } + + static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + for (int i = 0; i < 10; i++) { + try (ZooKeeperAdmin admin = createAdmin(config)) { + assertEquals("hi", new String(admin.getData(path, false, new Stat()), UTF_8)); + return; + } + catch (KeeperException.ConnectionLossException e) { + e.printStackTrace(); + Thread.sleep(10 << i); + } + } + } + + static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { + return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), + 10_000, + System.err::println, + new ZkClientConfigBuilder().toConfig()); + } + + static class ZooKeeper { + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Phaser phaser = new Phaser(2); + final AtomicReference<Future<?>> future = new AtomicReference<>(); + ZookeeperServerConfig config; + + void run() { + future.set(executor.submit(() -> { + Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); + phaser.arriveAndAwaitAdvance(); + while (config != null) { + new ReconfigurableVespaZooKeeperServer(reconfigurer, config); + phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here + phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff + } + reconfigurer.deconstruct(); + })); + } + + void await() throws ExecutionException, InterruptedException, TimeoutException { + future.get().get(30, SECONDS); + } + } + + static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { + return IntStream.rangeClosed(1, removed + retired + active) + .mapToObj(id -> getConfig(removed, retired, active, joining, id)) + .collect(toList()); + } + + // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. + static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { + if (id <= removed) + return null; + + Path tempDir = tempDirRoot.resolve("zookeeper-" + id); + return new ZookeeperServerConfig.Builder() + .clientPort(getPorts(id).get(0)) + .dataDir(tempDir.toString()) + .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) + .myid(id) + .myidFile(tempDir.resolve("myid").toString()) + .dynamicReconfiguration(true) + .server(IntStream.rangeClosed(removed + 1, removed + retired + active) + .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() + .id(i) + .clientPort(getPorts(i).get(0)) + .electionPort(getPorts(i).get(1)) + .quorumPort(getPorts(i).get(2)) + .hostname("localhost") + .joining(i - removed > retired + active - joining) + .retired(i - removed <= retired)) + .collect(toList())) + .build(); + } + + static List<Integer> getPorts(int id) { + if (ports.size() < id * 3) { + int previousPort; + if (ports.isEmpty()) { + String[] version = System.getProperty("zk-version").split("\\."); + int versionPortOffset = 0; + for (String part : version) + versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); + previousPort = 20000 + versionPortOffset % 30000; + } + else + previousPort = ports.get(ports.size() - 1); + + for (int i = 0; i < 3; i++) + ports.add(previousPort = nextPort(previousPort)); + } + return ports.subList(id * 3 - 3, id * 3); + } + + static int nextPort(int previousPort) { + for (int j = 1; j <= 30000; j++) { + int port = (previousPort + j); + while (port > 50000) + port -= 30000; + + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } + catch (IOException e) { + System.err.println("Could not bind port " + port + ": " + e); + } + } + throw new RuntimeException("No free ports"); + } + + static Path getTmpDir() { + try { + Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); + tempDir.toFile().deleteOnExit(); + return tempDir.toAbsolutePath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server/CMakeLists.txt b/zookeeper-server/zookeeper-server/CMakeLists.txt index c1cf27fc0bc..1022ee5b2cd 100644 --- a/zookeeper-server/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(zookeeper-server-3.7.0) -# Make symlink so that we have a default version +# Make symlink so that we have a default version, should be done only in zookeeper-server module install_symlink(lib/jars/zookeeper-server-3.7.0-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) |