diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-05-18 22:01:16 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2020-05-18 22:01:16 +0200 |
commit | d497cbbbf650991e7d783f7f8f1b5aa50fb9fbe1 (patch) | |
tree | 6cdd01e48646feae287348910e729ebfeeed8214 /zookeeper-server | |
parent | f0fd6a1ab95632f1d05ae576f0449ea92576150f (diff) |
Build implementations for Zookeeper 3.5.6 and 3.5.8
Diffstat (limited to 'zookeeper-server')
11 files changed, 763 insertions, 6 deletions
diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 6e8c82bc66e..b146390046c 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,3 +1,4 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.5) +add_subdirectory(zookeeper-server-3.5.6) +add_subdirectory(zookeeper-server-3.5.8) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index edfbdbad02e..28f18000c2d 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,8 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.5</module> + <module>zookeeper-server-3.5.6</module> + <module>zookeeper-server-3.5.8</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt new file mode 100644 index 00000000000..b68994d32e0 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zookeeper-server-3.5.6) +install_symlink(lib/jars/zookeeper-server-3.5.6-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.5/pom.xml b/zookeeper-server/zookeeper-server-3.5.6/pom.xml index 0bbb97bd38c..e50324a0488 100644 --- a/zookeeper-server/zookeeper-server-3.5/pom.xml +++ b/zookeeper-server/zookeeper-server-3.5.6/pom.xml @@ -8,7 +8,7 @@ <version>7-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.5</artifactId> + <artifactId>zookeeper-server-3.5.6</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> <dependencies> diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index 1f66c919a41..1f66c919a41 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java index 72351244cce..72351244cce 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java diff --git a/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt new file mode 100644 index 00000000000..66765e34997 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zookeeper-server-3.5.8) +# TODO: Needs to be included when this is the wanted default version (and other symlinks need to be removed) +#install_symlink(lib/jars/zookeeper-server-3.5.8-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.5.8/pom.xml b/zookeeper-server/zookeeper-server-3.5.8/pom.xml new file mode 100644 index 00000000000..e4000285ffa --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/pom.xml @@ -0,0 +1,78 @@ +<?xml version="1.0"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>zookeeper-server-3.5.8</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.5.8</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <forkMode>once</forkMode> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <importPackage>com.sun.management</importPackage> + <bundleSymbolicName>zookeeper-server</bundleSymbolicName> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..367742a450f --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,351 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import java.util.logging.Level; +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyStoreUtils; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.TransportSecurityOptions; +import com.yahoo.security.tls.TransportSecurityUtils; +import com.yahoo.text.Utf8; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * Writes zookeeper config and starts zookeeper server. + * + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); + private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; + static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; + private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; + private final String configFilePath; + private final String jksKeyStoreFilePath; + + VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer, + Optional<TransportSecurityOptions> transportSecurityOptions) { + this.zookeeperServerConfig = zookeeperServerConfig; + System.setProperty("zookeeper.jmx.log4j.disable", "true"); + System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); + System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); + + configFilePath = getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile()); + jksKeyStoreFilePath = getDefaults().underVespaHome(zookeeperServerConfig.jksKeyStoreFile()); + writeConfigToDisk(zookeeperServerConfig, transportSecurityOptions); + zkServerThread = new Thread(this, "zookeeper server"); + if (startServer) { + zkServerThread.start(); + } + } + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + this(zookeeperServerConfig, true, TransportSecurityUtils.getOptions()); + } + + private void writeConfigToDisk(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) { + new File(configFilePath).getParentFile().mkdirs(); + + try { + writeZooKeeperConfigFile(zookeeperServerConfig, transportSecurityOptions); + writeMyIdFile(config); + transportSecurityOptions.ifPresent(this::writeJksKeystore); + } catch (IOException e) { + throw new RuntimeException("Error writing zookeeper config", e); + } + } + + private void writeZooKeeperConfigFile(ZookeeperServerConfig config, + Optional<TransportSecurityOptions> transportSecurityOptions) throws IOException { + try (FileWriter writer = new FileWriter(configFilePath)) { + writer.write(transformConfigToString(config, transportSecurityOptions)); + } + } + + private String transformConfigToString(ZookeeperServerConfig config, + Optional<TransportSecurityOptions> transportSecurityOptions) { + StringBuilder sb = new StringBuilder(); + sb.append("tickTime=").append(config.tickTime()).append("\n"); + sb.append("initLimit=").append(config.initLimit()).append("\n"); + sb.append("syncLimit=").append(config.syncLimit()).append("\n"); + sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n"); + sb.append("snapCount=").append(config.snapshotCount()).append("\n"); + sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n"); + sb.append("clientPort=").append(config.clientPort()).append("\n"); + sb.append("secureClientPort=").append(config.secureClientPort()).append("\n"); + sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); + sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); + // See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands + // Includes all available commands in 3.5, except 'wchc' and 'wchp' + sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); + sb.append("admin.enableServer=false").append("\n"); + // Need NettyServerCnxnFactory to be able to use TLS for communication + sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n"); + sb.append("quorumListenOnAllIPs=true").append("\n"); + ensureThisServerIsRepresented(config.myid(), config.server()); + config.server().forEach(server -> addServerToCfg(sb, server)); + SSLContext sslContext = new SslContextBuilder().build(); + sb.append(new TlsQuorumConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions)); + sb.append(new TlsClientServerConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions)); + return sb.toString(); + } + + private void writeMyIdFile(ZookeeperServerConfig config) throws IOException { + if (config.server().size() > 1) { + try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) { + writer.write(config.myid() + "\n"); + } + } + } + + private void writeJksKeystore(TransportSecurityOptions options) { + Path privateKeyFile = options.getPrivateKeyFile().orElseThrow(() -> new RuntimeException("Could not find private key file")); + Path certificatesFile = options.getCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find certificates file")); + + PrivateKey privateKey; + List<X509Certificate> certificates; + try { + privateKey = KeyUtils.fromPemEncodedPrivateKey(Utf8.toString(Files.readAllBytes(privateKeyFile))); + certificates = X509CertificateUtils.certificateListFromPem(Utf8.toString(Files.readAllBytes(certificatesFile))); + } catch (IOException e) { + throw new RuntimeException(e); + } + KeyStoreBuilder keyStoreBuilder = KeyStoreBuilder + .withType(KeyStoreType.JKS) + .withKeyEntry("foo", privateKey, certificates); + + KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), Paths.get(jksKeyStoreFilePath)); + } + + private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { + boolean found = false; + for (ZookeeperServerConfig.Server server : servers) { + if (myid == server.id()) { + found = true; + break; + } + } + if (!found) { + throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")"); + } + } + + private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) { + sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n"); + } + + private void shutdown() { + zkServerThread.interrupt(); + try { + zkServerThread.join(); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Error joining server thread on shutdown", e); + } + } + + @Override + public void run() { + System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); + String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); + org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + } + + @Override + public void deconstruct() { + shutdown(); + super.deconstruct(); + } + + private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { + return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); + } + + private interface TlsConfig { + default Set<String> allowedCiphers(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedCipherSuites(sslContext)); } + + default Set<String> allowedProtocols(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedProtocols(sslContext)); } + + default Optional<String> getEnvironmentVariable(String variableName) { + return Optional.ofNullable(System.getenv().get(variableName)) + .filter(var -> !var.isEmpty()); + } + + default void validateOptions(Optional<TransportSecurityOptions> transportSecurityOptions, String tlsSetting) { + if (transportSecurityOptions.isEmpty() && !tlsSetting.equals("OFF")) + throw new RuntimeException("Could not retrieve transport security options"); + } + + String configFieldPrefix(); + + String jksKeyStoreFilePath(); + + SSLContext sslContext(); + + default String createCommonKeyStoreTrustStoreOptions(Optional<TransportSecurityOptions> transportSecurityOptions) { + StringBuilder sb = new StringBuilder(); + transportSecurityOptions.ifPresent(options -> { + sb.append(configFieldPrefix()).append(".keyStore.location=").append(jksKeyStoreFilePath()).append("\n"); + sb.append(configFieldPrefix()).append(".keyStore.type=JKS\n"); + + Path caCertificatesFile = options.getCaCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find ca certificates file")); + sb.append(configFieldPrefix()).append(".trustStore.location=").append(caCertificatesFile).append("\n"); + sb.append(configFieldPrefix()).append(".trustStore.type=PEM\n"); + }); + return sb.toString(); + } + + default String createCommonConfig() { + StringBuilder sb = new StringBuilder(); + sb.append(configFieldPrefix()).append(".hostnameVerification=false\n"); + sb.append(configFieldPrefix()).append(".clientAuth=NEED\n"); + sb.append(configFieldPrefix()).append(".ciphersuites=").append(String.join(",", allowedCiphers(sslContext()))).append("\n"); + sb.append(configFieldPrefix()).append(".enabledProtocols=").append(String.join(",", allowedProtocols(sslContext()))).append("\n"); + sb.append(configFieldPrefix()).append(".protocol=").append(sslContext().getProtocol()).append("\n"); + + return sb.toString(); + } + + } + + static class TlsClientServerConfig implements TlsConfig { + + private final SSLContext sslContext; + private final String jksKeyStoreFilePath; + + TlsClientServerConfig(SSLContext sslContext, String jksKeyStoreFilePath) { + this.sslContext = sslContext; + this.jksKeyStoreFilePath = jksKeyStoreFilePath; + } + + String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) { + String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION") + .orElse(config.tlsForClientServerCommunication().name()); + validateOptions(transportSecurityOptions, tlsSetting); + + StringBuilder sb = new StringBuilder(createCommonConfig()); + boolean portUnification; + switch (tlsSetting) { + case "OFF": + case "TLS_ONLY": + portUnification = false; + break; + case "PORT_UNIFICATION": + case "TLS_WITH_PORT_UNIFICATION": + portUnification = true; + break; + default: + throw new IllegalArgumentException("Unknown value of config setting tlsForClientServerCommunication: " + tlsSetting); + } + sb.append("client.portUnification=").append(portUnification).append("\n"); + sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions)); + + return sb.toString(); + } + + @Override + public String configFieldPrefix() { + return "ssl"; + } + + @Override + public String jksKeyStoreFilePath() { + return jksKeyStoreFilePath; + } + + @Override + public SSLContext sslContext() { + return sslContext; + } + } + + static class TlsQuorumConfig implements TlsConfig { + + private final SSLContext sslContext; + private final String jksKeyStoreFilePath; + + TlsQuorumConfig(SSLContext sslContext, String jksKeyStoreFilePath) { + this.sslContext = sslContext; + this.jksKeyStoreFilePath = jksKeyStoreFilePath; + } + + String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) { + String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION") + .orElse(config.tlsForQuorumCommunication().name()); + validateOptions(transportSecurityOptions, tlsSetting); + + StringBuilder sb = new StringBuilder(createCommonConfig()); + boolean sslQuorum; + boolean portUnification; + switch (tlsSetting) { + case "OFF": + sslQuorum = false; + portUnification = false; + break; + case "PORT_UNIFICATION": + sslQuorum = false; + portUnification = true; + break; + case "TLS_WITH_PORT_UNIFICATION": + sslQuorum = true; + portUnification = true; + break; + case "TLS_ONLY": + sslQuorum = true; + portUnification = false; + break; + default: throw new IllegalArgumentException("Unknown value of config setting tlsForQuorumCommunication: " + tlsSetting); + } + sb.append("sslQuorum=").append(sslQuorum).append("\n"); + sb.append("portUnification=").append(portUnification).append("\n"); + sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions)); + + return sb.toString(); + } + + @Override + public String configFieldPrefix() { + return "ssl.quorum"; + } + + @Override + public String jksKeyStoreFilePath() { + return jksKeyStoreFilePath; + } + + @Override + public SSLContext sslContext() { + return sslContext; + } + + } + +} diff --git a/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java new file mode 100644 index 00000000000..86d424bd41b --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java @@ -0,0 +1,322 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.TransportSecurityOptions; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import javax.security.auth.x500.X500Principal; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.util.Optional; + +import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForQuorumCommunication; +import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForClientServerCommunication; +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static java.time.Instant.EPOCH; +import static java.time.temporal.ChronoUnit.DAYS; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static org.junit.Assert.assertTrue; + +/** + * Tests the zookeeper server. + */ +public class VespaZooKeeperServerImplTest { + + private File cfgFile; + private File idFile; + private File jksKeyStoreFile; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + cfgFile = folder.newFile(); + idFile = folder.newFile(); + jksKeyStoreFile = folder.newFile(); + } + + @Test + public void config_is_written_correctly_when_one_server() throws IOException { + ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile); + createServer(builder); + validateConfigFileSingleHost(cfgFile); + validateIdFile(idFile, ""); + } + + @Test + public void config_is_written_correctly_when_multiple_servers() throws IOException { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.server(newServer(0, "foo", 123, 321)); + builder.server(newServer(1, "bar", 234, 432)); + builder.server(newServer(2, "baz", 345, 543)); + builder.myidFile(idFile.getAbsolutePath()); + builder.myid(1); + createServer(builder); + validateConfigFileMultipleHosts(cfgFile); + validateIdFile(idFile, "1\n"); + } + + @Test + public void config_is_written_correctly_with_tls_for_quorum_communication_port_unification() throws IOException { + ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile); + builder.tlsForQuorumCommunication(TlsForQuorumCommunication.PORT_UNIFICATION); + builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.PORT_UNIFICATION); + Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); + createServer(builder, transportSecurityOptions); + validateConfigFilePortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); + validateThatJksKeyStoreFileExists(jksKeyStoreFile); + } + + @Test + public void config_is_written_correctly_with_tls_for_quorum_communication_tls_with_port_unification() throws IOException { + ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile); + builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_WITH_PORT_UNIFICATION); + builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_WITH_PORT_UNIFICATION); + Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); + createServer(builder, transportSecurityOptions); + validateConfigFileTlsWithPortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); + validateThatJksKeyStoreFileExists(jksKeyStoreFile); + } + + @Test + public void config_is_written_correctly_with_tls_for_quorum_communication_tls_only() throws IOException { + ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile); + builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_ONLY); + builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_ONLY); + Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); + createServer(builder, transportSecurityOptions); + validateConfigFileTlsOnly(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); + validateThatJksKeyStoreFileExists(jksKeyStoreFile); + } + + private ZookeeperServerConfig.Builder createConfigBuilderForSingleHost(File cfgFile, File idFile, File jksKeyStoreFile) { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.myidFile(idFile.getAbsolutePath()); + builder.server(newServer(0, "foo", 123, 321)); + builder.myid(0); + builder.jksKeyStoreFile(jksKeyStoreFile.getAbsolutePath()); + return builder; + } + + private void createServer(ZookeeperServerConfig.Builder builder) { + createServer(builder, Optional.empty()); + } + + private void createServer(ZookeeperServerConfig.Builder builder, Optional<TransportSecurityOptions> options) { + new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false, options); + } + + @Test(expected = RuntimeException.class) + public void require_that_this_id_must_be_present_amongst_servers() { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.server(newServer(1, "bar", 234, 432)); + builder.server(newServer(2, "baz", 345, 543)); + builder.myid(0); + createServer(builder); + } + + @Test + public void juteMaxBufferCanBeSet() throws IOException { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.myid(0); + File idFile = folder.newFile(); + File cfgFile = folder.newFile(); + + builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost")); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.myidFile(idFile.getAbsolutePath()); + + createServer(builder); + assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); + + final int max_buffer = 1; + builder.juteMaxBuffer(max_buffer); + createServer(builder); + assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); + } + + private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) { + ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); + builder.id(id); + builder.hostname(hostName); + builder.electionPort(electionPort); + builder.quorumPort(quorumPort); + return builder; + } + + private void validateIdFile(File idFile, String expected) throws IOException { + String actual = IOUtils.readFile(idFile); + assertThat(actual, is(expected)); + } + + private String commonConfig() { + return "tickTime=2000\n" + + "initLimit=20\n" + + "syncLimit=15\n" + + "maxClientCnxns=0\n" + + "snapCount=50000\n" + + "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + + "clientPort=2181\n" + + "secureClientPort=2184\n" + + "autopurge.purgeInterval=1\n" + + "autopurge.snapRetainCount=15\n" + + "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + + "admin.enableServer=false\n" + + "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" + + "quorumListenOnAllIPs=true\n"; + } + + private String quorumKeyStoreAndTrustStoreConfig(File jksKeyStoreFilePath, File caCertificatesFilePath) { + StringBuilder sb = new StringBuilder(); + + sb.append("ssl.quorum.keyStore.location=").append(jksKeyStoreFilePath.getAbsolutePath()).append("\n"); + sb.append("ssl.quorum.keyStore.type=JKS\n"); + sb.append("ssl.quorum.trustStore.location=").append(caCertificatesFilePath.getAbsolutePath()).append("\n"); + sb.append("ssl.quorum.trustStore.type=PEM\n"); + return sb.toString(); + } + + private String clientServerKeyStoreAndTrustStoreConfig(File jksKeyStoreFilePath, File caCertificatesFilePath) { + StringBuilder sb = new StringBuilder(); + + sb.append("ssl.keyStore.location=").append(jksKeyStoreFilePath.getAbsolutePath()).append("\n"); + sb.append("ssl.keyStore.type=JKS\n"); + sb.append("ssl.trustStore.location=").append(caCertificatesFilePath.getAbsolutePath()).append("\n"); + sb.append("ssl.trustStore.type=PEM\n"); + return sb.toString(); + } + + private void validateConfigFileSingleHost(File cfgFile) throws IOException { + String expected = + commonConfig() + + "server.0=foo:321:123\n" + + commonTlsQuorumConfig() + + "sslQuorum=false\n" + + "portUnification=false\n" + + commonTlsClientServerConfig() + + "client.portUnification=false\n"; + validateConfigFile(cfgFile, expected); + } + + private String commonTlsQuorumConfig() { + return "ssl.quorum.hostnameVerification=false\n" + + "ssl.quorum.clientAuth=NEED\n" + + "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + + "ssl.quorum.enabledProtocols=TLSv1.2\n" + + "ssl.quorum.protocol=TLS\n"; + } + + private String commonTlsClientServerConfig() { + return "ssl.hostnameVerification=false\n" + + "ssl.clientAuth=NEED\n" + + "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + + "ssl.enabledProtocols=TLSv1.2\n" + + "ssl.protocol=TLS\n"; + } + + private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { + String expected = + commonConfig() + + "server.0=foo:321:123\n" + + "server.1=bar:432:234\n" + + "server.2=baz:543:345\n" + + commonTlsQuorumConfig() + + "sslQuorum=false\n" + + "portUnification=false\n" + + commonTlsClientServerConfig() + + "client.portUnification=false\n"; + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFilePortUnification(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException { + String expected = + commonConfig() + + "server.0=foo:321:123\n" + + commonTlsQuorumConfig() + + "sslQuorum=false\n" + + "portUnification=true\n" + + quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) + + commonTlsClientServerConfig() + + "client.portUnification=true\n" + + clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile); + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFileTlsWithPortUnification(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException { + String expected = + commonConfig() + + "server.0=foo:321:123\n" + + commonTlsQuorumConfig() + + "sslQuorum=true\n" + + "portUnification=true\n" + + quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) + + commonTlsClientServerConfig() + + "client.portUnification=true\n" + + clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile); + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFileTlsOnly(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException { + String expected = + commonConfig() + + "server.0=foo:321:123\n" + + commonTlsQuorumConfig() + + "sslQuorum=true\n" + + "portUnification=false\n" + + quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) + + commonTlsClientServerConfig() + + "client.portUnification=false\n" + + clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile); + validateConfigFile(cfgFile, expected); + } + + private void validateConfigFile(File cfgFile, String expected) throws IOException { + String actual = IOUtils.readFile(cfgFile); + assertThat(actual, is(expected)); + } + + private void validateThatJksKeyStoreFileExists(File cfgFile) { + assertTrue(cfgFile.exists() && cfgFile.canRead()); + } + + private Optional<TransportSecurityOptions> createTransportSecurityOptions() throws IOException { + KeyPair keyPair = KeyUtils.generateKeypair(EC); + Path privateKeyFile = folder.newFile().toPath(); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + + X509Certificate certificate = X509CertificateBuilder + .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, EPOCH.plus(1, DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + Path certificateChainFile = folder.newFile().toPath(); + String certificatePem = X509CertificateUtils.toPem(certificate); + Files.writeString(certificateChainFile, certificatePem); + + Path caCertificatesFile = folder.newFile().toPath(); + Files.writeString(caCertificatesFile, certificatePem); + + return Optional.of(new TransportSecurityOptions.Builder() + .withCertificates(certificateChainFile, privateKeyFile) + .withCaCertificates(caCertificatesFile) + .build()); + } + +} diff --git a/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt deleted file mode 100644 index 782c5f07b83..00000000000 --- a/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.5) -install_symlink(lib/jars/zookeeper-server-3.5-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) |