summaryrefslogtreecommitdiffstats
path: root/zookeeper-server
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2020-05-19 13:53:38 +0200
committerHarald Musum <musum@verizonmedia.com>2020-05-19 13:53:38 +0200
commit26b22f4c05dd4018e931bb9b6ab2ac78fe556c8d (patch)
tree2ff045428a13587c9a6b31030ef13f43334a281e /zookeeper-server
parentd497cbbbf650991e7d783f7f8f1b5aa50fb9fbe1 (diff)
Move common code and tests to Configurator and ConfiguratorTest
Diffstat (limited to 'zookeeper-server')
-rw-r--r--zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java309
-rw-r--r--zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java306
-rw-r--r--zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java322
-rw-r--r--zookeeper-server/zookeeper-server-common/pom.xml7
-rw-r--r--zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java307
-rw-r--r--zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java (renamed from zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java)31
6 files changed, 339 insertions, 943 deletions
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index 1f66c919a41..ee3695b02f8 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -4,34 +4,12 @@ package com.yahoo.vespa.zookeeper;
import com.google.inject.Inject;
import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.component.AbstractComponent;
-import java.util.logging.Level;
-import com.yahoo.security.KeyStoreBuilder;
-import com.yahoo.security.KeyStoreType;
-import com.yahoo.security.KeyStoreUtils;
-import com.yahoo.security.KeyUtils;
-import com.yahoo.security.SslContextBuilder;
-import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.security.tls.TlsContext;
-import com.yahoo.security.tls.TransportSecurityOptions;
import com.yahoo.security.tls.TransportSecurityUtils;
-import com.yahoo.text.Utf8;
-import javax.net.ssl.SSLContext;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.PrivateKey;
-import java.security.cert.X509Certificate;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
+import java.util.logging.Level;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames;
/**
* Writes zookeeper config and starts zookeeper server.
@@ -40,125 +18,16 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults;
* @author Harald Musum
*/
public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer {
-
private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName());
- private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable";
- static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer";
private final Thread zkServerThread;
private final ZookeeperServerConfig zookeeperServerConfig;
- private final String configFilePath;
- private final String jksKeyStoreFilePath;
-
- VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer,
- Optional<TransportSecurityOptions> transportSecurityOptions) {
- this.zookeeperServerConfig = zookeeperServerConfig;
- System.setProperty("zookeeper.jmx.log4j.disable", "true");
- System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString());
- System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString());
-
- configFilePath = getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile());
- jksKeyStoreFilePath = getDefaults().underVespaHome(zookeeperServerConfig.jksKeyStoreFile());
- writeConfigToDisk(zookeeperServerConfig, transportSecurityOptions);
- zkServerThread = new Thread(this, "zookeeper server");
- if (startServer) {
- zkServerThread.start();
- }
- }
@Inject
public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) {
- this(zookeeperServerConfig, true, TransportSecurityUtils.getOptions());
- }
-
- private void writeConfigToDisk(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- new File(configFilePath).getParentFile().mkdirs();
-
- try {
- writeZooKeeperConfigFile(zookeeperServerConfig, transportSecurityOptions);
- writeMyIdFile(config);
- transportSecurityOptions.ifPresent(this::writeJksKeystore);
- } catch (IOException e) {
- throw new RuntimeException("Error writing zookeeper config", e);
- }
- }
-
- private void writeZooKeeperConfigFile(ZookeeperServerConfig config,
- Optional<TransportSecurityOptions> transportSecurityOptions) throws IOException {
- try (FileWriter writer = new FileWriter(configFilePath)) {
- writer.write(transformConfigToString(config, transportSecurityOptions));
- }
- }
-
- private String transformConfigToString(ZookeeperServerConfig config,
- Optional<TransportSecurityOptions> transportSecurityOptions) {
- StringBuilder sb = new StringBuilder();
- sb.append("tickTime=").append(config.tickTime()).append("\n");
- sb.append("initLimit=").append(config.initLimit()).append("\n");
- sb.append("syncLimit=").append(config.syncLimit()).append("\n");
- sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n");
- sb.append("snapCount=").append(config.snapshotCount()).append("\n");
- sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n");
- sb.append("clientPort=").append(config.clientPort()).append("\n");
- sb.append("secureClientPort=").append(config.secureClientPort()).append("\n");
- sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
- sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
- // See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands
- // Includes all available commands in 3.5, except 'wchc' and 'wchp'
- sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
- sb.append("admin.enableServer=false").append("\n");
- // Need NettyServerCnxnFactory to be able to use TLS for communication
- sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n");
- sb.append("quorumListenOnAllIPs=true").append("\n");
- ensureThisServerIsRepresented(config.myid(), config.server());
- config.server().forEach(server -> addServerToCfg(sb, server));
- SSLContext sslContext = new SslContextBuilder().build();
- sb.append(new TlsQuorumConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
- sb.append(new TlsClientServerConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
- return sb.toString();
- }
-
- private void writeMyIdFile(ZookeeperServerConfig config) throws IOException {
- if (config.server().size() > 1) {
- try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) {
- writer.write(config.myid() + "\n");
- }
- }
- }
-
- private void writeJksKeystore(TransportSecurityOptions options) {
- Path privateKeyFile = options.getPrivateKeyFile().orElseThrow(() -> new RuntimeException("Could not find private key file"));
- Path certificatesFile = options.getCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find certificates file"));
-
- PrivateKey privateKey;
- List<X509Certificate> certificates;
- try {
- privateKey = KeyUtils.fromPemEncodedPrivateKey(Utf8.toString(Files.readAllBytes(privateKeyFile)));
- certificates = X509CertificateUtils.certificateListFromPem(Utf8.toString(Files.readAllBytes(certificatesFile)));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- KeyStoreBuilder keyStoreBuilder = KeyStoreBuilder
- .withType(KeyStoreType.JKS)
- .withKeyEntry("foo", privateKey, certificates);
-
- KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), Paths.get(jksKeyStoreFilePath));
- }
-
- private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
- boolean found = false;
- for (ZookeeperServerConfig.Server server : servers) {
- if (myid == server.id()) {
- found = true;
- break;
- }
- }
- if (!found) {
- throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")");
- }
- }
-
- private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) {
- sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n");
+ this.zookeeperServerConfig = zookeeperServerConfig;
+ new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions());
+ zkServerThread = new Thread(this, "zookeeper server");
+ zkServerThread.start();
}
private void shutdown() {
@@ -172,10 +41,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna
@Override
public void run() {
- System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())};
log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] +
- ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
+ ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args);
}
@@ -185,167 +53,4 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna
super.deconstruct();
}
- private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) {
- return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet());
- }
-
- private interface TlsConfig {
- default Set<String> allowedCiphers(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedCipherSuites(sslContext)); }
-
- default Set<String> allowedProtocols(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedProtocols(sslContext)); }
-
- default Optional<String> getEnvironmentVariable(String variableName) {
- return Optional.ofNullable(System.getenv().get(variableName))
- .filter(var -> !var.isEmpty());
- }
-
- default void validateOptions(Optional<TransportSecurityOptions> transportSecurityOptions, String tlsSetting) {
- if (transportSecurityOptions.isEmpty() && !tlsSetting.equals("OFF"))
- throw new RuntimeException("Could not retrieve transport security options");
- }
-
- String configFieldPrefix();
-
- String jksKeyStoreFilePath();
-
- SSLContext sslContext();
-
- default String createCommonKeyStoreTrustStoreOptions(Optional<TransportSecurityOptions> transportSecurityOptions) {
- StringBuilder sb = new StringBuilder();
- transportSecurityOptions.ifPresent(options -> {
- sb.append(configFieldPrefix()).append(".keyStore.location=").append(jksKeyStoreFilePath()).append("\n");
- sb.append(configFieldPrefix()).append(".keyStore.type=JKS\n");
-
- Path caCertificatesFile = options.getCaCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find ca certificates file"));
- sb.append(configFieldPrefix()).append(".trustStore.location=").append(caCertificatesFile).append("\n");
- sb.append(configFieldPrefix()).append(".trustStore.type=PEM\n");
- });
- return sb.toString();
- }
-
- default String createCommonConfig() {
- StringBuilder sb = new StringBuilder();
- sb.append(configFieldPrefix()).append(".hostnameVerification=false\n");
- sb.append(configFieldPrefix()).append(".clientAuth=NEED\n");
- sb.append(configFieldPrefix()).append(".ciphersuites=").append(String.join(",", allowedCiphers(sslContext()))).append("\n");
- sb.append(configFieldPrefix()).append(".enabledProtocols=").append(String.join(",", allowedProtocols(sslContext()))).append("\n");
- sb.append(configFieldPrefix()).append(".protocol=").append(sslContext().getProtocol()).append("\n");
-
- return sb.toString();
- }
-
- }
-
- static class TlsClientServerConfig implements TlsConfig {
-
- private final SSLContext sslContext;
- private final String jksKeyStoreFilePath;
-
- TlsClientServerConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
- this.sslContext = sslContext;
- this.jksKeyStoreFilePath = jksKeyStoreFilePath;
- }
-
- String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION")
- .orElse(config.tlsForClientServerCommunication().name());
- validateOptions(transportSecurityOptions, tlsSetting);
-
- StringBuilder sb = new StringBuilder(createCommonConfig());
- boolean portUnification;
- switch (tlsSetting) {
- case "OFF":
- case "TLS_ONLY":
- portUnification = false;
- break;
- case "PORT_UNIFICATION":
- case "TLS_WITH_PORT_UNIFICATION":
- portUnification = true;
- break;
- default:
- throw new IllegalArgumentException("Unknown value of config setting tlsForClientServerCommunication: " + tlsSetting);
- }
- sb.append("client.portUnification=").append(portUnification).append("\n");
- sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
-
- return sb.toString();
- }
-
- @Override
- public String configFieldPrefix() {
- return "ssl";
- }
-
- @Override
- public String jksKeyStoreFilePath() {
- return jksKeyStoreFilePath;
- }
-
- @Override
- public SSLContext sslContext() {
- return sslContext;
- }
- }
-
- static class TlsQuorumConfig implements TlsConfig {
-
- private final SSLContext sslContext;
- private final String jksKeyStoreFilePath;
-
- TlsQuorumConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
- this.sslContext = sslContext;
- this.jksKeyStoreFilePath = jksKeyStoreFilePath;
- }
-
- String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION")
- .orElse(config.tlsForQuorumCommunication().name());
- validateOptions(transportSecurityOptions, tlsSetting);
-
- StringBuilder sb = new StringBuilder(createCommonConfig());
- boolean sslQuorum;
- boolean portUnification;
- switch (tlsSetting) {
- case "OFF":
- sslQuorum = false;
- portUnification = false;
- break;
- case "PORT_UNIFICATION":
- sslQuorum = false;
- portUnification = true;
- break;
- case "TLS_WITH_PORT_UNIFICATION":
- sslQuorum = true;
- portUnification = true;
- break;
- case "TLS_ONLY":
- sslQuorum = true;
- portUnification = false;
- break;
- default: throw new IllegalArgumentException("Unknown value of config setting tlsForQuorumCommunication: " + tlsSetting);
- }
- sb.append("sslQuorum=").append(sslQuorum).append("\n");
- sb.append("portUnification=").append(portUnification).append("\n");
- sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
-
- return sb.toString();
- }
-
- @Override
- public String configFieldPrefix() {
- return "ssl.quorum";
- }
-
- @Override
- public String jksKeyStoreFilePath() {
- return jksKeyStoreFilePath;
- }
-
- @Override
- public SSLContext sslContext() {
- return sslContext;
- }
-
- }
-
}
diff --git a/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
index 367742a450f..15d5d2f6f31 100644
--- a/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
@@ -4,34 +4,12 @@ package com.yahoo.vespa.zookeeper;
import com.google.inject.Inject;
import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.component.AbstractComponent;
-import java.util.logging.Level;
-import com.yahoo.security.KeyStoreBuilder;
-import com.yahoo.security.KeyStoreType;
-import com.yahoo.security.KeyStoreUtils;
-import com.yahoo.security.KeyUtils;
-import com.yahoo.security.SslContextBuilder;
-import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.security.tls.TlsContext;
-import com.yahoo.security.tls.TransportSecurityOptions;
import com.yahoo.security.tls.TransportSecurityUtils;
-import com.yahoo.text.Utf8;
-import javax.net.ssl.SSLContext;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.PrivateKey;
-import java.security.cert.X509Certificate;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
+import java.util.logging.Level;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames;
/**
* Writes zookeeper config and starts zookeeper server.
@@ -42,123 +20,15 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults;
public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer {
private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName());
- private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable";
- static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer";
private final Thread zkServerThread;
private final ZookeeperServerConfig zookeeperServerConfig;
- private final String configFilePath;
- private final String jksKeyStoreFilePath;
- VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer,
- Optional<TransportSecurityOptions> transportSecurityOptions) {
+ @Inject
+ public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) {
this.zookeeperServerConfig = zookeeperServerConfig;
- System.setProperty("zookeeper.jmx.log4j.disable", "true");
- System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString());
- System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString());
-
- configFilePath = getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile());
- jksKeyStoreFilePath = getDefaults().underVespaHome(zookeeperServerConfig.jksKeyStoreFile());
- writeConfigToDisk(zookeeperServerConfig, transportSecurityOptions);
+ new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions());
zkServerThread = new Thread(this, "zookeeper server");
- if (startServer) {
zkServerThread.start();
- }
- }
-
- @Inject
- public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) {
- this(zookeeperServerConfig, true, TransportSecurityUtils.getOptions());
- }
-
- private void writeConfigToDisk(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- new File(configFilePath).getParentFile().mkdirs();
-
- try {
- writeZooKeeperConfigFile(zookeeperServerConfig, transportSecurityOptions);
- writeMyIdFile(config);
- transportSecurityOptions.ifPresent(this::writeJksKeystore);
- } catch (IOException e) {
- throw new RuntimeException("Error writing zookeeper config", e);
- }
- }
-
- private void writeZooKeeperConfigFile(ZookeeperServerConfig config,
- Optional<TransportSecurityOptions> transportSecurityOptions) throws IOException {
- try (FileWriter writer = new FileWriter(configFilePath)) {
- writer.write(transformConfigToString(config, transportSecurityOptions));
- }
- }
-
- private String transformConfigToString(ZookeeperServerConfig config,
- Optional<TransportSecurityOptions> transportSecurityOptions) {
- StringBuilder sb = new StringBuilder();
- sb.append("tickTime=").append(config.tickTime()).append("\n");
- sb.append("initLimit=").append(config.initLimit()).append("\n");
- sb.append("syncLimit=").append(config.syncLimit()).append("\n");
- sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n");
- sb.append("snapCount=").append(config.snapshotCount()).append("\n");
- sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n");
- sb.append("clientPort=").append(config.clientPort()).append("\n");
- sb.append("secureClientPort=").append(config.secureClientPort()).append("\n");
- sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
- sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
- // See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands
- // Includes all available commands in 3.5, except 'wchc' and 'wchp'
- sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
- sb.append("admin.enableServer=false").append("\n");
- // Need NettyServerCnxnFactory to be able to use TLS for communication
- sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n");
- sb.append("quorumListenOnAllIPs=true").append("\n");
- ensureThisServerIsRepresented(config.myid(), config.server());
- config.server().forEach(server -> addServerToCfg(sb, server));
- SSLContext sslContext = new SslContextBuilder().build();
- sb.append(new TlsQuorumConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
- sb.append(new TlsClientServerConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
- return sb.toString();
- }
-
- private void writeMyIdFile(ZookeeperServerConfig config) throws IOException {
- if (config.server().size() > 1) {
- try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) {
- writer.write(config.myid() + "\n");
- }
- }
- }
-
- private void writeJksKeystore(TransportSecurityOptions options) {
- Path privateKeyFile = options.getPrivateKeyFile().orElseThrow(() -> new RuntimeException("Could not find private key file"));
- Path certificatesFile = options.getCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find certificates file"));
-
- PrivateKey privateKey;
- List<X509Certificate> certificates;
- try {
- privateKey = KeyUtils.fromPemEncodedPrivateKey(Utf8.toString(Files.readAllBytes(privateKeyFile)));
- certificates = X509CertificateUtils.certificateListFromPem(Utf8.toString(Files.readAllBytes(certificatesFile)));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- KeyStoreBuilder keyStoreBuilder = KeyStoreBuilder
- .withType(KeyStoreType.JKS)
- .withKeyEntry("foo", privateKey, certificates);
-
- KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), Paths.get(jksKeyStoreFilePath));
- }
-
- private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
- boolean found = false;
- for (ZookeeperServerConfig.Server server : servers) {
- if (myid == server.id()) {
- found = true;
- break;
- }
- }
- if (!found) {
- throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")");
- }
- }
-
- private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) {
- sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n");
}
private void shutdown() {
@@ -172,10 +42,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna
@Override
public void run() {
- System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())};
log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] +
- ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
+ ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args);
}
@@ -185,167 +54,4 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna
super.deconstruct();
}
- private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) {
- return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet());
- }
-
- private interface TlsConfig {
- default Set<String> allowedCiphers(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedCipherSuites(sslContext)); }
-
- default Set<String> allowedProtocols(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedProtocols(sslContext)); }
-
- default Optional<String> getEnvironmentVariable(String variableName) {
- return Optional.ofNullable(System.getenv().get(variableName))
- .filter(var -> !var.isEmpty());
- }
-
- default void validateOptions(Optional<TransportSecurityOptions> transportSecurityOptions, String tlsSetting) {
- if (transportSecurityOptions.isEmpty() && !tlsSetting.equals("OFF"))
- throw new RuntimeException("Could not retrieve transport security options");
- }
-
- String configFieldPrefix();
-
- String jksKeyStoreFilePath();
-
- SSLContext sslContext();
-
- default String createCommonKeyStoreTrustStoreOptions(Optional<TransportSecurityOptions> transportSecurityOptions) {
- StringBuilder sb = new StringBuilder();
- transportSecurityOptions.ifPresent(options -> {
- sb.append(configFieldPrefix()).append(".keyStore.location=").append(jksKeyStoreFilePath()).append("\n");
- sb.append(configFieldPrefix()).append(".keyStore.type=JKS\n");
-
- Path caCertificatesFile = options.getCaCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find ca certificates file"));
- sb.append(configFieldPrefix()).append(".trustStore.location=").append(caCertificatesFile).append("\n");
- sb.append(configFieldPrefix()).append(".trustStore.type=PEM\n");
- });
- return sb.toString();
- }
-
- default String createCommonConfig() {
- StringBuilder sb = new StringBuilder();
- sb.append(configFieldPrefix()).append(".hostnameVerification=false\n");
- sb.append(configFieldPrefix()).append(".clientAuth=NEED\n");
- sb.append(configFieldPrefix()).append(".ciphersuites=").append(String.join(",", allowedCiphers(sslContext()))).append("\n");
- sb.append(configFieldPrefix()).append(".enabledProtocols=").append(String.join(",", allowedProtocols(sslContext()))).append("\n");
- sb.append(configFieldPrefix()).append(".protocol=").append(sslContext().getProtocol()).append("\n");
-
- return sb.toString();
- }
-
- }
-
- static class TlsClientServerConfig implements TlsConfig {
-
- private final SSLContext sslContext;
- private final String jksKeyStoreFilePath;
-
- TlsClientServerConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
- this.sslContext = sslContext;
- this.jksKeyStoreFilePath = jksKeyStoreFilePath;
- }
-
- String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION")
- .orElse(config.tlsForClientServerCommunication().name());
- validateOptions(transportSecurityOptions, tlsSetting);
-
- StringBuilder sb = new StringBuilder(createCommonConfig());
- boolean portUnification;
- switch (tlsSetting) {
- case "OFF":
- case "TLS_ONLY":
- portUnification = false;
- break;
- case "PORT_UNIFICATION":
- case "TLS_WITH_PORT_UNIFICATION":
- portUnification = true;
- break;
- default:
- throw new IllegalArgumentException("Unknown value of config setting tlsForClientServerCommunication: " + tlsSetting);
- }
- sb.append("client.portUnification=").append(portUnification).append("\n");
- sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
-
- return sb.toString();
- }
-
- @Override
- public String configFieldPrefix() {
- return "ssl";
- }
-
- @Override
- public String jksKeyStoreFilePath() {
- return jksKeyStoreFilePath;
- }
-
- @Override
- public SSLContext sslContext() {
- return sslContext;
- }
- }
-
- static class TlsQuorumConfig implements TlsConfig {
-
- private final SSLContext sslContext;
- private final String jksKeyStoreFilePath;
-
- TlsQuorumConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
- this.sslContext = sslContext;
- this.jksKeyStoreFilePath = jksKeyStoreFilePath;
- }
-
- String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
- String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION")
- .orElse(config.tlsForQuorumCommunication().name());
- validateOptions(transportSecurityOptions, tlsSetting);
-
- StringBuilder sb = new StringBuilder(createCommonConfig());
- boolean sslQuorum;
- boolean portUnification;
- switch (tlsSetting) {
- case "OFF":
- sslQuorum = false;
- portUnification = false;
- break;
- case "PORT_UNIFICATION":
- sslQuorum = false;
- portUnification = true;
- break;
- case "TLS_WITH_PORT_UNIFICATION":
- sslQuorum = true;
- portUnification = true;
- break;
- case "TLS_ONLY":
- sslQuorum = true;
- portUnification = false;
- break;
- default: throw new IllegalArgumentException("Unknown value of config setting tlsForQuorumCommunication: " + tlsSetting);
- }
- sb.append("sslQuorum=").append(sslQuorum).append("\n");
- sb.append("portUnification=").append(portUnification).append("\n");
- sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
-
- return sb.toString();
- }
-
- @Override
- public String configFieldPrefix() {
- return "ssl.quorum";
- }
-
- @Override
- public String jksKeyStoreFilePath() {
- return jksKeyStoreFilePath;
- }
-
- @Override
- public SSLContext sslContext() {
- return sslContext;
- }
-
- }
-
}
diff --git a/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
deleted file mode 100644
index 86d424bd41b..00000000000
--- a/zookeeper-server/zookeeper-server-3.5.8/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
+++ /dev/null
@@ -1,322 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.zookeeper;
-
-import com.yahoo.cloud.config.ZookeeperServerConfig;
-import com.yahoo.io.IOUtils;
-import com.yahoo.security.KeyUtils;
-import com.yahoo.security.X509CertificateBuilder;
-import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.security.tls.TransportSecurityOptions;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import javax.security.auth.x500.X500Principal;
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.security.KeyPair;
-import java.security.cert.X509Certificate;
-import java.util.Optional;
-
-import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForQuorumCommunication;
-import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForClientServerCommunication;
-import static com.yahoo.security.KeyAlgorithm.EC;
-import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
-import static java.time.Instant.EPOCH;
-import static java.time.temporal.ChronoUnit.DAYS;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests the zookeeper server.
- */
-public class VespaZooKeeperServerImplTest {
-
- private File cfgFile;
- private File idFile;
- private File jksKeyStoreFile;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void setup() throws IOException {
- cfgFile = folder.newFile();
- idFile = folder.newFile();
- jksKeyStoreFile = folder.newFile();
- }
-
- @Test
- public void config_is_written_correctly_when_one_server() throws IOException {
- ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile);
- createServer(builder);
- validateConfigFileSingleHost(cfgFile);
- validateIdFile(idFile, "");
- }
-
- @Test
- public void config_is_written_correctly_when_multiple_servers() throws IOException {
- ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
- builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
- builder.server(newServer(0, "foo", 123, 321));
- builder.server(newServer(1, "bar", 234, 432));
- builder.server(newServer(2, "baz", 345, 543));
- builder.myidFile(idFile.getAbsolutePath());
- builder.myid(1);
- createServer(builder);
- validateConfigFileMultipleHosts(cfgFile);
- validateIdFile(idFile, "1\n");
- }
-
- @Test
- public void config_is_written_correctly_with_tls_for_quorum_communication_port_unification() throws IOException {
- ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile);
- builder.tlsForQuorumCommunication(TlsForQuorumCommunication.PORT_UNIFICATION);
- builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.PORT_UNIFICATION);
- Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
- validateConfigFilePortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
- validateThatJksKeyStoreFileExists(jksKeyStoreFile);
- }
-
- @Test
- public void config_is_written_correctly_with_tls_for_quorum_communication_tls_with_port_unification() throws IOException {
- ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile);
- builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_WITH_PORT_UNIFICATION);
- builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_WITH_PORT_UNIFICATION);
- Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
- validateConfigFileTlsWithPortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
- validateThatJksKeyStoreFileExists(jksKeyStoreFile);
- }
-
- @Test
- public void config_is_written_correctly_with_tls_for_quorum_communication_tls_only() throws IOException {
- ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile);
- builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_ONLY);
- builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_ONLY);
- Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
- validateConfigFileTlsOnly(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
- validateThatJksKeyStoreFileExists(jksKeyStoreFile);
- }
-
- private ZookeeperServerConfig.Builder createConfigBuilderForSingleHost(File cfgFile, File idFile, File jksKeyStoreFile) {
- ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
- builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
- builder.myidFile(idFile.getAbsolutePath());
- builder.server(newServer(0, "foo", 123, 321));
- builder.myid(0);
- builder.jksKeyStoreFile(jksKeyStoreFile.getAbsolutePath());
- return builder;
- }
-
- private void createServer(ZookeeperServerConfig.Builder builder) {
- createServer(builder, Optional.empty());
- }
-
- private void createServer(ZookeeperServerConfig.Builder builder, Optional<TransportSecurityOptions> options) {
- new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false, options);
- }
-
- @Test(expected = RuntimeException.class)
- public void require_that_this_id_must_be_present_amongst_servers() {
- ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
- builder.server(newServer(1, "bar", 234, 432));
- builder.server(newServer(2, "baz", 345, 543));
- builder.myid(0);
- createServer(builder);
- }
-
- @Test
- public void juteMaxBufferCanBeSet() throws IOException {
- ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
- builder.myid(0);
- File idFile = folder.newFile();
- File cfgFile = folder.newFile();
-
- builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost"));
- builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
- builder.myidFile(idFile.getAbsolutePath());
-
- createServer(builder);
- assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer()));
-
- final int max_buffer = 1;
- builder.juteMaxBuffer(max_buffer);
- createServer(builder);
- assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer));
- }
-
- private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) {
- ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder();
- builder.id(id);
- builder.hostname(hostName);
- builder.electionPort(electionPort);
- builder.quorumPort(quorumPort);
- return builder;
- }
-
- private void validateIdFile(File idFile, String expected) throws IOException {
- String actual = IOUtils.readFile(idFile);
- assertThat(actual, is(expected));
- }
-
- private String commonConfig() {
- return "tickTime=2000\n" +
- "initLimit=20\n" +
- "syncLimit=15\n" +
- "maxClientCnxns=0\n" +
- "snapCount=50000\n" +
- "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" +
- "clientPort=2181\n" +
- "secureClientPort=2184\n" +
- "autopurge.purgeInterval=1\n" +
- "autopurge.snapRetainCount=15\n" +
- "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
- "admin.enableServer=false\n" +
- "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" +
- "quorumListenOnAllIPs=true\n";
- }
-
- private String quorumKeyStoreAndTrustStoreConfig(File jksKeyStoreFilePath, File caCertificatesFilePath) {
- StringBuilder sb = new StringBuilder();
-
- sb.append("ssl.quorum.keyStore.location=").append(jksKeyStoreFilePath.getAbsolutePath()).append("\n");
- sb.append("ssl.quorum.keyStore.type=JKS\n");
- sb.append("ssl.quorum.trustStore.location=").append(caCertificatesFilePath.getAbsolutePath()).append("\n");
- sb.append("ssl.quorum.trustStore.type=PEM\n");
- return sb.toString();
- }
-
- private String clientServerKeyStoreAndTrustStoreConfig(File jksKeyStoreFilePath, File caCertificatesFilePath) {
- StringBuilder sb = new StringBuilder();
-
- sb.append("ssl.keyStore.location=").append(jksKeyStoreFilePath.getAbsolutePath()).append("\n");
- sb.append("ssl.keyStore.type=JKS\n");
- sb.append("ssl.trustStore.location=").append(caCertificatesFilePath.getAbsolutePath()).append("\n");
- sb.append("ssl.trustStore.type=PEM\n");
- return sb.toString();
- }
-
- private void validateConfigFileSingleHost(File cfgFile) throws IOException {
- String expected =
- commonConfig() +
- "server.0=foo:321:123\n" +
- commonTlsQuorumConfig() +
- "sslQuorum=false\n" +
- "portUnification=false\n" +
- commonTlsClientServerConfig() +
- "client.portUnification=false\n";
- validateConfigFile(cfgFile, expected);
- }
-
- private String commonTlsQuorumConfig() {
- return "ssl.quorum.hostnameVerification=false\n" +
- "ssl.quorum.clientAuth=NEED\n" +
- "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
- "ssl.quorum.enabledProtocols=TLSv1.2\n" +
- "ssl.quorum.protocol=TLS\n";
- }
-
- private String commonTlsClientServerConfig() {
- return "ssl.hostnameVerification=false\n" +
- "ssl.clientAuth=NEED\n" +
- "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
- "ssl.enabledProtocols=TLSv1.2\n" +
- "ssl.protocol=TLS\n";
- }
-
- private void validateConfigFileMultipleHosts(File cfgFile) throws IOException {
- String expected =
- commonConfig() +
- "server.0=foo:321:123\n" +
- "server.1=bar:432:234\n" +
- "server.2=baz:543:345\n" +
- commonTlsQuorumConfig() +
- "sslQuorum=false\n" +
- "portUnification=false\n" +
- commonTlsClientServerConfig() +
- "client.portUnification=false\n";
- validateConfigFile(cfgFile, expected);
- }
-
- private void validateConfigFilePortUnification(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException {
- String expected =
- commonConfig() +
- "server.0=foo:321:123\n" +
- commonTlsQuorumConfig() +
- "sslQuorum=false\n" +
- "portUnification=true\n" +
- quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) +
- commonTlsClientServerConfig() +
- "client.portUnification=true\n" +
- clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile);
- validateConfigFile(cfgFile, expected);
- }
-
- private void validateConfigFileTlsWithPortUnification(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException {
- String expected =
- commonConfig() +
- "server.0=foo:321:123\n" +
- commonTlsQuorumConfig() +
- "sslQuorum=true\n" +
- "portUnification=true\n" +
- quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) +
- commonTlsClientServerConfig() +
- "client.portUnification=true\n" +
- clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile);
- validateConfigFile(cfgFile, expected);
- }
-
- private void validateConfigFileTlsOnly(File cfgFile, File jksKeyStoreFile, File caCertificatesFile) throws IOException {
- String expected =
- commonConfig() +
- "server.0=foo:321:123\n" +
- commonTlsQuorumConfig() +
- "sslQuorum=true\n" +
- "portUnification=false\n" +
- quorumKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile) +
- commonTlsClientServerConfig() +
- "client.portUnification=false\n" +
- clientServerKeyStoreAndTrustStoreConfig(jksKeyStoreFile, caCertificatesFile);
- validateConfigFile(cfgFile, expected);
- }
-
- private void validateConfigFile(File cfgFile, String expected) throws IOException {
- String actual = IOUtils.readFile(cfgFile);
- assertThat(actual, is(expected));
- }
-
- private void validateThatJksKeyStoreFileExists(File cfgFile) {
- assertTrue(cfgFile.exists() && cfgFile.canRead());
- }
-
- private Optional<TransportSecurityOptions> createTransportSecurityOptions() throws IOException {
- KeyPair keyPair = KeyUtils.generateKeypair(EC);
- Path privateKeyFile = folder.newFile().toPath();
- Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
-
- X509Certificate certificate = X509CertificateBuilder
- .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, EPOCH.plus(1, DAYS), SHA256_WITH_ECDSA, BigInteger.ONE)
- .build();
- Path certificateChainFile = folder.newFile().toPath();
- String certificatePem = X509CertificateUtils.toPem(certificate);
- Files.writeString(certificateChainFile, certificatePem);
-
- Path caCertificatesFile = folder.newFile().toPath();
- Files.writeString(caCertificatesFile, certificatePem);
-
- return Optional.of(new TransportSecurityOptions.Builder()
- .withCertificates(certificateChainFile, privateKeyFile)
- .withCaCertificates(caCertificatesFile)
- .build());
- }
-
-}
diff --git a/zookeeper-server/zookeeper-server-common/pom.xml b/zookeeper-server/zookeeper-server-common/pom.xml
index 66e5bc2075c..b05136cd151 100644
--- a/zookeeper-server/zookeeper-server-common/pom.xml
+++ b/zookeeper-server/zookeeper-server-common/pom.xml
@@ -11,6 +11,13 @@
<artifactId>zookeeper-server-common</artifactId>
<packaging>container-plugin</packaging>
<version>7-SNAPSHOT</version>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
<build>
<plugins>
<plugin>
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
new file mode 100644
index 00000000000..119509dd9cc
--- /dev/null
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
@@ -0,0 +1,307 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.zookeeper;
+
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.security.KeyStoreBuilder;
+import com.yahoo.security.KeyStoreType;
+import com.yahoo.security.KeyStoreUtils;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SslContextBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.security.tls.TlsContext;
+import com.yahoo.security.tls.TransportSecurityOptions;
+import com.yahoo.text.Utf8;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+
+public class Configurator {
+ private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable";
+ static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer";
+
+ private final ZookeeperServerConfig zookeeperServerConfig;
+ private final String configFilePath;
+ private final String jksKeyStoreFilePath;
+
+ public Configurator(ZookeeperServerConfig zookeeperServerConfig) {
+ this.zookeeperServerConfig = zookeeperServerConfig;
+ this.configFilePath = zookeeperServerConfig.zooKeeperConfigFile();
+ this.jksKeyStoreFilePath = zookeeperServerConfig.jksKeyStoreFile();
+ System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
+ System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString());
+ System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString());
+ }
+
+ void writeConfigToDisk(Optional<TransportSecurityOptions> transportSecurityOptions) {
+ new File(configFilePath).getParentFile().mkdirs();
+
+ try {
+ writeZooKeeperConfigFile(zookeeperServerConfig, transportSecurityOptions);
+ writeMyIdFile(zookeeperServerConfig);
+ transportSecurityOptions.ifPresent(this::writeJksKeystore);
+ } catch (IOException e) {
+ throw new RuntimeException("Error writing zookeeper config", e);
+ }
+ }
+
+ private void writeZooKeeperConfigFile(ZookeeperServerConfig config,
+ Optional<TransportSecurityOptions> transportSecurityOptions) throws IOException {
+ try (FileWriter writer = new FileWriter(configFilePath)) {
+ writer.write(transformConfigToString(config, transportSecurityOptions));
+ }
+ }
+
+ private String transformConfigToString(ZookeeperServerConfig config,
+ Optional<TransportSecurityOptions> transportSecurityOptions) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("tickTime=").append(config.tickTime()).append("\n");
+ sb.append("initLimit=").append(config.initLimit()).append("\n");
+ sb.append("syncLimit=").append(config.syncLimit()).append("\n");
+ sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n");
+ sb.append("snapCount=").append(config.snapshotCount()).append("\n");
+ sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n");
+ sb.append("clientPort=").append(config.clientPort()).append("\n");
+ sb.append("secureClientPort=").append(config.secureClientPort()).append("\n");
+ sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
+ sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
+ // See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands
+ // Includes all available commands in 3.5, except 'wchc' and 'wchp'
+ sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
+ sb.append("admin.enableServer=false").append("\n");
+ // Need NettyServerCnxnFactory to be able to use TLS for communication
+ sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n");
+ sb.append("quorumListenOnAllIPs=true").append("\n");
+ ensureThisServerIsRepresented(config.myid(), config.server());
+ config.server().forEach(server -> addServerToCfg(sb, server));
+ SSLContext sslContext = new SslContextBuilder().build();
+ sb.append(new TlsQuorumConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
+ sb.append(new TlsClientServerConfig(sslContext, jksKeyStoreFilePath).createConfig(config, transportSecurityOptions));
+ return sb.toString();
+ }
+
+ private void writeMyIdFile(ZookeeperServerConfig config) throws IOException {
+ if (config.server().size() > 1) {
+ try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) {
+ writer.write(config.myid() + "\n");
+ }
+ }
+ }
+
+ private void writeJksKeystore(TransportSecurityOptions options) {
+ Path privateKeyFile = options.getPrivateKeyFile().orElseThrow(() -> new RuntimeException("Could not find private key file"));
+ Path certificatesFile = options.getCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find certificates file"));
+
+ PrivateKey privateKey;
+ List<X509Certificate> certificates;
+ try {
+ privateKey = KeyUtils.fromPemEncodedPrivateKey(Utf8.toString(Files.readAllBytes(privateKeyFile)));
+ certificates = X509CertificateUtils.certificateListFromPem(Utf8.toString(Files.readAllBytes(certificatesFile)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ KeyStoreBuilder keyStoreBuilder = KeyStoreBuilder
+ .withType(KeyStoreType.JKS)
+ .withKeyEntry("foo", privateKey, certificates);
+
+ KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), Paths.get(jksKeyStoreFilePath));
+ }
+
+ private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
+ boolean found = false;
+ for (ZookeeperServerConfig.Server server : servers) {
+ if (myid == server.id()) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")");
+ }
+ }
+
+ private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) {
+ sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n");
+ }
+
+
+
+
+ static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) {
+ return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet());
+ }
+
+ private interface TlsConfig {
+ default Set<String> allowedCiphers(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedCipherSuites(sslContext)); }
+
+ default Set<String> allowedProtocols(SSLContext sslContext) { return new TreeSet<>(TlsContext.getAllowedProtocols(sslContext)); }
+
+ default Optional<String> getEnvironmentVariable(String variableName) {
+ return Optional.ofNullable(System.getenv().get(variableName))
+ .filter(var -> !var.isEmpty());
+ }
+
+ default void validateOptions(Optional<TransportSecurityOptions> transportSecurityOptions, String tlsSetting) {
+ if (transportSecurityOptions.isEmpty() && !tlsSetting.equals("OFF"))
+ throw new RuntimeException("Could not retrieve transport security options");
+ }
+
+ String configFieldPrefix();
+
+ String jksKeyStoreFilePath();
+
+ SSLContext sslContext();
+
+ default String createCommonKeyStoreTrustStoreOptions(Optional<TransportSecurityOptions> transportSecurityOptions) {
+ StringBuilder sb = new StringBuilder();
+ transportSecurityOptions.ifPresent(options -> {
+ sb.append(configFieldPrefix()).append(".keyStore.location=").append(jksKeyStoreFilePath()).append("\n");
+ sb.append(configFieldPrefix()).append(".keyStore.type=JKS\n");
+
+ Path caCertificatesFile = options.getCaCertificatesFile().orElseThrow(() -> new RuntimeException("Could not find ca certificates file"));
+ sb.append(configFieldPrefix()).append(".trustStore.location=").append(caCertificatesFile).append("\n");
+ sb.append(configFieldPrefix()).append(".trustStore.type=PEM\n");
+ });
+ return sb.toString();
+ }
+
+ default String createCommonConfig() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(configFieldPrefix()).append(".hostnameVerification=false\n");
+ sb.append(configFieldPrefix()).append(".clientAuth=NEED\n");
+ sb.append(configFieldPrefix()).append(".ciphersuites=").append(String.join(",", allowedCiphers(sslContext()))).append("\n");
+ sb.append(configFieldPrefix()).append(".enabledProtocols=").append(String.join(",", allowedProtocols(sslContext()))).append("\n");
+ sb.append(configFieldPrefix()).append(".protocol=").append(sslContext().getProtocol()).append("\n");
+
+ return sb.toString();
+ }
+
+ }
+
+ static class TlsClientServerConfig implements TlsConfig {
+
+ private final SSLContext sslContext;
+ private final String jksKeyStoreFilePath;
+
+ TlsClientServerConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
+ this.sslContext = sslContext;
+ this.jksKeyStoreFilePath = jksKeyStoreFilePath;
+ }
+
+ String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
+ String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION")
+ .orElse(config.tlsForClientServerCommunication().name());
+ validateOptions(transportSecurityOptions, tlsSetting);
+
+ StringBuilder sb = new StringBuilder(createCommonConfig());
+ boolean portUnification;
+ switch (tlsSetting) {
+ case "OFF":
+ case "TLS_ONLY":
+ portUnification = false;
+ break;
+ case "PORT_UNIFICATION":
+ case "TLS_WITH_PORT_UNIFICATION":
+ portUnification = true;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown value of config setting tlsForClientServerCommunication: " + tlsSetting);
+ }
+ sb.append("client.portUnification=").append(portUnification).append("\n");
+ sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
+
+ return sb.toString();
+ }
+
+ @Override
+ public String configFieldPrefix() {
+ return "ssl";
+ }
+
+ @Override
+ public String jksKeyStoreFilePath() {
+ return jksKeyStoreFilePath;
+ }
+
+ @Override
+ public SSLContext sslContext() {
+ return sslContext;
+ }
+ }
+
+ static class TlsQuorumConfig implements TlsConfig {
+
+ private final SSLContext sslContext;
+ private final String jksKeyStoreFilePath;
+
+ TlsQuorumConfig(SSLContext sslContext, String jksKeyStoreFilePath) {
+ this.sslContext = sslContext;
+ this.jksKeyStoreFilePath = jksKeyStoreFilePath;
+ }
+
+ String createConfig(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) {
+ String tlsSetting = getEnvironmentVariable("VESPA_TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION")
+ .orElse(config.tlsForQuorumCommunication().name());
+ validateOptions(transportSecurityOptions, tlsSetting);
+
+ StringBuilder sb = new StringBuilder(createCommonConfig());
+ boolean sslQuorum;
+ boolean portUnification;
+ switch (tlsSetting) {
+ case "OFF":
+ sslQuorum = false;
+ portUnification = false;
+ break;
+ case "PORT_UNIFICATION":
+ sslQuorum = false;
+ portUnification = true;
+ break;
+ case "TLS_WITH_PORT_UNIFICATION":
+ sslQuorum = true;
+ portUnification = true;
+ break;
+ case "TLS_ONLY":
+ sslQuorum = true;
+ portUnification = false;
+ break;
+ default: throw new IllegalArgumentException("Unknown value of config setting tlsForQuorumCommunication: " + tlsSetting);
+ }
+ sb.append("sslQuorum=").append(sslQuorum).append("\n");
+ sb.append("portUnification=").append(portUnification).append("\n");
+ sb.append(createCommonKeyStoreTrustStoreOptions(transportSecurityOptions));
+
+ return sb.toString();
+ }
+
+ @Override
+ public String configFieldPrefix() {
+ return "ssl.quorum";
+ }
+
+ @Override
+ public String jksKeyStoreFilePath() {
+ return jksKeyStoreFilePath;
+ }
+
+ @Override
+ public SSLContext sslContext() {
+ return sslContext;
+ }
+
+ }
+
+}
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java
index 72351244cce..fb104f7aeee 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
+++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java
@@ -26,6 +26,7 @@ import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForQuorumCommunica
import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForClientServerCommunication;
import static com.yahoo.security.KeyAlgorithm.EC;
import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
+import static com.yahoo.vespa.zookeeper.Configurator.ZOOKEEPER_JUTE_MAX_BUFFER;
import static java.time.Instant.EPOCH;
import static java.time.temporal.ChronoUnit.DAYS;
import static org.hamcrest.CoreMatchers.is;
@@ -36,7 +37,7 @@ import static org.junit.Assert.assertTrue;
/**
* Tests the zookeeper server.
*/
-public class VespaZooKeeperServerImplTest {
+public class ConfiguratorTest {
private File cfgFile;
private File idFile;
@@ -55,7 +56,7 @@ public class VespaZooKeeperServerImplTest {
@Test
public void config_is_written_correctly_when_one_server() throws IOException {
ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile);
- createServer(builder);
+ new Configurator(builder.build()).writeConfigToDisk(Optional.empty());
validateConfigFileSingleHost(cfgFile);
validateIdFile(idFile, "");
}
@@ -69,7 +70,7 @@ public class VespaZooKeeperServerImplTest {
builder.server(newServer(2, "baz", 345, 543));
builder.myidFile(idFile.getAbsolutePath());
builder.myid(1);
- createServer(builder);
+ new Configurator(builder.build()).writeConfigToDisk(Optional.empty());
validateConfigFileMultipleHosts(cfgFile);
validateIdFile(idFile, "1\n");
}
@@ -80,7 +81,7 @@ public class VespaZooKeeperServerImplTest {
builder.tlsForQuorumCommunication(TlsForQuorumCommunication.PORT_UNIFICATION);
builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.PORT_UNIFICATION);
Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
+ new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions);
validateConfigFilePortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
validateThatJksKeyStoreFileExists(jksKeyStoreFile);
}
@@ -91,7 +92,7 @@ public class VespaZooKeeperServerImplTest {
builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_WITH_PORT_UNIFICATION);
builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_WITH_PORT_UNIFICATION);
Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
+ new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions);
validateConfigFileTlsWithPortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
validateThatJksKeyStoreFileExists(jksKeyStoreFile);
}
@@ -102,7 +103,7 @@ public class VespaZooKeeperServerImplTest {
builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_ONLY);
builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_ONLY);
Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions();
- createServer(builder, transportSecurityOptions);
+ new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions);
validateConfigFileTlsOnly(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile());
validateThatJksKeyStoreFileExists(jksKeyStoreFile);
}
@@ -117,21 +118,13 @@ public class VespaZooKeeperServerImplTest {
return builder;
}
- private void createServer(ZookeeperServerConfig.Builder builder) {
- createServer(builder, Optional.empty());
- }
-
- private void createServer(ZookeeperServerConfig.Builder builder, Optional<TransportSecurityOptions> options) {
- new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false, options);
- }
-
@Test(expected = RuntimeException.class)
public void require_that_this_id_must_be_present_amongst_servers() {
ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
builder.server(newServer(1, "bar", 234, 432));
builder.server(newServer(2, "baz", 345, 543));
builder.myid(0);
- createServer(builder);
+ new Configurator(builder.build()).writeConfigToDisk(Optional.empty());
}
@Test
@@ -145,13 +138,13 @@ public class VespaZooKeeperServerImplTest {
builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
builder.myidFile(idFile.getAbsolutePath());
- createServer(builder);
- assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer()));
+ new Configurator(builder.build()).writeConfigToDisk(Optional.empty());
+ assertThat(System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer()));
final int max_buffer = 1;
builder.juteMaxBuffer(max_buffer);
- createServer(builder);
- assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer));
+ new Configurator(builder.build()).writeConfigToDisk(Optional.empty());
+ assertThat(System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer));
}
private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) {