diff options
16 files changed, 250 insertions, 31 deletions
diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 8d62acb0fb4..02f61517083 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -79,6 +79,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <!-- Not used by this module, but compilation fails without it because zookeeper uses these annotations. Provided scoped here to avoid dependents getting it transitively. --> <groupId>com.github.spotbugs</groupId> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 4cdbb49dedc..5e0d050cbcf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -1,35 +1,35 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.database; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.AnnotatedClusterState; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.rpc.EnvelopedClusterStateBundleCodec; import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.StringTokenizer; import java.util.TreeMap; import java.util.logging.Level; -import com.yahoo.vdslib.state.NodeState; -import com.yahoo.vdslib.state.State; -import com.yahoo.vdslib.state.Node; - import java.util.logging.Logger; -import java.io.IOException; -import java.io.StringWriter; -import java.io.PrintWriter; -import java.nio.charset.Charset; public class ZooKeeperDatabase extends Database { @@ -108,7 +108,7 @@ public class ZooKeeperDatabase extends Database { public ZooKeeperDatabase(ContentCluster cluster, int nodeIndex, String address, int timeout, Database.DatabaseListener zksl) throws IOException, KeeperException, InterruptedException { this.nodeIndex = nodeIndex; zooKeeperRoot = "/vespa/fleetcontroller/" + cluster.getName() + "/"; - session = new ZooKeeper(address, timeout, watcher); + session = new ZooKeeper(address, timeout, watcher, new ZkClientConfigBuilder().toConfig()); boolean completedOk = false; try{ this.listener = zksl; @@ -149,6 +149,7 @@ <module>vsm</module> <module>yolean</module> <module>zkfacade</module> + <module>zookeeper-client-common</module> <module>zookeeper-command-line-client</module> <module>zookeeper-server</module> </modules> diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index d9bd377ffa1..7563ef068a9 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -42,6 +42,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> </dependency> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 4cbb6c95cb4..9a6ef3f74f1 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -4,14 +4,12 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.api.VespaCurator; import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.zookeeper.VespaSslContextProvider; import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -30,6 +28,8 @@ import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -124,20 +124,12 @@ public class Curator implements VespaCurator, AutoCloseable { private static ZKClientConfig createClientConfig(Optional<File> clientConfigFile) { if (clientConfigFile.isPresent()) { - boolean useSecureClient = Boolean.parseBoolean(getEnvironmentVariable("VESPA_USE_TLS_FOR_ZOOKEEPER_CLIENT").orElse("false")); - StringBuilder configBuilder = new StringBuilder("zookeeper.client.secure=").append(useSecureClient).append("\n"); - if (useSecureClient) { - configBuilder.append("zookeeper.ssl.context.supplier.class=").append(VespaSslContextProvider.class.getName()).append("\n") - .append("zookeeper.ssl.enabledProtocols=").append(VespaSslContextProvider.enabledTlsProtocolConfigValue()).append("\n") - .append("zookeeper.ssl.ciphersuites=").append(VespaSslContextProvider.enabledTlsCiphersConfigValue()).append("\n") - .append("zookeeper.ssl.clientAuth=NEED\n"); - } - clientConfigFile.get().getParentFile().mkdirs(); - IOUtils.writeFile(clientConfigFile.get(), Utf8.toBytes(configBuilder.toString())); try { - return new ZKClientConfig(clientConfigFile.get()); + return new ZkClientConfigBuilder().toConfig(clientConfigFile.get().toPath()); } catch (QuorumPeerConfig.ConfigException e) { throw new RuntimeException("Unable to create ZooKeeper client config file " + clientConfigFile.get()); + } catch (IOException e) { + throw new UncheckedIOException(e); } } else { return new ZKClientConfig(); @@ -406,9 +398,4 @@ public class Curator implements VespaCurator, AutoCloseable { */ public int zooKeeperEnsembleCount() { return connectionSpec.ensembleSize(); } - private static Optional<String> getEnvironmentVariable(String variableName) { - return Optional.ofNullable(System.getenv().get(variableName)) - .filter(var -> !var.isEmpty()); - } - } diff --git a/zkfacade/src/main/java/org/apache/zookeeper/client/package-info.java b/zkfacade/src/main/java/org/apache/zookeeper/client/package-info.java new file mode 100644 index 00000000000..e8c8623647a --- /dev/null +++ b/zkfacade/src/main/java/org/apache/zookeeper/client/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Required for export of {@link org.apache.zookeeper.client.ZKClientConfig}. + * + * @author bjorncs + */ +@ExportPackage +package org.apache.zookeeper.client; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/zkfacade/src/main/java/org/apache/zookeeper/common/package-info.java b/zkfacade/src/main/java/org/apache/zookeeper/common/package-info.java new file mode 100644 index 00000000000..3da12af57ec --- /dev/null +++ b/zkfacade/src/main/java/org/apache/zookeeper/common/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Required for export of {@link org.apache.zookeeper.client.ZKClientConfig} (which inherits from {@link org.apache.zookeeper.common.ZKConfig}). + * + * @author bjorncs + */ +@ExportPackage +package org.apache.zookeeper.common; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/zkfacade/src/main/java/org/apache/zookeeper/server/quorum/package-info.java b/zkfacade/src/main/java/org/apache/zookeeper/server/quorum/package-info.java new file mode 100644 index 00000000000..833193c52bb --- /dev/null +++ b/zkfacade/src/main/java/org/apache/zookeeper/server/quorum/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Required for export of {@link org.apache.zookeeper.client.ZKClientConfig} (which inherits from {@link org.apache.zookeeper.common.ZKConfig}). + * + * @author bjorncs + */ +@ExportPackage +package org.apache.zookeeper.server.quorum; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/zookeeper-client-common/OWNERS b/zookeeper-client-common/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/zookeeper-client-common/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/zookeeper-client-common/README.md b/zookeeper-client-common/README.md new file mode 100644 index 00000000000..51c757a8af2 --- /dev/null +++ b/zookeeper-client-common/README.md @@ -0,0 +1,3 @@ +# zookeeper-client-common + +Shared client configuration logic for ZooKeeper clients diff --git a/zookeeper-client-common/pom.xml b/zookeeper-client-common/pom.xml new file mode 100644 index 00000000000..fd799e13fca --- /dev/null +++ b/zookeeper-client-common/pom.xml @@ -0,0 +1,47 @@ +<?xml version="1.0"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>zookeeper-client-common</artifactId> + <packaging>jar</packaging> + <version>7-SNAPSHOT</version> + + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.client.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile scope --> + <!-- test scope --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/VespaSslContextProvider.java b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/VespaSslContextProvider.java new file mode 100644 index 00000000000..209e08db6cc --- /dev/null +++ b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/VespaSslContextProvider.java @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper.client; + +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.TransportSecurityUtils; + +import javax.net.ssl.SSLContext; +import java.util.function.Supplier; + +/** + * Provider for Vespa {@link SSLContext} instance to Zookeeper + misc utility methods for providing Vespa TLS specific ZK configuration. + * + * @author bjorncs + */ +public class VespaSslContextProvider implements Supplier<SSLContext> { + + private static final SSLContext sslContext = TransportSecurityUtils.getSystemTlsContext().map(TlsContext::context).orElse(null); + + @Override + public SSLContext get() { + if (sslContext == null) throw new IllegalStateException("Vespa TLS is not enabled"); + return sslContext; + } + +} diff --git a/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java new file mode 100644 index 00000000000..af75bd7aa5c --- /dev/null +++ b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java @@ -0,0 +1,82 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper.client; + +import com.yahoo.security.tls.MixedMode; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.TransportSecurityUtils; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Builder for ZK client configuration + * + * @author bjorncs + */ +public class ZkClientConfigBuilder { + + public static final String CLIENT_SECURE_PROPERTY = "zookeeper.client.secure"; + public static final String SSL_CONTEXT_SUPPLIER_CLASS_PROPERTY = "zookeeper.ssl.context.supplier.class"; + public static final String SSL_ENABLED_PROTOCOLS_PROPERTY = "zookeeper.ssl.enabledProtocols"; + public static final String SSL_ENABLED_CIPHERSUITES_PROPERTY = "zookeeper.ssl.ciphersuites"; + public static final String SSL_CLIENTAUTH_PROPERTY = "zookeeper.ssl.clientAuth"; + + private static final TlsContext tlsContext = getTlsContext().orElse(null); + + public ZkClientConfigBuilder() {} + + public ZKClientConfig toConfig(Path configFile) throws IOException, QuorumPeerConfig.ConfigException { + String configString = toConfigString(); + Files.createDirectories(configFile.getParent()); + Path tempFile = configFile.resolveSibling("." + configFile.getFileName() + ".tmp"); + Files.writeString(tempFile, configString); + Files.move(tempFile, configFile, StandardCopyOption.ATOMIC_MOVE); + return new ZKClientConfig(configFile.toString()); + } + + public ZKClientConfig toConfig() { + ZKClientConfig config = new ZKClientConfig(); + Map<String, String> configProperties = toConfigProperties(); + configProperties.forEach(config::setProperty); + return config; + } + + public String toConfigString() { + StringBuilder builder = new StringBuilder(); + Map<String, String> properties = toConfigProperties(); + properties.forEach((key, value) -> builder.append(key).append('=').append(value).append('\n')); + return builder.toString(); + } + + public Map<String, String> toConfigProperties() { + Map<String, String> builder = new HashMap<>(); + builder.put(CLIENT_SECURE_PROPERTY, Boolean.toString(tlsContext != null)); + if (tlsContext != null) { + builder.put(SSL_CONTEXT_SUPPLIER_CLASS_PROPERTY, VespaSslContextProvider.class.getName()); + String protocolsConfigValue = Arrays.stream(tlsContext.parameters().getProtocols()).sorted().collect(Collectors.joining(",")); + builder.put(SSL_ENABLED_PROTOCOLS_PROPERTY, protocolsConfigValue); + String ciphersConfigValue = Arrays.stream(tlsContext.parameters().getCipherSuites()).sorted().collect(Collectors.joining(",")); + builder.put(SSL_ENABLED_CIPHERSUITES_PROPERTY, ciphersConfigValue); + builder.put(SSL_CLIENTAUTH_PROPERTY, "NEED"); + } + return Map.copyOf(builder); + } + + private static Optional<TlsContext> getTlsContext() { + // TODO(bjorncs) Remove handling of temporary feature flag + boolean temporaryFeatureFlag = Optional.ofNullable(System.getenv("VESPA_USE_TLS_FOR_ZOOKEEPER_CLIENT")).map(Boolean::parseBoolean).orElse(false); + if (!temporaryFeatureFlag) return Optional.empty(); + + if (TransportSecurityUtils.getInsecureMixedMode() == MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) return Optional.empty(); + return TransportSecurityUtils.getSystemTlsContext(); + } +} diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index 748a156b30c..c186b377eb6 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -17,6 +17,18 @@ <version>${zookeeper.client.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <!-- Needed by vespa-zkcli --> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> @@ -34,7 +46,7 @@ <configuration> <archive> <manifest> - <mainClass>org.apache.zookeeper.ZooKeeperMain</mainClass> + <mainClass>com.yahoo.vespa.zookeeper.cli.Main</mainClass> </manifest> </archive> <descriptorRefs> diff --git a/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java new file mode 100644 index 00000000000..1bc1a21a237 --- /dev/null +++ b/zookeeper-command-line-client/src/main/java/com/yahoo/vespa/zookeeper/cli/Main.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper.cli; + +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.ZooKeeperMain; + +import java.io.IOException; + +/** + * @author bjorncs + */ +public class Main { + public static void main(String[] args) throws IOException, InterruptedException { + new ZkClientConfigBuilder() + .toConfigProperties() + .forEach(System::setProperty); + ZooKeeperMain.main(args); + } +} diff --git a/zookeeper-command-line-client/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli index 5c29f9bb084..1a5858b3222 100755 --- a/zookeeper-command-line-client/src/main/sh/vespa-zkcli +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli @@ -95,4 +95,4 @@ done $sudo java \ -cp $VESPA_HOME/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar \ -Dlog4j.configuration="log4j-vespa.properties" -Xms32m -Xmx512m \ - org.apache.zookeeper.ZooKeeperMain "$@" + com.yahoo.vespa.zookeeper.cli.Main "$@" |