diff options
Diffstat (limited to 'zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo')
6 files changed, 348 insertions, 0 deletions
diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java new file mode 100644 index 00000000000..d986f02d89a --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java @@ -0,0 +1,43 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import java.nio.file.Path; + +/** + * + * Server used for starting config server, needed to be able to have different behavior for hosted and + * self-hosted Vespa (controlled by zookeeperServerConfig.dynamicReconfiguration). + * + * @author Harald Musum + */ +public class ConfigServerZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { + + private final VespaZooKeeperServer zooKeeperServer; + + @Inject + public ConfigServerZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig) { + this.zooKeeperServer = zookeeperServerConfig.dynamicReconfiguration() + ? new ReconfigurableVespaZooKeeperServer(new Reconfigurer(new VespaZooKeeperAdminImpl()), zookeeperServerConfig) + : new VespaZooKeeperServerImpl(zookeeperServerConfig); + } + + @Override + public void deconstruct() { zooKeeperServer.shutdown(); } + + @Override + public void shutdown() { + zooKeeperServer.shutdown(); + } + + @Override + public void start(Path configFilePath) { + zooKeeperServer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { return zooKeeperServer.reconfigurable(); } + +} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java new file mode 100644 index 00000000000..1b469beb1b8 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -0,0 +1,47 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import ai.vespa.validation.Validation; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import java.nio.file.Path; +import java.time.Duration; + +/** + * Starts or reconfigures zookeeper cluster. + * The QuorumPeer conditionally created here is owned by the Reconfigurer; + * when it already has a peer, that peer is used here in case start or shutdown is required. + * Guarantees that server is up by writing a node to ZooKeeper successfully before + * returning from constructor. + * + * @author hmusum + */ +public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { + + private QuorumPeer peer; + + @Inject + public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { + Validation.require(zookeeperServerConfig.dynamicReconfiguration(), + zookeeperServerConfig.dynamicReconfiguration(), + "dynamicReconfiguration must be true"); + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return true; + } + +} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java new file mode 100644 index 00000000000..68f7459530e --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java @@ -0,0 +1,51 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.security.X509SslContext; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.auth.AuthenticationProvider; +import org.apache.zookeeper.server.auth.X509AuthenticationProvider; + +import javax.net.ssl.X509KeyManager; +import javax.net.ssl.X509TrustManager; +import java.security.cert.X509Certificate; +import java.util.logging.Logger; + +/** + * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS + * + * @author bjorncs + */ +public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { + + private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); + + public VespaMtlsAuthenticationProvider() { + super(trustManager(), keyManager()); + } + + private static X509KeyManager keyManager() { + return new VespaSslContextProvider().tlsContext().map(X509SslContext::keyManager).orElse(null); + } + + private static X509TrustManager trustManager() { + return new VespaSslContextProvider().tlsContext().map(X509SslContext::trustManager).orElse(null); + } + + @Override + public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { + // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. + // The client is authorized once the SSL handshake has completed. + X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); + if (certificateChain == null || certificateChain.length == 0) { + log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); + return KeeperException.Code.AUTHFAILED; + } + X509Certificate certificate = certificateChain[0]; + cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); + return KeeperException.Code.OK; + } + +} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java new file mode 100644 index 00000000000..dd5ac4e252b --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java @@ -0,0 +1,60 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.protect.Process; +import org.apache.zookeeper.server.admin.AdminServer; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Starts or stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps + * exceptions so that it can be used by code that does not depend on ZooKeeper. + * + * @author hmusum + */ +class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); + + @Override + public void start(Path path) { + initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); + } + + @Override + public void shutdown(Duration timeout) { + if (quorumPeer != null) { + log.log(Level.FINE, "Shutting down ZooKeeper server"); + try { + quorumPeer.shutdown(); + quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete + if (quorumPeer.isAlive()) + throw new IllegalStateException("Peer still alive after " + timeout); + } catch (RuntimeException | InterruptedException e) { + // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. + // + // When a VespaZooKeeperServer component receives a new config, the container will try to start a new + // server with the new config, this will fail until the old server is deconstructed. If the old server + // fails to deconstruct/shutdown, the new one will never start and if that happens forcing a restart is + // the better option. + Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); + } + } + } + + @Override + protected void initializeAndRun(String[] args) { + try { + super.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { + throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java new file mode 100644 index 00000000000..1f15c758583 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -0,0 +1,93 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hmusum + */ +@SuppressWarnings("unused") // Created by injection +public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); + + @Override + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { + long fromConfig = -1; + // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); + log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private ZooKeeperAdmin createAdmin(String connectionSpec) { + return uncheck(() -> new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), + (event) -> log.log(Level.FINE, event.toString()), new ZkClientConfigBuilder().toConfig())); + } + + /** Creates a node in zookeeper, with hostname as part of node name, this ensures that server is up and working before returning */ + void createDummyNode(ZookeeperServerConfig zookeeperServerConfig) { + int sleepTime = 2_000; + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(localConnectionSpec(zookeeperServerConfig))) { + Instant end = Instant.now().plus(Duration.ofMinutes(5)); + Exception exception = null; + do { + try { + zooKeeperAdmin.create("/dummy-node-" + HostName.getLocalhost(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + return; + } catch (KeeperException e) { + if (e instanceof KeeperException.NodeExistsException) { + try { + zooKeeperAdmin.setData("/dummy-node-" + HostName.getLocalhost(), new byte[0], -1); + return; + } catch (KeeperException ex) { + log.log(Level.FINE, e.getMessage()); + Thread.sleep(sleepTime); + continue; + } + } + log.log(Level.FINE, e.getMessage()); + exception = e; + Thread.sleep(sleepTime); + } + } while (Instant.now().isBefore(end)); + throw new RuntimeException("Unable to create dummy node: ", exception); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} + diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..4a7f85d6985 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,54 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import ai.vespa.validation.Validation; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import java.nio.file.Path; +import java.time.Duration; + +/** + * ZooKeeper server. Guarantees that the server is up by writing a node to ZooKeeper successfully before + * returning from constructor. + * + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { + + private final VespaQuorumPeer peer; + private final ZooKeeperRunner runner; + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + Validation.require(! zookeeperServerConfig.dynamicReconfiguration(), + ! zookeeperServerConfig.dynamicReconfiguration(), + "dynamicReconfiguration must be false"); + this.peer = new VespaQuorumPeer(); + this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); + new VespaZooKeeperAdminImpl().createDummyNode(zookeeperServerConfig); + } + + @Override + public void deconstruct() { + runner.shutdown(); + super.deconstruct(); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return false; + } + +} |