diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-01-11 11:08:23 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-11 11:08:23 +0100 |
commit | f3a177246053d109d8c7f1b4bbf9808233cd09db (patch) | |
tree | acbb5643ff53cd30545d33d58eca61812d5b9a9f | |
parent | 125b87cbfb5d71948155ae2580111b30014fd188 (diff) | |
parent | 6e94f1f8cffb95ca848577681d9774053f1a8574 (diff) |
Merge pull request #20743 from vespa-engine/hmusum/add-support-for-zookeeper-snapshot-methodv7.526.8
Support configuring ZooKeeper snapshot method [run-systemtest]
7 files changed, 30 insertions, 4 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 8ac30f66ae7..302e8eff2d8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -47,7 +47,7 @@ public class ConfigserverCluster extends AbstractConfigProducer this.containerCluster = containerCluster; // If we are in a config server cluster the correct zone is propagated through cloud config options, - // not through config to deployment options (see StandaloneContainerApplication.scala), + // not through config to deployment options (see StandaloneContainerApplication.java), // so we need to propagate the zone options into the container from here Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment(); RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName(); @@ -83,6 +83,8 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } + + builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index fe3bd271f2f..c61c140c05b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -38,4 +38,5 @@ public interface CloudConfigOptions { Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); + String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 7f49efa8770..4ca85a19c35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -47,16 +47,17 @@ public class ConfigserverClusterTest { } @Test - public void zookeeperConfig_only_config_servers_set() { + public void zookeeperConfig_only_config_servers_set_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.emptyList()); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2); assertEquals(1, config.myid()); + assertEquals("gz", config.snapshotMethod()); } @Test - public void zookeeperConfig_with_config_servers_and_zk_ids() { + public void zookeeperConfig_with_config_servers_and_zk_ids_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3)); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); @@ -72,6 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); + assertEquals("", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -150,7 +152,8 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar"); + .region("bar") + .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 9352dac85a4..fc7f8674149 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import java.util.Objects; import java.util.Optional; /** @@ -17,6 +18,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); + private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -106,6 +108,9 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } + @Override + public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -130,4 +135,11 @@ public class TestOptions implements CloudConfigOptions { this.hostedVespa = Optional.of(hostedVespa); return this; } + + public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { + Objects.requireNonNull(snapshotMethod); + this.zooKeeperSnapshotMethod = snapshotMethod; + return this; + } + } diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index cf923d58d7d..04632ffd35f 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -51,3 +51,4 @@ tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIF jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" dynamicReconfiguration bool default=false +snapshotMethod string default="" diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 5ffb37feb70..d1a56b8ac8a 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -121,6 +121,12 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } + @Override + public String zooKeeperSnapshotMethod() { + String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); + return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; + } + static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index f95704f8b58..39d0312915f 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -41,6 +41,7 @@ public class Configurator { System.setProperty("zookeeper.authProvider.x509", "com.yahoo.vespa.zookeeper.VespaMtlsAuthenticationProvider"); // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled) System.setProperty("zookeeper.globalOutstandingLimit", "1000"); + System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod()); } void writeConfigToDisk() { writeConfigToDisk(VespaTlsConfig.fromSystem()); } |