Merge pull request #20743 from vespa-engine/hmusum/add-support-for-zookeeper-snapshot-methodv7.526.8

Support configuring ZooKeeper snapshot method [run-systemtest]

7 files changed, 30 insertions, 4 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
index 8ac30f66ae7..302e8eff2d8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
@@ -47,7 +47,7 @@ public class ConfigserverCluster extends AbstractConfigProducer
         this.containerCluster = containerCluster;
 
         // If we are in a config server cluster the correct zone is propagated through cloud config options,
-        // not through config to deployment options (see StandaloneContainerApplication.scala),
+        // not through config to deployment options (see StandaloneContainerApplication.java),
         // so we need to propagate the zone options into the container from here
         Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment();
         RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName();
@@ -83,6 +83,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
         if (options.zookeeperClientPort().isPresent()) {
             builder.clientPort(options.zookeeperClientPort().get());
         }
+
+        builder.snapshotMethod(options.zooKeeperSnapshotMethod());
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
index fe3bd271f2f..c61c140c05b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
@@ -38,4 +38,5 @@ public interface CloudConfigOptions {
     Optional<String> loadBalancerAddress();
     Optional<String> athenzDnsSuffix();
     Optional<String> ztsUrl();
+    String zooKeeperSnapshotMethod();
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
index 7f49efa8770..4ca85a19c35 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
@@ -47,16 +47,17 @@ public class ConfigserverClusterTest {
     }
 
     @Test
-    public void zookeeperConfig_only_config_servers_set() {
+    public void zookeeperConfig_only_config_servers_set_hosted() {
         TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.emptyList());
         ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions);
         assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3");
         assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2);
         assertEquals(1, config.myid());
+        assertEquals("gz", config.snapshotMethod());
     }
 
     @Test
-    public void zookeeperConfig_with_config_servers_and_zk_ids() {
+    public void zookeeperConfig_with_config_servers_and_zk_ids_hosted() {
         TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3));
         ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions);
         assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3");
@@ -72,6 +73,7 @@ public class ConfigserverClusterTest {
         assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3");
         assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3);
         assertEquals(2, config.myid());
+        assertEquals("", config.snapshotMethod());
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -150,7 +152,8 @@ public class ConfigserverClusterTest {
                 .useVespaVersionInRequest(true)
                 .hostedVespa(hostedVespa)
                 .environment("test")
-                .region("bar");
+                .region("bar")
+                .zooKeeperSnapshotMethod(hostedVespa ? "gz" : "");
 
         Optional.of(configServerHostnames)
                 .filter(hostnames -> !hostnames.isEmpty())
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
index 9352dac85a4..fc7f8674149 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.configserver;
 
 import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
 
+import java.util.Objects;
 import java.util.Optional;
 
 /**
@@ -17,6 +18,7 @@ public class TestOptions implements CloudConfigOptions {
     private Optional<String> region = Optional.empty();
     private Optional<Boolean> useVespaVersionInRequest = Optional.empty();
     private Optional<Boolean> hostedVespa = Optional.empty();
+    private String zooKeeperSnapshotMethod = "";
 
     @Override
     public Optional<Integer> rpcPort() {
@@ -106,6 +108,9 @@ public class TestOptions implements CloudConfigOptions {
         return Optional.empty();
     }
 
+    @Override
+    public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; }
+
     public TestOptions configServers(ConfigServer[] configServers) {
         this.configServers = configServers;
         return this;
@@ -130,4 +135,11 @@ public class TestOptions implements CloudConfigOptions {
         this.hostedVespa = Optional.of(hostedVespa);
         return this;
     }
+
+    public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) {
+        Objects.requireNonNull(snapshotMethod);
+        this.zooKeeperSnapshotMethod = snapshotMethod;
+        return this;
+    }
+
 }
diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def
index cf923d58d7d..04632ffd35f 100644
--- a/configdefinitions/src/vespa/zookeeper-server.def
+++ b/configdefinitions/src/vespa/zookeeper-server.def
@@ -51,3 +51,4 @@ tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIF
 jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks"
 
 dynamicReconfiguration bool default=false
+snapshotMethod string default=""
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java
index 5ffb37feb70..d1a56b8ac8a 100644
--- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java
@@ -121,6 +121,12 @@ public class CloudConfigInstallVariables implements CloudConfigOptions {
         return getInstallVariable("zts_url");
     }
 
+    @Override
+    public String zooKeeperSnapshotMethod() {
+        String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD");
+        return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod;
+    }
+
     static ConfigServer[] toConfigServers(String configserversString) {
         return multiValueParameterStream(configserversString)
                 .map(CloudConfigInstallVariables::toConfigServer)
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
index f95704f8b58..39d0312915f 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
@@ -41,6 +41,7 @@ public class Configurator {
         System.setProperty("zookeeper.authProvider.x509", "com.yahoo.vespa.zookeeper.VespaMtlsAuthenticationProvider");
         // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled)
         System.setProperty("zookeeper.globalOutstandingLimit", "1000");
+        System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod());
     }
 
     void writeConfigToDisk() { writeConfigToDisk(VespaTlsConfig.fromSystem()); }