Merge pull request #27413 from vespa-engine/hmusum/add-zookeeper-snapshot-service-dump-command

Add ZooKeeperSnapshotDumper

4 files changed, 74 insertions, 6 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java
index 4f72c1e0adb..bf39d9d3e88 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java
@@ -49,7 +49,8 @@ class ArtifactProducers {
                 new JvmDumper.Jstat(),
                 new JvmDumper.Jstack(),
                 new PmapReporter(),
-                new VespaLogDumper(sleeper));
+                new VespaLogDumper(sleeper),
+                new ZooKeeperSnapshotDumper());
         var aliases =
                 Map.of(
                         "jvm-dump",
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java
new file mode 100644
index 00000000000..0887a8bb3d5
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java
@@ -0,0 +1,27 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
+
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL;
+
+/**
+ * Performs dump of ZooKeeper snapshots. Can be used for controllers, config servers, cluster controllers and tenant containers
+ * where zookeeper is configured.
+ *
+ * @author hmusum
+ */
+class ZooKeeperSnapshotDumper implements ArtifactProducer {
+    @Override public String artifactName() { return "zookeeper-snapshot"; }
+    @Override public String description() { return "Dumps ZooKeeper snapshots"; }
+
+    @Override
+    public List<Artifact> produceArtifacts(Context ctx) {
+        ContainerPath zookeeperSnapshot = ctx.outputContainerPath().resolve("zookeeper-snapshot.tgz");
+        List<String> cmd = List.of("bash", "-c", String.format("/opt/vespa/bin/vespa-backup-zk-data.sh -o %s -k -f", zookeeperSnapshot.pathInContainer()));
+        ctx.executeCommandInNode(cmd, true);
+        return List.of(Artifact.newBuilder().classification(CONFIDENTIAL).file(zookeeperSnapshot).build());
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java
index fd7d0d8c1c0..fc1bc41a06b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java
@@ -22,7 +22,7 @@ class ArtifactProducersTest {
         String expectedMsg =
                 "Invalid artifact type 'unknown-artifact'. Valid types are " +
                         "['jvm-heap-dump', 'jvm-jfr', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'perf-report', 'pmap', " +
-                        "'vespa-log'] " +
+                        "'vespa-log', 'zookeeper-snapshot'] " +
                         "and valid aliases are " +
                         "['jvm-dump': ['jvm-heap-dump', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'vespa-log']]";
         assertEquals(expectedMsg, exception.getMessage());
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java
index cbe42c90a20..30f5d09303d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java
@@ -27,6 +27,7 @@ import java.nio.file.Path;
 import java.time.Instant;
 import java.util.List;
 
+import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.DumpOptions;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
@@ -112,8 +113,7 @@ class VespaServiceDumperImplTest {
         SyncClient syncClient = createSyncClientMock();
         NodeRepoMock nodeRepository = new NodeRepoMock();
         TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
-        NodeSpec nodeSpec = createNodeSpecWithDumpRequest(
-                nodeRepository, List.of("jvm-jfr"), new ServiceDumpReport.DumpOptions(null, null, null));
+        NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("jvm-jfr"));
 
         VespaServiceDumper reporter = new VespaServiceDumperImpl(
                 ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
@@ -141,6 +141,43 @@ class VespaServiceDumperImplTest {
     }
 
     @Test
+    void invokes_zookeeper_backup_command_when_generating_snapshot() {
+        // Setup mocks
+        ContainerOperations operations = mock(ContainerOperations.class);
+        when(operations.executeCommandInContainer(any(), any(), any()))
+                .thenReturn(new CommandResult(null, 0, "12345"))
+                .thenReturn(new CommandResult(null, 0, ""))
+                .thenReturn(new CommandResult(null, 0, ""));
+        SyncClient syncClient = createSyncClientMock();
+        NodeRepoMock nodeRepository = new NodeRepoMock();
+        TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L));
+        NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("zookeeper-snapshot"));
+
+        VespaServiceDumper reporter = new VespaServiceDumperImpl(
+                ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer);
+        NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec)
+                .fileSystem(fileSystem)
+                .build();
+        reporter.processServiceDumpRequest(context);
+
+        verify(operations).executeCommandInContainer(
+                context,
+                context.users().vespa(),
+                "bash",
+                "-c",
+                "/opt/vespa/bin/vespa-backup-zk-data.sh -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/zookeeper-snapshot.tgz -k -f");
+
+        String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," +
+                "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," +
+                "\"configId\":\"default/container.1\",\"artifacts\":[\"zookeeper-snapshot\"],\"dumpOptions\":{}}";
+        assertReportEquals(nodeRepository, expectedJson);
+
+        List<URI> expectedUris = List.of(
+                URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/zookeeper-snapshot.tgz"));
+        assertSyncedFiles(context, syncClient, expectedUris);
+    }
+
+    @Test
     void handles_multiple_artifact_types() {
         // Setup mocks
         ContainerOperations operations = mock(ContainerOperations.class);
@@ -198,8 +235,11 @@ class VespaServiceDumperImplTest {
         assertReportEquals(nodeRepository, expectedJson);
     }
 
-    private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts,
-                                                          ServiceDumpReport.DumpOptions options) {
+    private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts) {
+        return createNodeSpecWithDumpRequest(repository, artifacts, new DumpOptions(null, null, null));
+    }
+
+    private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, DumpOptions options) {
         ServiceDumpReport request = ServiceDumpReport.createRequestReport(
                 Instant.ofEpochMilli(1600000000000L), null, "default/container.1", artifacts, options);
         NodeSpec spec = NodeSpec.Builder