Merge pull request #22512 from vespa-engine/freva/sync-zk-logs

Sync ZK logs

2 files changed, 28 insertions, 1 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
index a9f2cd219b9..ffce5e6e68b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
@@ -2,10 +2,13 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 
 import java.net.URI;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -15,6 +18,9 @@ import java.util.Optional;
  */
 public class SyncFileInfo {
 
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter
+            .ofPattern("yyyy-MM-dd.HH-mm-ss").withZone(ZoneOffset.UTC);
+
     private final Path source;
     private final URI destination;
     private final Compression uploadCompression;
@@ -53,12 +59,19 @@ public class SyncFileInfo {
         String filename = logFile.getFileName().toString();
         Compression compression;
         String dir = null;
+        String remoteFilename = logFile.getFileName().toString();
         Duration minDurationBetweenSync = null;
 
         if (filename.startsWith("vespa.log")) {
             dir = "logs/vespa/";
             compression = Compression.ZSTD;
             minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
+        } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) {
+            compression = Compression.ZSTD;
+            dir = "logs/zookeeper/";
+            remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" :
+                    "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime());
+            minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
         } else {
             compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD;
             if (rotatedOnly && compression != Compression.NONE)
@@ -71,7 +84,7 @@ public class SyncFileInfo {
 
         if (dir == null) return Optional.empty();
         return Optional.of(new SyncFileInfo(
-                logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner),
+                logFile, uri.resolve(dir + remoteFilename + compression.extension), compression, defaultTags(owner),
                 minDurationBetweenSync));
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
index 54701b0f5a7..1b5c51082c5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Test;
 
@@ -9,6 +10,7 @@ import java.net.URI;
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE;
@@ -31,6 +33,8 @@ public class SyncFileInfoTest {
     private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210212.zst");
     private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log");
     private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12");
+    private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log");
+    private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log");
 
     @Test
     public void access_logs() {
@@ -65,6 +69,16 @@ public class SyncFileInfoTest {
         assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false);
     }
 
+    @Test
+    public void zookeeper_logs() {
+        assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true);
+        assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false);
+
+        new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
+        assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true);
+        assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false);
+    }
+
     private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) {
         assertForLogFile(srcPath, destination, compression, null, rotatedOnly);
     }