From 73e8f00c796952138f36560679266325822eca4a Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 9 May 2022 14:26:03 +0200 Subject: Sync ZK logs --- .../hosted/node/admin/maintenance/sync/SyncFileInfo.java | 15 ++++++++++++++- .../node/admin/maintenance/sync/SyncFileInfoTest.java | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) (limited to 'node-admin') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index a9f2cd219b9..ffce5e6e68b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import java.net.URI; import java.nio.file.Path; import java.time.Duration; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -15,6 +18,9 @@ import java.util.Optional; */ public class SyncFileInfo { + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter + .ofPattern("yyyy-MM-dd.HH-mm-ss").withZone(ZoneOffset.UTC); + private final Path source; private final URI destination; private final Compression uploadCompression; @@ -53,12 +59,19 @@ public class SyncFileInfo { String filename = logFile.getFileName().toString(); Compression compression; String dir = null; + String remoteFilename = logFile.getFileName().toString(); Duration minDurationBetweenSync = null; if (filename.startsWith("vespa.log")) { dir = "logs/vespa/"; compression = Compression.ZSTD; minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; + } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) { + compression = Compression.ZSTD; + dir = "logs/zookeeper/"; + remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" : + "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); + minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) @@ -71,7 +84,7 @@ public class SyncFileInfo { if (dir == null) return Optional.empty(); return Optional.of(new SyncFileInfo( - logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner), + logFile, uri.resolve(dir + remoteFilename + compression.extension), compression, defaultTags(owner), minDurationBetweenSync)); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 54701b0f5a7..1b5c51082c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; @@ -9,6 +10,7 @@ import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; @@ -31,6 +33,8 @@ public class SyncFileInfoTest { private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210212.zst"); private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); + private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); @Test public void access_logs() { @@ -65,6 +69,16 @@ public class SyncFileInfoTest { assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); } + @Test + public void zookeeper_logs() { + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false); + + new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); + } + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { assertForLogFile(srcPath, destination, compression, null, rotatedOnly); } -- cgit v1.2.3