From 01932261846216408c5addd0e90f3ddeae35ae41 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 24 Nov 2021 09:40:32 +0100 Subject: Sync vespa.log at least once an hour --- .../hosted/node/admin/maintenance/sync/SyncFileInfo.java | 16 ++++++++++++---- .../node/admin/maintenance/sync/SyncFileInfoTest.java | 10 ++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'node-admin/src') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index 1854ed67389..a9f2cd219b9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import java.net.URI; import java.nio.file.Path; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -18,13 +19,15 @@ public class SyncFileInfo { private final URI destination; private final Compression uploadCompression; private final Map tags; + private final Optional minDurationBetweenSync; private SyncFileInfo(Path source, URI destination, Compression uploadCompression, - Map tags) { + Map tags, Duration minDurationBetweenSyncOrNull) { this.source = source; this.destination = destination; this.uploadCompression = uploadCompression; this.tags = Map.copyOf(tags); + this.minDurationBetweenSync = Optional.ofNullable(minDurationBetweenSyncOrNull); } /** Source path of the file to sync */ @@ -44,14 +47,18 @@ public class SyncFileInfo { public Map tags() { return tags; } + public Optional minDurationBetweenSync() { return minDurationBetweenSync; } + public static Optional forLogFile(URI uri, Path logFile, boolean rotatedOnly, ApplicationId owner) { String filename = logFile.getFileName().toString(); Compression compression; String dir = null; + Duration minDurationBetweenSync = null; - if ((!rotatedOnly && filename.equals("vespa.log")) || filename.startsWith("vespa.log-")) { + if (filename.startsWith("vespa.log")) { dir = "logs/vespa/"; compression = Compression.ZSTD; + minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) @@ -64,7 +71,8 @@ public class SyncFileInfo { if (dir == null) return Optional.empty(); return Optional.of(new SyncFileInfo( - logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner))); + logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner), + minDurationBetweenSync)); } public static SyncFileInfo forServiceDump(URI destinationDir, Path file, Compression compression, @@ -75,7 +83,7 @@ public class SyncFileInfo { if (assetClassification != null) { tags.put("vespa:AssetClassification", assetClassification); } - return new SyncFileInfo(file, location, compression, tags); + return new SyncFileInfo(file, location, compression, tags, null); } private static Map defaultTags(ApplicationId owner) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 7130ac54430..54701b0f5a7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -8,6 +8,7 @@ import org.junit.Test; import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.Path; +import java.time.Duration; import java.util.Optional; import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; @@ -57,16 +58,21 @@ public class SyncFileInfoTest { @Test public void vespa_logs() { - assertForLogFile(vespaLogPath1, null, null, true); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, false); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ZERO, false); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); } private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { + assertForLogFile(srcPath, destination, compression, null, rotatedOnly); + } + + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) { Optional sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId()); assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null)); assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null)); + assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null)); } } -- cgit v1.2.3