Sync vespa.log at least once an hour

2 files changed, 20 insertions, 6 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
index 1854ed67389..a9f2cd219b9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId;
 
 import java.net.URI;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -18,13 +19,15 @@ public class SyncFileInfo {
     private final URI destination;
     private final Compression uploadCompression;
     private final Map<String, String> tags;
+    private final Optional<Duration> minDurationBetweenSync;
 
     private SyncFileInfo(Path source, URI destination, Compression uploadCompression,
-                         Map<String, String> tags) {
+                         Map<String, String> tags, Duration minDurationBetweenSyncOrNull) {
         this.source = source;
         this.destination = destination;
         this.uploadCompression = uploadCompression;
         this.tags = Map.copyOf(tags);
+        this.minDurationBetweenSync = Optional.ofNullable(minDurationBetweenSyncOrNull);
     }
 
     /** Source path of the file to sync */
@@ -44,14 +47,18 @@ public class SyncFileInfo {
 
     public Map<String, String> tags() { return tags; }
 
+    public Optional<Duration> minDurationBetweenSync() { return minDurationBetweenSync; }
+
     public static Optional<SyncFileInfo> forLogFile(URI uri, Path logFile, boolean rotatedOnly, ApplicationId owner) {
         String filename = logFile.getFileName().toString();
         Compression compression;
         String dir = null;
+        Duration minDurationBetweenSync = null;
 
-        if ((!rotatedOnly && filename.equals("vespa.log")) || filename.startsWith("vespa.log-")) {
+        if (filename.startsWith("vespa.log")) {
             dir = "logs/vespa/";
             compression = Compression.ZSTD;
+            minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
         } else {
             compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD;
             if (rotatedOnly && compression != Compression.NONE)
@@ -64,7 +71,8 @@ public class SyncFileInfo {
 
         if (dir == null) return Optional.empty();
         return Optional.of(new SyncFileInfo(
-                logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner)));
+                logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner),
+                minDurationBetweenSync));
     }
 
     public static SyncFileInfo forServiceDump(URI destinationDir, Path file, Compression compression,
@@ -75,7 +83,7 @@ public class SyncFileInfo {
         if (assetClassification != null) {
             tags.put("vespa:AssetClassification", assetClassification);
         }
-        return new SyncFileInfo(file, location, compression, tags);
+        return new SyncFileInfo(file, location, compression, tags, null);
     }
 
     private static Map<String, String> defaultTags(ApplicationId owner) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
index 7130ac54430..54701b0f5a7 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -8,6 +8,7 @@ import org.junit.Test;
 import java.net.URI;
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE;
@@ -57,16 +58,21 @@ public class SyncFileInfoTest {
 
     @Test
     public void vespa_logs() {
-        assertForLogFile(vespaLogPath1, null, null, true);
-        assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, false);
+        assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true);
+        assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ZERO, false);
 
         assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true);
         assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false);
     }
 
     private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) {
+        assertForLogFile(srcPath, destination, compression, null, rotatedOnly);
+    }
+
+    private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) {
         Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId());
         assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null));
         assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null));
+        assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null));
     }
 }