Don't discard messages from the system, even though user log is truncated

3 files changed, 28 insertions, 13 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 19b2afb3af9..881107fa0f9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -158,7 +158,7 @@ public class JobController {
     /** Stores the given log entries for the given run and step. */
     public void log(RunId id, Step step, List<LogEntry> entries) {
         locked(id, __ -> {
-            logs.append(id.application(), id.type(), step, entries);
+            logs.append(id.application(), id.type(), step, entries, true);
             return __;
         });
     }
@@ -211,7 +211,7 @@ public class JobController {
             if (log.isEmpty())
                 return run;
 
-            logs.append(id.application(), id.type(), Step.copyVespaLogs, log);
+            logs.append(id.application(), id.type(), Step.copyVespaLogs, log, false);
             return run.with(log.get(log.size() - 1).at());
         });
     }
@@ -230,7 +230,7 @@ public class JobController {
             if (entries.isEmpty())
                 return run;
 
-            logs.append(id.application(), id.type(), step.get(), entries);
+            logs.append(id.application(), id.type(), step.get(), entries, false);
             return run.with(entries.stream().mapToLong(LogEntry::id).max().getAsLong());
         });
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
index 9721026c628..ecb9db8195f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
@@ -49,7 +49,7 @@ public class BufferedLogStore {
     }
 
     /** Appends to the log of the given, active run, reassigning IDs as counted here, and converting to Vespa log levels. */
-    public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries) {
+    public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries, boolean forceLog) {
         if (entries.isEmpty())
             return;
 
@@ -58,7 +58,7 @@ public class BufferedLogStore {
         long lastEntryId = buffer.readLastLogEntryId(id, type).orElse(-1L);
         long lastChunkId = buffer.getLogChunkIds(id, type).max().orElse(0);
         long numberOfChunks = Math.max(1, buffer.getLogChunkIds(id, type).count());
-        if (numberOfChunks > maxLogSize / chunkSize)
+        if (numberOfChunks > maxLogSize / chunkSize && ! forceLog)
             return; // Max size exceeded — store no more.
 
         byte[] emptyChunk = "[]".getBytes();
@@ -72,8 +72,12 @@ public class BufferedLogStore {
                 buffer.writeLastLogEntryId(id, type, lastEntryId);
                 buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log));
                 lastChunkId = lastEntryId + 1;
-                if (++numberOfChunks > maxLogSize / chunkSize) {
-                    log = Map.of(step, List.of(new LogEntry(++lastEntryId, entry.at(), LogEntry.Type.warning, "Max log size of " + (maxLogSize >> 20) + "Mb exceeded; further entries are discarded.")));
+                if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) {
+                    log = Map.of(step, List.of(new LogEntry(++lastEntryId,
+                                                            entry.at(),
+                                                            LogEntry.Type.warning,
+                                                            "Max log size of " + (maxLogSize >> 20) +
+                                                            "Mb exceeded; further user entries are discarded.")));
                     break;
                 }
                 log = new HashMap<>();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java
index 0f7f97d333a..87280c0c1a3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java
@@ -50,19 +50,19 @@ public class BufferedLogStoreTest {
         assertEquals(Optional.empty(), logs.readFinished(id, -1));
         assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1));
 
-        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry));
+        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false);
         assertEquals(List.of(entry0),
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal));
         assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 0));
 
-        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry));
+        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false);
         assertEquals(List.of(entry0, entry1),
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal));
         assertEquals(List.of(entry1),
                      logs.readActive(id.application(), id.type(), 0).get(Step.deployReal));
         assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 1));
 
-        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry));
+        logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry), false);
         assertEquals(List.of(entry0, entry1, entry2, entry3, entry4),
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal));
         assertEquals(List.of(entry1, entry2, entry3, entry4),
@@ -105,17 +105,28 @@ public class BufferedLogStoreTest {
         logged.remove(logged.size() - 1);
         logged.remove(logged.size() - 1);
         logged.remove(logged.size() - 1);
-        logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further entries are discarded."));
+        logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further user entries are discarded."));
 
-        logs.append(id.application(), id.type(), Step.deployReal, monsterLog);
+        logs.append(id.application(), id.type(), Step.deployReal, monsterLog, false);
         assertEquals(logged.size(),
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size());
         assertEquals(logged,
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal));
 
+        // An additional, forced entry is appended.
+        LogEntry forced = new LogEntry(logged.size(), entry.at(), entry.type(), entry.message());
+        logs.append(id.application(), id.type(), Step.deployReal, List.of(forced), true);
+        logged.add(forced);
+        assertEquals(logged.size(),
+                     logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size());
+        assertEquals(logged,
+                     logs.readActive(id.application(), id.type(), -1).get(Step.deployReal));
+        logged.remove(logged.size() - 1);
+
+        // Flushing the buffer clears it again, and makes it ready for reuse.
         logs.flush(id);
         for (int i = 0; i < 2 * maxChunks + 3; i++)
-            logs.append(id.application(), id.type(), Step.deployReal, List.of(entry));
+            logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false);
         assertEquals(logged.size(),
                      logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size());
         assertEquals(logged,