From 91cd390b2550820ee99998a51e34f185e5bc2ca1 Mon Sep 17 00:00:00 2001 From: jonmv Date: Fri, 8 Jul 2022 11:11:31 +0200 Subject: Don't discard messages from the system, even though user log is truncated --- .../controller/deployment/JobController.java | 6 +++--- .../controller/persistence/BufferedLogStore.java | 12 +++++++---- .../persistence/BufferedLogStoreTest.java | 23 ++++++++++++++++------ 3 files changed, 28 insertions(+), 13 deletions(-) (limited to 'controller-server/src') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 19b2afb3af9..881107fa0f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -158,7 +158,7 @@ public class JobController { /** Stores the given log entries for the given run and step. */ public void log(RunId id, Step step, List entries) { locked(id, __ -> { - logs.append(id.application(), id.type(), step, entries); + logs.append(id.application(), id.type(), step, entries, true); return __; }); } @@ -211,7 +211,7 @@ public class JobController { if (log.isEmpty()) return run; - logs.append(id.application(), id.type(), Step.copyVespaLogs, log); + logs.append(id.application(), id.type(), Step.copyVespaLogs, log, false); return run.with(log.get(log.size() - 1).at()); }); } @@ -230,7 +230,7 @@ public class JobController { if (entries.isEmpty()) return run; - logs.append(id.application(), id.type(), step.get(), entries); + logs.append(id.application(), id.type(), step.get(), entries, false); return run.with(entries.stream().mapToLong(LogEntry::id).max().getAsLong()); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 9721026c628..ecb9db8195f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -49,7 +49,7 @@ public class BufferedLogStore { } /** Appends to the log of the given, active run, reassigning IDs as counted here, and converting to Vespa log levels. */ - public void append(ApplicationId id, JobType type, Step step, List entries) { + public void append(ApplicationId id, JobType type, Step step, List entries, boolean forceLog) { if (entries.isEmpty()) return; @@ -58,7 +58,7 @@ public class BufferedLogStore { long lastEntryId = buffer.readLastLogEntryId(id, type).orElse(-1L); long lastChunkId = buffer.getLogChunkIds(id, type).max().orElse(0); long numberOfChunks = Math.max(1, buffer.getLogChunkIds(id, type).count()); - if (numberOfChunks > maxLogSize / chunkSize) + if (numberOfChunks > maxLogSize / chunkSize && ! forceLog) return; // Max size exceeded — store no more. byte[] emptyChunk = "[]".getBytes(); @@ -72,8 +72,12 @@ public class BufferedLogStore { buffer.writeLastLogEntryId(id, type, lastEntryId); buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); lastChunkId = lastEntryId + 1; - if (++numberOfChunks > maxLogSize / chunkSize) { - log = Map.of(step, List.of(new LogEntry(++lastEntryId, entry.at(), LogEntry.Type.warning, "Max log size of " + (maxLogSize >> 20) + "Mb exceeded; further entries are discarded."))); + if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) { + log = Map.of(step, List.of(new LogEntry(++lastEntryId, + entry.at(), + LogEntry.Type.warning, + "Max log size of " + (maxLogSize >> 20) + + "Mb exceeded; further user entries are discarded."))); break; } log = new HashMap<>(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index 0f7f97d333a..87280c0c1a3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -50,19 +50,19 @@ public class BufferedLogStoreTest { assertEquals(Optional.empty(), logs.readFinished(id, -1)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 0)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0, entry1), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1), logs.readActive(id.application(), id.type(), 0).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry), false); assertEquals(List.of(entry0, entry1, entry2, entry3, entry4), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1, entry2, entry3, entry4), @@ -105,17 +105,28 @@ public class BufferedLogStoreTest { logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); - logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further entries are discarded.")); + logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further user entries are discarded.")); - logs.append(id.application(), id.type(), Step.deployReal, monsterLog); + logs.append(id.application(), id.type(), Step.deployReal, monsterLog, false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + // An additional, forced entry is appended. + LogEntry forced = new LogEntry(logged.size(), entry.at(), entry.type(), entry.message()); + logs.append(id.application(), id.type(), Step.deployReal, List.of(forced), true); + logged.add(forced); + assertEquals(logged.size(), + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); + assertEquals(logged, + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + logged.remove(logged.size() - 1); + + // Flushing the buffer clears it again, and makes it ready for reuse. logs.flush(id); for (int i = 0; i < 2 * maxChunks + 3; i++) - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, -- cgit v1.2.3