Limit run data and logs to a history of the 256 last ones

3 files changed, 22 insertions, 5 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 13c7860ebe3..1c1c436bda7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -25,10 +25,12 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import java.net.URI;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -55,6 +57,8 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests;
  */
 public class JobController {
 
+    private static final int historyLength = 256;
+
     private final Controller controller;
     private final CuratorDb curator;
     private final BufferedLogStore logs;
@@ -187,9 +191,17 @@ public class JobController {
 
     /** Changes the status of the given run to inactive, and stores it as a historic run. */
     public void finish(RunId id) {
-        locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails.
+        locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails.
             Run finishedRun = run.finished(controller.clock().instant());
-            locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun));
+            locked(id.application(), id.type(), runs -> {
+                runs.put(run.id(), finishedRun);
+                long last = id.number();
+                Iterator<RunId> ids = runs.keySet().iterator();
+                for (RunId old = ids.next(); old.number() < last - historyLength; old = ids.next()) {
+                    logs.delete(old);
+                    ids.remove();
+                }
+            });
             logs.flush(id);
             return finishedRun;
         });
@@ -336,9 +348,9 @@ public class JobController {
     }
 
     /** Locks and modifies the list of historic runs for the given application and job type. */
-    private void locked(ApplicationId id, JobType type, Consumer<Map<RunId, Run>> modifications) {
+    private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) {
         try (Lock __ = curator.lock(id, type)) {
-            Map<RunId, Run> runs = curator.readHistoricRuns(id, type);
+            SortedMap<RunId, Run> runs = curator.readHistoricRuns(id, type);
             modifications.accept(runs);
             curator.writeHistoricRuns(id, type, runs.values());
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index d631df2921e..089f3012e7e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -75,7 +75,7 @@ public class JobRunner extends Maintainer {
             advance(jobs.run(run.id()).get());
         }
         else if (run.readySteps().isEmpty())
-            jobs.finish(run.id());
+            executors.execute(() -> jobs.finish(run.id()));
         else
             run.readySteps().forEach(step -> executors.execute(() -> advance(run.id(), step)));
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
index a49061c9d67..3df4451f900 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java
@@ -77,6 +77,11 @@ public class BufferedLogStore {
         buffer.deleteLog(id.application(), id.type());
     }
 
+    /** Deletes the logs for the given run, if already moved to storage. */
+    public void delete(RunId id) {
+        store.delete(id);
+    }
+
     /** Deletes all logs for the given application. */
     public void delete(ApplicationId id) {
         for (JobType type : JobType.values())