diff options
5 files changed, 30 insertions, 5 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java index fe936b3a752..418d1089428 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/RunDataStore.java @@ -17,6 +17,9 @@ public interface RunDataStore { /** Stores the given log for the given deployment job. */ void put(RunId id, byte[] log); + /** Deletes the run logs for the given deployment job. */ + void delete(RunId id); + /** Deletes all data associated with the given application. */ void delete(ApplicationId id); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java index 0d7d8eca10e..16874f996a5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRunDataStore.java @@ -27,6 +27,11 @@ public class MockRunDataStore implements RunDataStore { } @Override + public void delete(RunId id) { + logs.remove(id); + } + + @Override public void delete(ApplicationId id) { logs.keySet().removeIf(runId -> runId.application().equals(id)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 13c7860ebe3..1c1c436bda7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -25,10 +25,12 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.net.URI; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.SortedMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -55,6 +57,8 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; */ public class JobController { + private static final int historyLength = 256; + private final Controller controller; private final CuratorDb curator; private final BufferedLogStore logs; @@ -187,9 +191,17 @@ public class JobController { /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { - locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. + locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. Run finishedRun = run.finished(controller.clock().instant()); - locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun)); + locked(id.application(), id.type(), runs -> { + runs.put(run.id(), finishedRun); + long last = id.number(); + Iterator<RunId> ids = runs.keySet().iterator(); + for (RunId old = ids.next(); old.number() < last - historyLength; old = ids.next()) { + logs.delete(old); + ids.remove(); + } + }); logs.flush(id); return finishedRun; }); @@ -336,9 +348,9 @@ public class JobController { } /** Locks and modifies the list of historic runs for the given application and job type. */ - private void locked(ApplicationId id, JobType type, Consumer<Map<RunId, Run>> modifications) { + private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { try (Lock __ = curator.lock(id, type)) { - Map<RunId, Run> runs = curator.readHistoricRuns(id, type); + SortedMap<RunId, Run> runs = curator.readHistoricRuns(id, type); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index d631df2921e..089f3012e7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -75,7 +75,7 @@ public class JobRunner extends Maintainer { advance(jobs.run(run.id()).get()); } else if (run.readySteps().isEmpty()) - jobs.finish(run.id()); + executors.execute(() -> jobs.finish(run.id())); else run.readySteps().forEach(step -> executors.execute(() -> advance(run.id(), step))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index a49061c9d67..3df4451f900 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -77,6 +77,11 @@ public class BufferedLogStore { buffer.deleteLog(id.application(), id.type()); } + /** Deletes the logs for the given run, if already moved to storage. */ + public void delete(RunId id) { + store.delete(id); + } + /** Deletes all logs for the given application. */ public void delete(ApplicationId id) { for (JobType type : JobType.values()) |