Daft draft to share -- don't stare

9 files changed, 290 insertions, 125 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index dc339dd9330..98ec77566f7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -1,12 +1,19 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.google.common.collect.ImmutableList;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.LogStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
 
 import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
 
 /**
  * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs.
@@ -42,22 +49,67 @@ public class JobController {
 
     /** Returns all job types which have been run for the given application. */
     public List<JobType> jobs(ApplicationId id) {
-        return null;
+        return ImmutableList.copyOf(Stream.of(JobType.values())
+                                          .filter(type -> ! runs(id, type).isEmpty())
+                                          .iterator());
     }
 
-    /** Returns a list of meta information about all known runs of the given job type. */
+    /** Returns a list of meta information about all known runs of the given job type for the given application. */
     public List<RunStatus> runs(ApplicationId id, JobType type) {
-        return null;
+        ImmutableList.Builder<RunStatus> runs = ImmutableList.builder();
+        runs.addAll(controller.curator().readHistoricRuns(id, type));
+        activeRuns().stream()
+                    .filter(run -> run.id().application().equals(id) && run.id().type() == type)
+                    .forEach(runs::add);
+        return runs.build();
     }
 
-    /** Returns the current status of the given job. */
-    public RunStatus status(RunId id) {
-        return null;
+    List<RunStatus> activeRuns() {
+        return controller.curator().readActiveRuns();
+    }
+
+    /** Returns the updated status of the given job, if it is active. */
+    public Optional<RunStatus> currentStatus(RunId id) {
+        try (Lock __ = controller.curator().lockActiveRuns()) {
+            return activeRuns().stream() // TODO jvenstad: Change these to Map<RunId, RunStatus>.
+                               .filter(run -> run.id().equals(id))
+                               .findAny();
+        }
+    }
+
+    public Optional<RunStatus> update(RunId id, Step.Status status, LockedStep step) {
+        return currentStatus(id).map(run -> {
+            run = run.with(status, step);
+            controller.curator().writeActiveRun(run);
+            return run;
+        });
+    }
+
+    public void locked(RunId id, Step step, Consumer<LockedStep> action) {
+        try (Lock lock = controller.curator().lock(id.application(), id.type(), step)) {
+            for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running.
+                try (Lock __ = controller.curator().lock(id.application(), id.type(), prerequisite)) { ; }
+
+            action.accept(new LockedStep(lock, step));
+        }
+        catch (TimeoutException e) {
+            // Something else is already running that step, or a prerequisite -- try again later!
+        }
+    }
+
+    public void finish(RunId id) {
+        controller.applications().lockIfPresent(id.application(), __ -> {
+            currentStatus(id).ifPresent(run -> {
+                controller.curator().writeHistoricRun(run.with(controller.clock().instant()));
+            });
+        });
     }
 
     /** Returns the details for the given job. */
     public RunDetails details(RunId id) {
-        return null;
+        try (Lock __ = controller.curator().lock(id.application(), id.type())) {
+            return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id));
+        }
     }
 
     /** Registers the given application, such that it may have deployment jobs run here. */
@@ -66,16 +118,23 @@ public class JobController {
                 controller.applications().store(application.withBuiltInternally(true)));
     }
 
-    /** Accepts and stores a new appliaction package and test jar pair, and returns the reference these will have. */
-    public ApplicationVersion submit(byte[] applicationPackage, byte[] applicationTestJar) {
+    /** Accepts and stores a new appliaction package and test jar pair. */
+    public void submit(ApplicationId id, byte[] applicationPackage, byte[] applicationTestJar) {
+        controller.applications().lockOrThrow(id, application -> {
+            ApplicationVersion version = nextVersion(id);
 
-        // TODO jvenstad: Return versions with increasing numbers.
+            // TODO smorgrav: Store the pair.
 
-        return ApplicationVersion.unknown;
+            notifyOfNewSubmission(id);
+        });
     }
 
     /** Orders a run of the given type, and returns the id of the created job. */
     public RunId run(ApplicationId id, JobType type) {
+        try (Lock __ = controller.curator().lock(id, type);
+             Lock ___ = controller.curator().lockActiveRuns()) {
+            List<RunStatus> runs = controller.curator().readHistoricRuns(id, type);
+        }
         return null;
     }
 
@@ -94,4 +153,19 @@ public class JobController {
         ;
     }
 
+
+    private void advanceJobs() {
+        activeRuns().forEach(run -> {
+
+        });
+    }
+
+    private ApplicationVersion nextVersion(ApplicationId id) {
+        throw new AssertionError();
+    }
+
+    private void notifyOfNewSubmission(ApplicationId id) {
+        ;
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
index f0093d20f56..4e8495ee10b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
@@ -1,5 +1,71 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
-public class JobProfile {
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+import static com.yahoo.vespa.hosted.controller.deployment.Step.*;
+
+/**
+ * Static profiles defining the {@link Step}s of a deployment job.
+ *
+ * @author jonmv
+ */
+public enum JobProfile {
+
+    systemTest(EnumSet.of(deployReal,
+                          installReal,
+                          deployTester,
+                          installTester,
+                          runTests),
+               EnumSet.of(storeData,
+                          deactivateTester,
+                          deactivateReal)),
+
+    stagingTest(EnumSet.of(deployInitialReal,
+                           installInitialReal,
+                           deployReal,
+                           installReal,
+                           deployTester,
+                           installTester,
+                           runTests),
+                EnumSet.of(storeData,
+                           deactivateTester,
+                           deactivateReal)),
+
+    production(EnumSet.of(deployReal,
+                          installReal,
+                          deployTester,
+                          installTester,
+                          runTests),
+               EnumSet.of(storeData,
+                          deactivateTester));
+
+
+    private final Set<Step> steps;
+    private final Set<Step> alwaysRun;
+
+    JobProfile(Set<Step> runWhileSuccess, Set<Step> alwaysRun) {
+        runWhileSuccess.addAll(alwaysRun);
+        this.steps = Collections.unmodifiableSet(runWhileSuccess);
+        this.alwaysRun = Collections.unmodifiableSet(alwaysRun);
+    }
+
+    public static JobProfile of(JobType type) {
+        switch (type.environment()) {
+            case test: return systemTest;
+            case staging: return stagingTest;
+            case prod: return production;
+            default: throw new IllegalArgumentException("Unexpected environment " + type.environment());
+        }
+    }
+
+    /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */
+    public Set<Step> steps() { return steps; }
+
+    /** Returns the set of steps that should always be run, regardless of outcome. */
+    public Set<Step> alwaysRun() { return alwaysRun; }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java
index cc2e46e5132..7586b80d228 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java
@@ -1,5 +1,12 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.yahoo.vespa.curator.Lock;
+
 public class LockedStep {
 
+    private final Step step;
+
+    LockedStep(Lock lock, Step step) { this.step = step; }
+    public Step get() { return step; }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
index 9bdf0c76d14..aaf43097908 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
@@ -13,8 +13,8 @@ public enum RunResult {
     /** Deployment of the real application was rejected. */
     deploymentFailed,
 
-    /** Convergence of the real application timed out. */
-    convergenceFailed,
+    /** Installation of the real application timed out. */
+    installationFailed,
 
     /** Real application was deployed, but the tester application was not. */
     testError,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
index f200ca4f82d..6fbbe92def9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
@@ -1,28 +1,18 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 
 import java.time.Instant;
-import java.util.List;
+import java.util.Collections;
+import java.util.EnumMap;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
-import java.util.Set;
-import java.util.Stack;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.pending;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
 import static java.util.Objects.requireNonNull;
-import static java.util.function.Function.identity;
-import static java.util.stream.Collectors.toMap;
 
 /**
- * Contains state information for a deployment job run by an {@link InternalBuildService}.
- *
- * Immutable.
+ * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}.
  *
  * @author jonmv
  */
@@ -30,29 +20,30 @@ public class RunStatus {
 
     private final RunId id;
     private final Map<Step, Step.Status> status;
-    private final Set<Step> alwaysRun;
-    private final RunResult result;
     private final Instant start;
     private final Instant end;
 
-    RunStatus(RunId id, Map<Step, Step.Status> status, Set<Step> alwaysRun, RunResult result, Instant start, Instant end) {
+    RunStatus(RunId id, Map<Step, Step.Status> status, Instant start, Instant end) {
         this.id = id;
         this.status = status;
-        this.alwaysRun = alwaysRun;
-        this.result = result;
         this.start = start;
         this.end = end;
     }
 
-    public static RunStatus initial(RunId id, Set<Step> runWhileSuccess, Set<Step> alwaysRun, Instant now) {
-        ImmutableMap.Builder<Step, Step.Status> status = ImmutableMap.builder();
-        runWhileSuccess.forEach(step -> status.put(step, pending));
-        alwaysRun.forEach(step -> status.put(step, pending));
-        return new RunStatus(requireNonNull(id), status.build(), alwaysRun, null, requireNonNull(now), null);
+    public static RunStatus initial(RunId id, Instant now) {
+        Map<Step, Step.Status> status = new EnumMap<>(Step.class);
+        JobProfile.of(id.type()).steps().forEach(step -> status.put(step, unfinished));
+        return new RunStatus(requireNonNull(id), status, requireNonNull(now), null);
+    }
+
+    public RunStatus with(Step.Status update, LockedStep step) {
+        RunStatus run = new RunStatus(id, status, start, end);
+        run.status.put(step.get(), update);
+        return run;
     }
 
-    public RunStatus with(Step.Status update, Step step) {
-        return new RunStatus(id, ImmutableMap.<Step, Step.Status>builder().putAll(status).put(step, update).build(), alwaysRun, result, start, end);
+    public RunStatus with(Instant now) {
+        return new RunStatus(id, status, start, now);
     }
 
     /** Returns the id of this run. */
@@ -60,14 +51,15 @@ public class RunStatus {
         return id;
     }
 
-    /** Returns the status of all steps in this run. */
+    /** Returns an unmodifiable view of the status of all steps in this run. */
     public Map<Step, Step.Status> status() {
-        return status;
+        return Collections.unmodifiableMap(status);
     }
 
     /** Returns the final result of this run, if it has ended. */
     public Optional<RunResult> result() {
-        return Optional.ofNullable(result);
+        // TODO jvenstad: To implement, or not ... If so, base on status.
+        throw new AssertionError();
     }
 
     /** Returns the instant at which this run began. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index ccd7434a513..7eba0a4bfb5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -1,8 +1,14 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
+/**
+ * Steps that make up a deployment job. See {@link JobProfile} for preset profiles.
+ *
+ * @author jonmv
+ */
 public enum Step {
 
     /** Download and deploy the initial real application, for staging tests. */
@@ -14,58 +20,41 @@ public enum Step {
     /** Download and deploy real application, restarting services if required. */
     deployReal(installInitialReal),
 
+    /** Find test endpoints, download test-jar, and assemble and deploy tester application. */
+    deployTester(deployReal), // TODO jvenstad: Move this up when config can be POSTed.
+
     /** See that real application has had its nodes converge to the wanted version and generation. */
     installReal(deployReal),
 
-    /** Find test endpoints, download test-jar, and assemble and deploy tester application. */
-    deployTester(deployReal),
-
     /** See that tester is done deploying, and is ready to serve. */
     installTester(deployTester),
 
     /** Ask the tester to run its tests. */
     runTests(installReal, installTester),
 
-    /** Download data from the tester, and store it. */
+    /** Download data from the tester and store it. */
     storeData(runTests),
 
-    /** Deactivate the tester, and the real deployment if test or staging environment. */
-    tearDown(storeData);
+    /** Delete the real application -- used for test deployments. */
+    deactivateReal(deployInitialReal, deployReal, runTests),
+
+    /** Deactivate the tester. */
+    deactivateTester(deployTester, storeData);
 
 
     private final List<Step> prerequisites;
 
     Step(Step... prerequisites) {
-        this.prerequisites = Arrays.asList(prerequisites);
-        // Hmm ... Need to pick out only the relevant prerequisites, and to allow storeData and tearDown to always run.
+        this.prerequisites = Collections.unmodifiableList(Arrays.asList(prerequisites));
     }
 
-
-    public enum Profile {
-
-        systemTest(deployReal, installReal, deployTester, installTester, runTests),
-
-        stagingTest,
-
-        productionTest;
-
-
-        private final List<Step> steps;
-
-        Profile(Step... steps) {
-            this.steps = Arrays.asList(steps);
-        }
-
-    }
+    public List<Step> prerequisites() { return prerequisites; }
 
 
     public enum Status {
 
-        /** Step is waiting for its prerequisites to succeed. */
-        pending,
-
-        /** Step is currently running. */
-        running,
+        /** Step is waiting for its prerequisites to succeed, or is running. */
+        unfinished,
 
         /** Step failed, and subsequent steps can not start. */
         failed,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java
index 14acaf97fa8..2e6ff6c77d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java
@@ -1,19 +1,9 @@
-package com.yahoo.vespa.hosted.controller.deployment;
+package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.IntStream;
-
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.aborted;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
 
 /**
  * Advances a given job run by running the appropriate {@link Step}s, based on their current status.
@@ -24,52 +14,18 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe
  *
  * @author jonmv
  */
-public class JobRunner {
+public interface StepRunner {
 
     /**
      * Attempts to run the given step, and returns the new status.
-     *
-     * If the step fails,
      */
-    RunStatus run(Step step, RunStatus run) {
+    default RunStatus run(Step step, RunStatus run) {
         switch (step) {
             default: throw new AssertionError();
         }
     }
 
-    private Step.Status deployInitialReal(ApplicationId id, JobType type) {
-        throw new AssertionError();
-    }
-
-    /**
-     * Attempts to advance the given job run by running the first eligible step, and returns the new status.
-     *
-     * Only the first unfinished step is attempted, to split the jobs into the smallest possible chunks, in case
-     * of sudden shutdown, etc..
-     */
-    public RunStatus advance(RunStatus run, Instant now) {
-        // If the run has failed, run any remaining alwaysRun steps, and return.
-        if (run.status().values().contains(failed))
-            return JobProfile.of(run.id().type()).alwaysRun().stream()
-                             .filter(step -> run.status().get(step) == unfinished)
-                             .findFirst()
-                             .map(step -> run(step, run))
-                             .orElse(run.with(now));
-
-        // Otherwise, try to run the first unfinished step.
-        return run.status().entrySet().stream()
-                  .filter(entry ->    entry.getValue() == unfinished
-                                   && entry.getKey().prerequisites().stream()
-                                           .filter(run.status().keySet()::contains)
-                                           .map(run.status()::get)
-                                           .allMatch(succeeded::equals))
-                  .findFirst()
-                  .map(entry -> run(entry.getKey(), run))
-                  .orElse(run.with(now));
-    }
-
-    RunStatus forceEnd(RunStatus run) {
-        // Run each pending alwaysRun step.
+    default Step.Status deployInitialReal(ApplicationId id, JobType type) {
         throw new AssertionError();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 22a6233ebe3..6ee9b15e049 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -8,12 +8,16 @@ import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.log.event.Collection;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
@@ -55,6 +59,7 @@ public class CuratorDb {
     private static final Path lockRoot = root.append("locks");
     private static final Path tenantRoot = root.append("tenants");
     private static final Path applicationRoot = root.append("applications");
+    private static final Path jobRoot = root.append("jobs");
     private static final Path controllerRoot = root.append("controllers");
 
     private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
@@ -63,6 +68,7 @@ public class CuratorDb {
     private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer();
     private final TenantSerializer tenantSerializer = new TenantSerializer();
     private final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
+    private final JobSerializer jobSerializer = new JobSerializer();
 
     private final Curator curator;
 
@@ -103,6 +109,18 @@ public class CuratorDb {
         return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
     }
 
+    public Lock lock(ApplicationId id, JobType type) {
+        return lock(lockPath(id, type), defaultLockTimeout);
+    }
+
+    public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException {
+        return tryLock(lockPath(id, type, step));
+    }
+
+    public Lock lockActiveRuns() {
+        return lock(lockRoot.append("activeRuns"), defaultLockTimeout);
+    }
+
     public Lock lockRotations() {
         return lock(lockRoot.append("rotations"), defaultLockTimeout);
     }
@@ -290,6 +308,39 @@ public class CuratorDb {
         curator.delete(applicationPath(application));
     }
 
+    // -------------- Job Runs ------------------------------------------------
+
+    public void writeActiveRun(RunStatus run) {
+        appendRun(run, activeRunsPath());
+    }
+
+    public void writeHistoricRun(RunStatus run) {
+        appendRun(run, jobRunPath(run.id().application(), run.id().type()));
+    }
+
+    public List<RunStatus> readActiveRuns() {
+        return Collections.unmodifiableList(readRuns(activeRunsPath()));
+    }
+
+    public List<RunStatus> readHistoricRuns(ApplicationId id, JobType type) {
+        // TODO jvenstad: Add, somewhere, a retention filter based on age or count.
+        return Collections.unmodifiableList(readRuns(jobRunPath(id, type)));
+    }
+
+    private void appendRun(RunStatus run, Path runsPath) {
+        List<RunStatus> runs = readRuns(runsPath);
+        runs.add(run);
+        writeRuns(runsPath, runs);
+    }
+
+    private List<RunStatus> readRuns(Path runsPath) {
+        return readSlime(runsPath).map(jobSerializer::fromSlime).orElse(Collections.emptyList());
+    }
+
+    private void writeRuns(Path runsPaths, Iterable<RunStatus> runs) {
+        curator.set(runsPaths, asJson(jobSerializer.toSlime(runs)));
+    }
+
     // -------------- Provisioning (called by internal code) ------------------
 
     @SuppressWarnings("unused")
@@ -345,6 +396,27 @@ public class CuratorDb {
         return lockPath;
     }
 
+    private Path lockPath(ApplicationId application, JobType type) {
+        Path lockPath = lockRoot
+                .append(application.tenant().value())
+                .append(application.application().value())
+                .append(application.instance().value())
+                .append(type.jobName());
+        curator.create(lockPath);
+        return lockPath;
+    }
+
+    private Path lockPath(ApplicationId application, JobType type, Step step) {
+        Path lockPath = lockRoot
+                .append(application.tenant().value())
+                .append(application.application().value())
+                .append(application.instance().value())
+                .append(type.jobName())
+                .append(step.name());
+        curator.create(lockPath);
+        return lockPath;
+    }
+
     private Path lockPath(String provisionId) {
         Path lockPath = lockRoot
                 .append(provisionStatePath())
@@ -393,6 +465,14 @@ public class CuratorDb {
         return applicationRoot.append(application.serializedForm());
     }
 
+    private static Path jobRunPath(ApplicationId id, JobType type) {
+        return jobRoot.append(id.serializedForm()).append(type.jobName());
+    }
+
+    private static Path activeRunsPath() {
+        return jobRoot.append("active");
+    }
+
     private static Path controllerPath(String hostname) {
         return controllerRoot.append(hostname);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java
index 4b13c9acc20..9f56988382c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java
@@ -15,7 +15,8 @@ public class JobSerializer {
         List<RunStatus> runs = new ArrayList<>();
         Inspector runArray = slime.get();
         runArray.traverse((ArrayTraverser) (__, runObject) ->
-                         runs.add(runFromSlime(runObject)));
+                runs.add(runFromSlime(runObject)));
+
         return runs;
     }