From 127573ef4f908eb985e30ee3aac9fbe78556e1bb Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 21 Jun 2018 15:16:34 +0200 Subject: Renaming and skeleton for steps --- .../controller/deployment/JobController.java | 28 +++++--- .../hosted/controller/deployment/JobDetails.java | 22 ------ .../vespa/hosted/controller/deployment/JobId.java | 57 --------------- .../hosted/controller/deployment/JobMeta.java | 46 ------------ .../hosted/controller/deployment/JobOutcome.java | 31 -------- .../hosted/controller/deployment/JobState.java | 28 -------- .../hosted/controller/deployment/RunDetails.java | 23 ++++++ .../vespa/hosted/controller/deployment/RunId.java | 55 ++++++++++++++ .../hosted/controller/deployment/RunResult.java | 31 ++++++++ .../hosted/controller/deployment/RunStatus.java | 83 ++++++++++++++++++++++ .../vespa/hosted/controller/deployment/Step.java | 78 ++++++++++++++++++++ .../hosted/controller/deployment/StepRunner.java | 22 ++++++ 12 files changed, 309 insertions(+), 195 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 4da13632eef..aff60191f7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -3,7 +3,8 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import java.util.List; @@ -11,7 +12,7 @@ import java.util.List; * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. * * Keys are the {@link ApplicationId} of the real application, for which the deployment job is run, and the - * {@link ZoneId} of the real deployment to test. + * {@link JobType} of the real deployment to test. * * Although the deployment jobs are themselves applications, their IDs are not to be referenced. * @@ -35,27 +36,27 @@ public class JobController { } /** Returns a list of all application which have registered. */ - List applications() { + public List applications() { return null; } /** Returns all job types which have been run for the given application. */ - List jobs(ApplicationId application) { + public List jobs(ApplicationId application) { return null; } /** Returns a list of meta information about all known runs of the given job type. */ - List runs(ApplicationId application, ZoneId zone) { + public List runs(ApplicationId application, JobType type) { return null; } /** Returns the current status of the given job. */ - JobMeta status(JobId job) { + public RunStatus status(RunId job) { return null; } /** Returns the details for the given job. */ - JobDetails details(JobId job) { + public RunDetails details(RunId job) { return null; } @@ -66,27 +67,32 @@ public class JobController { ; } + /** Accepts and stores a new appliaction package and test jar pair, and returns the reference these will have. */ + public ApplicationVersion submit(byte[] applicationPackage, byte[] applicationTestJar) { + return ApplicationVersion.unknown; + } + /** Orders a run of the given type, and returns the id of the created job. */ - JobId run(ApplicationId application, ZoneId zone) { + public RunId run(ApplicationId application, JobType type) { return null; } // PUT: /** Stores the given details for the given job. */ - void store(JobDetails details, JobId job) { + public void store(RunDetails details, RunId job) { ; } // DELETE: /** Unregisters the given application, and deletes all associated data. */ - void unregister(ApplicationId application) { + public void unregister(ApplicationId application) { ; } /** Aborts the given job. */ - void abort(JobId job) { + public void abort(RunId job) { ; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java deleted file mode 100644 index 3c787c8314f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.ActivateResult; - -/** - * Contains details about a deployment job run. - * - * @author jonmv - */ -public class JobDetails { - - private final ActivateResult deploymentResult; - private final String convergenceLog; - private final String testLog; - - public JobDetails(ActivateResult deploymentResult, String convergenceLog, String testLog) { - this.deploymentResult = deploymentResult; - this.convergenceLog = convergenceLog; - this.testLog = testLog; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java deleted file mode 100644 index 541494a23fc..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; - -import java.util.Objects; - -/** - * Immutable ID of a job run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public class JobId { - - private final ApplicationId application; - private final ZoneId zone; - private final long number; - - public JobId(ApplicationId application, ZoneId zone, long number) { - this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); - this.zone = Objects.requireNonNull(zone, "ZoneId cannot be null!"); - if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); - this.number = number; - } - - public ApplicationId application() { return application; } - public ZoneId zone() { return zone; } - public long number() { return number; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof JobId)) return false; - - JobId id = (JobId) o; - - if (number != id.number) return false; - if ( ! application.equals(id.application)) return false; - return zone == id.zone; - } - - @Override - public int hashCode() { - int result = application.hashCode(); - result = 31 * result + zone.hashCode(); - result = 31 * result + (int) (number ^ (number >>> 32)); - return result; - } - - @Override - public String toString() { - return "Run " + number + " in " + zone + " for " + application; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java deleted file mode 100644 index dde675402ce..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import java.time.Instant; -import java.util.Optional; - -/** - * Contains state information for a deployment job run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public class JobMeta { - - private final JobId id; - private final JobState state; - private final JobOutcome outcome; - private final Instant start; - private final Instant end; - - public JobMeta(JobId id, JobState state, JobOutcome outcome, Instant start, Instant end) { - this.id = id; - this.state = state; - this.outcome = outcome; - this.start = start; - this.end = end; - } - - public JobId id() { - return id; - } - - public JobState state() { - return state; - } - - public JobOutcome outcome() { - return outcome; - } - - public Instant start() { - return start; - } - - public Optional end() { - return Optional.ofNullable(end); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java deleted file mode 100644 index caecdcffb9b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -/** - * Outcomes of jobs run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public enum JobOutcome { - - /** Deployment of the real application was rejected due to missing capacity. */ - outOfCapacity, - - /** Deployment of the real application was rejected. */ - deploymentFailed, - - /** Convergence of the real application timed out. */ - convergenceFailed, - - /** Real application was deployed, but the tester application was not. */ - testError, - - /** Real application was deployed, but the tests failed. */ - testFailure, - - /** Deployment and tests completed with great success! */ - success, - - /** Job completed abnormally, due to user intervention or unexpected system error. */ - aborted - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java deleted file mode 100644 index 19e575efaf8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -/** - * Status of jobs run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public enum JobState { - - /** Job is not currently running, and may be started. */ - idle, - - /** Real application is deploying. */ - deploying, - - /** Real application is converging. */ - converging, - - /** Tester is starting up, but is not yet ready to serve its status. */ - initializing, - - /** Job is up and running normally. */ - running, - - /** Tests are complete, and results may be fetched. */ - finished - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java new file mode 100644 index 00000000000..300452ac0a8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; + +/** + * Contains details about a deployment job run. + * + * @author jonmv + */ +public class RunDetails { + + private final PrepareResponse deploymentResult; + private final String convergenceLog; + private final String testLog; + + public RunDetails(PrepareResponse deploymentResult, String convergenceLog, String testLog) { + this.deploymentResult = deploymentResult; + this.convergenceLog = convergenceLog; + this.testLog = testLog; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java new file mode 100644 index 00000000000..d78dbd6e636 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Objects; + +/** + * Immutable ID of a job run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public class RunId { + + private final ApplicationId application; + private final JobType type; + private final long number; + + public RunId(ApplicationId application, JobType type, long number) { + this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); + this.type = Objects.requireNonNull(type, "JobType cannot be null!"); + if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); + this.number = number; + } + + public ApplicationId application() { return application; } + public JobType type() { return type; } + public long number() { return number; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof RunId)) return false; + + RunId id = (RunId) o; + + if (number != id.number) return false; + if ( ! application.equals(id.application)) return false; + return type == id.type; + } + + @Override + public int hashCode() { + int result = application.hashCode(); + result = 31 * result + type.hashCode(); + result = 31 * result + (int) (number ^ (number >>> 32)); + return result; + } + + @Override + public String toString() { + return "Run " + number + " of " + type + " for " + application; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java new file mode 100644 index 00000000000..9bdf0c76d14 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +/** + * Outcomes of jobs run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public enum RunResult { + + /** Deployment of the real application was rejected due to missing capacity. */ + outOfCapacity, + + /** Deployment of the real application was rejected. */ + deploymentFailed, + + /** Convergence of the real application timed out. */ + convergenceFailed, + + /** Real application was deployed, but the tester application was not. */ + testError, + + /** Real application was deployed, but the tests failed. */ + testFailure, + + /** Deployment and tests completed with great success! */ + success, + + /** Job completed abnormally, due to user intervention or unexpected system error. */ + aborted + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java new file mode 100644 index 00000000000..f200ca4f82d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -0,0 +1,83 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.Stack; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.pending; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +/** + * Contains state information for a deployment job run by an {@link InternalBuildService}. + * + * Immutable. + * + * @author jonmv + */ +public class RunStatus { + + private final RunId id; + private final Map status; + private final Set alwaysRun; + private final RunResult result; + private final Instant start; + private final Instant end; + + RunStatus(RunId id, Map status, Set alwaysRun, RunResult result, Instant start, Instant end) { + this.id = id; + this.status = status; + this.alwaysRun = alwaysRun; + this.result = result; + this.start = start; + this.end = end; + } + + public static RunStatus initial(RunId id, Set runWhileSuccess, Set alwaysRun, Instant now) { + ImmutableMap.Builder status = ImmutableMap.builder(); + runWhileSuccess.forEach(step -> status.put(step, pending)); + alwaysRun.forEach(step -> status.put(step, pending)); + return new RunStatus(requireNonNull(id), status.build(), alwaysRun, null, requireNonNull(now), null); + } + + public RunStatus with(Step.Status update, Step step) { + return new RunStatus(id, ImmutableMap.builder().putAll(status).put(step, update).build(), alwaysRun, result, start, end); + } + + /** Returns the id of this run. */ + public RunId id() { + return id; + } + + /** Returns the status of all steps in this run. */ + public Map status() { + return status; + } + + /** Returns the final result of this run, if it has ended. */ + public Optional result() { + return Optional.ofNullable(result); + } + + /** Returns the instant at which this run began. */ + public Instant start() { + return start; + } + + /** Returns the instant at which this run ended, if it has. */ + public Optional end() { + return Optional.ofNullable(end); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java new file mode 100644 index 00000000000..ccd7434a513 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -0,0 +1,78 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import java.util.Arrays; +import java.util.List; + +public enum Step { + + /** Download and deploy the initial real application, for staging tests. */ + deployInitialReal, + + /** See that the real application has had its nodes converge to the initial state. */ + installInitialReal(deployInitialReal), + + /** Download and deploy real application, restarting services if required. */ + deployReal(installInitialReal), + + /** See that real application has had its nodes converge to the wanted version and generation. */ + installReal(deployReal), + + /** Find test endpoints, download test-jar, and assemble and deploy tester application. */ + deployTester(deployReal), + + /** See that tester is done deploying, and is ready to serve. */ + installTester(deployTester), + + /** Ask the tester to run its tests. */ + runTests(installReal, installTester), + + /** Download data from the tester, and store it. */ + storeData(runTests), + + /** Deactivate the tester, and the real deployment if test or staging environment. */ + tearDown(storeData); + + + private final List prerequisites; + + Step(Step... prerequisites) { + this.prerequisites = Arrays.asList(prerequisites); + // Hmm ... Need to pick out only the relevant prerequisites, and to allow storeData and tearDown to always run. + } + + + public enum Profile { + + systemTest(deployReal, installReal, deployTester, installTester, runTests), + + stagingTest, + + productionTest; + + + private final List steps; + + Profile(Step... steps) { + this.steps = Arrays.asList(steps); + } + + } + + + public enum Status { + + /** Step is waiting for its prerequisites to succeed. */ + pending, + + /** Step is currently running. */ + running, + + /** Step failed, and subsequent steps can not start. */ + failed, + + /** Step succeeded, and subsequent steps may not start. */ + succeeded; + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java new file mode 100644 index 00000000000..5e1b8d904c6 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java @@ -0,0 +1,22 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; + +/** + * Executor which runs given {@link Step}s, for given {@link ApplicationId} and {@link JobType} combinations. + * + * @author jonmv + */ +public class StepRunner { + + /** Returns the new status of the given step for the implied job run. */ + Step.Status run(Step step, ApplicationId application, JobType jobType) { + switch (step) { + default: return succeeded; + } + } + +} -- cgit v1.2.3 From 39673be2199068018cd68b6cea2e055be56475b9 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 21 Jun 2018 15:26:14 +0200 Subject: Expose JobController in Controller --- .../src/main/java/com/yahoo/vespa/hosted/controller/Controller.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c90ab5d19ba..631125d9368 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -141,6 +141,9 @@ public class Controller extends AbstractComponent { /** Returns the instance controlling applications */ public ApplicationController applications() { return applicationController; } + /** Returns the instance controlling deployment jobs. */ + public JobController jobController() { return jobController; } + public List getDomainList(String prefix) { return athenzClientFactory.createZmsClientWithServicePrincipal().getDomainList(prefix); } -- cgit v1.2.3 From 3dee94396f87eae9ec5c421a407333b01d1c7853 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 22 Jun 2018 12:14:32 +0200 Subject: Add setting for where to build stuff --- .../controller/api/integration/LogStore.java | 4 ++ .../yahoo/vespa/hosted/controller/Application.java | 2 +- .../vespa/hosted/controller/LockedApplication.java | 6 +++ .../controller/application/DeploymentJobs.java | 25 ++++++++---- .../controller/deployment/JobController.java | 46 +++++++++++----------- .../persistence/ApplicationSerializer.java | 5 ++- .../controller/persistence/JobSerializer.java | 37 +++++++++++++++++ .../persistence/ApplicationSerializerTest.java | 2 +- 8 files changed, 92 insertions(+), 35 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java index a4093bc80af..07cdff9a9f4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -4,6 +4,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +/** + * @author freva + */ public interface LogStore { /** @return the test log of the given deployment job. */ @@ -26,4 +29,5 @@ public interface LogStore { /** Deletes all data associated with test of a given deployment job */ void deleteTestData(ApplicationId applicationId, JobType jobType, long buildId); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 9208537dd98..677f2363c08 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -53,7 +53,7 @@ public class Application { /** Creates an empty application */ public Application(ApplicationId id) { this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(), - new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty()), + new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), new ApplicationMetrics(0, 0), Optional.empty()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 79e7fa0295a..2209cdf3013 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -89,6 +89,12 @@ public class LockedApplication { outstandingChange, ownershipIssueId, metrics, rotation); } + public LockedApplication withBuiltInternally(boolean builtInternally) { + return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, + deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, + ownershipIssueId, metrics, rotation); + } + public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 4968e161a35..65ba7e68d31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -27,19 +27,22 @@ public class DeploymentJobs { private final OptionalLong projectId; private final ImmutableMap status; private final Optional issueId; + private final boolean builtInternally; public DeploymentJobs(OptionalLong projectId, Collection jobStatusEntries, - Optional issueId) { - this(projectId, asMap(jobStatusEntries), issueId); + Optional issueId, boolean builtInternally) { + this(projectId, asMap(jobStatusEntries), issueId, builtInternally); } - private DeploymentJobs(OptionalLong projectId, Map status, Optional issueId) { + private DeploymentJobs(OptionalLong projectId, Map status, Optional issueId, + boolean builtInternally) { requireId(projectId, "projectId must be a positive integer"); Objects.requireNonNull(status, "status cannot be null"); Objects.requireNonNull(issueId, "issueId cannot be null"); this.projectId = projectId; this.status = ImmutableMap.copyOf(status); this.issueId = issueId; + this.builtInternally = builtInternally; } private static Map asMap(Collection jobStatusEntries) { @@ -56,7 +59,7 @@ public class DeploymentJobs { if (job == null) job = JobStatus.initial(jobType); return job.withCompletion(completion, jobError); }); - return new DeploymentJobs(OptionalLong.of(projectId), status, issueId); + return new DeploymentJobs(OptionalLong.of(projectId), status, issueId, builtInternally); } public DeploymentJobs withTriggering(JobType jobType, JobStatus.JobRun jobRun) { @@ -65,21 +68,25 @@ public class DeploymentJobs { if (job == null) job = JobStatus.initial(jobType); return job.withTriggering(jobRun); }); - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); } public DeploymentJobs withProjectId(OptionalLong projectId) { - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); } public DeploymentJobs with(IssueId issueId) { - return new DeploymentJobs(projectId, status, Optional.ofNullable(issueId)); + return new DeploymentJobs(projectId, status, Optional.ofNullable(issueId), builtInternally); } public DeploymentJobs without(JobType job) { Map status = new HashMap<>(this.status); status.remove(job); - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); + } + + public DeploymentJobs withBuiltInternally(boolean builtInternally) { + return new DeploymentJobs(projectId, status, issueId, builtInternally); } /** Returns an immutable map of the status entries in this */ @@ -107,6 +114,8 @@ public class DeploymentJobs { public Optional issueId() { return issueId; } + public boolean builtInternally() { return builtInternally; } + private static OptionalLong requireId(OptionalLong id, String message) { Objects.requireNonNull(id, message); if ( ! id.isPresent()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index aff60191f7f..dc339dd9330 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -28,11 +28,11 @@ public class JobController { this.logs = logStore; } - -// GET: /** Returns whether the given application has registered with this build service. */ - boolean builds(ApplicationId application) { - return false; + public boolean builds(ApplicationId id) { + return controller.applications().get(id) + .map(application -> application.deploymentJobs().builtInternally()) + .orElse(false); } /** Returns a list of all application which have registered. */ @@ -41,58 +41,56 @@ public class JobController { } /** Returns all job types which have been run for the given application. */ - public List jobs(ApplicationId application) { + public List jobs(ApplicationId id) { return null; } /** Returns a list of meta information about all known runs of the given job type. */ - public List runs(ApplicationId application, JobType type) { + public List runs(ApplicationId id, JobType type) { return null; } /** Returns the current status of the given job. */ - public RunStatus status(RunId job) { + public RunStatus status(RunId id) { return null; } /** Returns the details for the given job. */ - public RunDetails details(RunId job) { + public RunDetails details(RunId id) { return null; } - -// POST: /** Registers the given application, such that it may have deployment jobs run here. */ - void register(ApplicationId application) { - ; + void register(ApplicationId id) { + controller.applications().lockIfPresent(id, application -> + controller.applications().store(application.withBuiltInternally(true))); } /** Accepts and stores a new appliaction package and test jar pair, and returns the reference these will have. */ public ApplicationVersion submit(byte[] applicationPackage, byte[] applicationTestJar) { + + // TODO jvenstad: Return versions with increasing numbers. + return ApplicationVersion.unknown; } /** Orders a run of the given type, and returns the id of the created job. */ - public RunId run(ApplicationId application, JobType type) { + public RunId run(ApplicationId id, JobType type) { return null; } + /** Unregisters the given application, and deletes all associated data. */ + public void unregister(ApplicationId id) { + controller.applications().lockIfPresent(id, application -> { -// PUT: - /** Stores the given details for the given job. */ - public void store(RunDetails details, RunId job) { - ; - } - + // TODO jvenstad: Clean out data for jobs. -// DELETE: - /** Unregisters the given application, and deletes all associated data. */ - public void unregister(ApplicationId application) { - ; + controller.applications().store(application.withBuiltInternally(false)); + }); } /** Aborts the given job. */ - public void abort(RunId job) { + public void abort(RunId id) { ; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index d804afdf98e..763d26834e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -80,6 +80,7 @@ public class ApplicationSerializer { private final String projectIdField = "projectId"; private final String jobStatusField = "jobStatus"; private final String issueIdField = "jiraIssueId"; + private final String builtInternallyField = "builtInternally"; // JobStatus field private final String jobTypeField = "jobType"; @@ -227,6 +228,7 @@ public class ApplicationSerializer { deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value())); + cursor.setBool(builtInternallyField, deploymentJobs.builtInternally()); } private void jobStatusToSlime(Collection jobStatuses, Cursor jobStatusArray) { @@ -374,8 +376,9 @@ public class ApplicationSerializer { OptionalLong projectId = optionalLong(object.field(projectIdField)); List jobStatusList = jobStatusListFromSlime(object.field(jobStatusField)); Optional issueId = optionalString(object.field(issueIdField)).map(IssueId::from); + boolean builtInternally = object.field(builtInternallyField).asBool(); - return new DeploymentJobs(projectId, jobStatusList, issueId); + return new DeploymentJobs(projectId, jobStatusList, issueId, builtInternally); } private Change changeFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java new file mode 100644 index 00000000000..4b13c9acc20 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; + +import java.util.ArrayList; +import java.util.List; + +public class JobSerializer { + + public List fromSlime(Slime slime) { + List runs = new ArrayList<>(); + Inspector runArray = slime.get(); + runArray.traverse((ArrayTraverser) (__, runObject) -> + runs.add(runFromSlime(runObject))); + return runs; + } + + private RunStatus runFromSlime(Inspector runObject) { + throw new AssertionError(); + } + + public Slime toSlime(Iterable runs) { + Slime slime = new Slime(); + Cursor runArray = slime.setArray(); + runs.forEach(run -> toSlime(run, runArray.addObject())); + return slime; + } + + private Slime toSlime(RunStatus run, Cursor runObject) { + throw new AssertionError(); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index e96d41f5b44..a17584f9bfa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -87,7 +87,7 @@ public class ApplicationSerializerTest { .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, deployments.stream().findFirst(), "Test 3", Instant.ofEpochMilli(6)) .withCompletion(11, empty(), Instant.ofEpochMilli(7))); - DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty()); + DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true); Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, -- cgit v1.2.3 From 61e0edbca9756906f1cae34c66365ce62269bca5 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 25 Jun 2018 09:27:52 +0200 Subject: Avoid hiding unexpected lock timeouts --- .../hosted/controller/deployment/JobProfile.java | 5 ++ .../hosted/controller/deployment/LockedStep.java | 5 ++ .../hosted/controller/deployment/StepRunner.java | 22 ------- .../hosted/controller/maintenance/Maintainer.java | 3 +- .../hosted/controller/maintenance/StepRunner.java | 76 ++++++++++++++++++++++ .../hosted/controller/persistence/CuratorDb.java | 22 +++++-- 6 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java new file mode 100644 index 00000000000..f0093d20f56 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -0,0 +1,5 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +public class JobProfile { + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java new file mode 100644 index 00000000000..cc2e46e5132 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -0,0 +1,5 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +public class LockedStep { + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java deleted file mode 100644 index 5e1b8d904c6..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; - -/** - * Executor which runs given {@link Step}s, for given {@link ApplicationId} and {@link JobType} combinations. - * - * @author jonmv - */ -public class StepRunner { - - /** Returns the new status of the given step for the implied job run. */ - Step.Status run(Step step, ApplicationId application, JobType jobType) { - switch (step) { - default: return succeeded; - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java index 40563c4cf95..f6ccbf6aa4e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -12,6 +12,7 @@ import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -50,7 +51,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } } } - catch (UncheckedTimeoutException e) { + catch (TimeoutException e) { // another controller instance is running this job at the moment; ok } catch (Throwable t) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java new file mode 100644 index 00000000000..14acaf97fa8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java @@ -0,0 +1,76 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.time.Instant; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Map; +import java.util.Optional; +import java.util.stream.IntStream; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; + +/** + * Advances a given job run by running the appropriate {@link Step}s, based on their current status. + * + * When an attempt is made to advance a given job, a lock for that job (application and type) is + * taken, and released again only when the attempt finishes. Multiple other attempts may be made in + * the meantime, but they should give up unless the lock is promptly acquired. + * + * @author jonmv + */ +public class JobRunner { + + /** + * Attempts to run the given step, and returns the new status. + * + * If the step fails, + */ + RunStatus run(Step step, RunStatus run) { + switch (step) { + default: throw new AssertionError(); + } + } + + private Step.Status deployInitialReal(ApplicationId id, JobType type) { + throw new AssertionError(); + } + + /** + * Attempts to advance the given job run by running the first eligible step, and returns the new status. + * + * Only the first unfinished step is attempted, to split the jobs into the smallest possible chunks, in case + * of sudden shutdown, etc.. + */ + public RunStatus advance(RunStatus run, Instant now) { + // If the run has failed, run any remaining alwaysRun steps, and return. + if (run.status().values().contains(failed)) + return JobProfile.of(run.id().type()).alwaysRun().stream() + .filter(step -> run.status().get(step) == unfinished) + .findFirst() + .map(step -> run(step, run)) + .orElse(run.with(now)); + + // Otherwise, try to run the first unfinished step. + return run.status().entrySet().stream() + .filter(entry -> entry.getValue() == unfinished + && entry.getKey().prerequisites().stream() + .filter(run.status().keySet()::contains) + .map(run.status()::get) + .allMatch(succeeded::equals)) + .findFirst() + .map(entry -> run(entry.getKey(), run)) + .orElse(run.with(now)); + } + + RunStatus forceEnd(RunStatus run) { + // Run each pending alwaysRun step. + throw new AssertionError(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 184ac90691a..09daaef2af6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.component.Vtag; @@ -31,6 +32,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Level; @@ -113,11 +115,8 @@ public class CuratorDb { return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout); } - public Lock lockMaintenanceJob(String jobName) { - // Use a short timeout such that if maintenance jobs are started at about the same time on different nodes - // and the maintenance job takes a long time to complete, only one of the nodes will run the job - // in each maintenance interval - return lock(lockRoot.append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1)); + public Lock lockMaintenanceJob(String jobName) throws TimeoutException { + return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); } @SuppressWarnings("unused") // Called by internal code @@ -137,6 +136,19 @@ public class CuratorDb { // -------------- Helpers ------------------------------------------ + /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. + * + * Useful for maintenance jobs, where there is no point in running the jobs back to back. + */ + private Lock tryLock(Path path) throws TimeoutException { + try { + return lock(path, Duration.ofSeconds(1)); + } + catch (UncheckedTimeoutException e) { + throw new TimeoutException(e.getMessage()); + } + } + private Optional read(Path path, Function mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } -- cgit v1.2.3 From b3a8d1bafbafa156468f374b459a18d546bd4aae Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 25 Jun 2018 10:39:04 +0200 Subject: Move timeout definitions to CuratorDb --- .../api/integration/deployment/RunId.java | 55 ++++++++++++++++++++++ .../hosted/controller/ApplicationController.java | 2 +- .../vespa/hosted/controller/TenantController.java | 2 +- .../vespa/hosted/controller/deployment/RunId.java | 55 ---------------------- .../hosted/controller/persistence/CuratorDb.java | 8 ++-- 5 files changed, 61 insertions(+), 61 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java new file mode 100644 index 00000000000..d78dbd6e636 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Objects; + +/** + * Immutable ID of a job run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public class RunId { + + private final ApplicationId application; + private final JobType type; + private final long number; + + public RunId(ApplicationId application, JobType type, long number) { + this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); + this.type = Objects.requireNonNull(type, "JobType cannot be null!"); + if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); + this.number = number; + } + + public ApplicationId application() { return application; } + public JobType type() { return type; } + public long number() { return number; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof RunId)) return false; + + RunId id = (RunId) o; + + if (number != id.number) return false; + if ( ! application.equals(id.application)) return false; + return type == id.type; + } + + @Override + public int hashCode() { + int result = application.hashCode(); + result = 31 * result + type.hashCode(); + result = 31 * result + (int) (number ^ (number >>> 32)); + return result; + } + + @Override + public String toString() { + return "Run " + number + " of " + type + " for " + application; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 57708bfc89c..d3aa051088c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -599,7 +599,7 @@ public class ApplicationController { * and store the application, and finally release (close) the lock. */ Lock lock(ApplicationId application) { - return curator.lock(application, Duration.ofMinutes(10)); + return curator.lock(application)); } /** Verify that each of the production zones listed in the deployment spec exist in this system. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 2e8fe795fb5..bd746a2fa8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -203,7 +203,7 @@ public class TenantController { * and store the tenant, and finally release (close) the lock. */ private Lock lock(TenantName tenant) { - return curator.lock(tenant, Duration.ofMinutes(10)); + return curator.lock(tenant); } private static boolean inDomain(Tenant tenant, AthenzDomain domain) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java deleted file mode 100644 index d78dbd6e636..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunId.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.util.Objects; - -/** - * Immutable ID of a job run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public class RunId { - - private final ApplicationId application; - private final JobType type; - private final long number; - - public RunId(ApplicationId application, JobType type, long number) { - this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); - this.type = Objects.requireNonNull(type, "JobType cannot be null!"); - if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); - this.number = number; - } - - public ApplicationId application() { return application; } - public JobType type() { return type; } - public long number() { return number; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof RunId)) return false; - - RunId id = (RunId) o; - - if (number != id.number) return false; - if ( ! application.equals(id.application)) return false; - return type == id.type; - } - - @Override - public int hashCode() { - int result = application.hashCode(); - result = 31 * result + type.hashCode(); - result = 31 * result + (int) (number ^ (number >>> 32)); - return result; - } - - @Override - public String toString() { - return "Run " + number + " of " + type + " for " + application; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 09daaef2af6..22a6233ebe3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -95,12 +95,12 @@ public class CuratorDb { return lock; } - public Lock lock(TenantName name, Duration timeout) { - return lock(lockPath(name), timeout); + public Lock lock(TenantName name) { + return lock(lockPath(name), defaultLockTimeout.multipliedBy(2)); } - public Lock lock(ApplicationId id, Duration timeout) { - return lock(lockPath(id), timeout); + public Lock lock(ApplicationId id) { + return lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); } public Lock lockRotations() { -- cgit v1.2.3 From 12cded316027010b5279a44a4acef40d75a4f032 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 25 Jun 2018 10:43:50 +0200 Subject: Daft draft to share -- don't stare --- .../controller/api/integration/LogStore.java | 17 ++-- .../api/integration/deployment/RunId.java | 5 +- .../controller/deployment/JobController.java | 96 +++++++++++++++++++--- .../hosted/controller/deployment/JobProfile.java | 68 ++++++++++++++- .../hosted/controller/deployment/LockedStep.java | 7 ++ .../hosted/controller/deployment/RunResult.java | 4 +- .../hosted/controller/deployment/RunStatus.java | 52 +++++------- .../vespa/hosted/controller/deployment/Step.java | 49 +++++------ .../hosted/controller/maintenance/StepRunner.java | 56 ++----------- .../hosted/controller/persistence/CuratorDb.java | 80 ++++++++++++++++++ .../controller/persistence/JobSerializer.java | 3 +- 11 files changed, 300 insertions(+), 137 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java index 07cdff9a9f4..959ac2b8680 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -1,8 +1,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; /** * @author freva @@ -10,24 +9,24 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; public interface LogStore { /** @return the test log of the given deployment job. */ - String getTestLog(ApplicationId applicationId, JobType jobType, long buildId); + String getTestLog(RunId id); /** Stores the given test log for the given deployment job. */ - void setTestLog(ApplicationId applicationId, JobType jobType, long buildId, String testLog); + void setTestLog(RunId id, String testLog); /** @return the convergence log of the given deployment job. */ - String getConvergenceLog(ApplicationId applicationId, JobType jobType, long buildId); + String getConvergenceLog(RunId id); /** Stores the given convergence log for the given deployment job. */ - void setConvergenceLog(ApplicationId applicationId, JobType jobType, long buildId, String convergenceLog); + void setConvergenceLog(RunId id, String convergenceLog); /** @return the result of prepare of the test application for the given deployment job. */ - PrepareResponse getPrepareResponse(ApplicationId applicationId, JobType jobType, long buildId); + PrepareResponse getPrepareResponse(RunId id); /** Stores the given result of prepare of the test application for the given deployment job. */ - void setPrepareResponse(ApplicationId applicationId, JobType jobType, long buildId, PrepareResponse prepareResponse); + void setPrepareResponse(RunId id, PrepareResponse prepareResponse); /** Deletes all data associated with test of a given deployment job */ - void deleteTestData(ApplicationId applicationId, JobType jobType, long buildId); + void deleteTestData(RunId id); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java index d78dbd6e636..a46cec1bb40 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/RunId.java @@ -1,12 +1,11 @@ -package com.yahoo.vespa.hosted.controller.deployment; +package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.util.Objects; /** - * Immutable ID of a job run by an {@link InternalBuildService}. + * Immutable ID of a job run by a {@link com.yahoo.vespa.hosted.controller.api.integration.BuildService}. * * @author jonmv */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index dc339dd9330..98ec77566f7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,12 +1,19 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import java.util.stream.Stream; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -42,22 +49,67 @@ public class JobController { /** Returns all job types which have been run for the given application. */ public List jobs(ApplicationId id) { - return null; + return ImmutableList.copyOf(Stream.of(JobType.values()) + .filter(type -> ! runs(id, type).isEmpty()) + .iterator()); } - /** Returns a list of meta information about all known runs of the given job type. */ + /** Returns a list of meta information about all known runs of the given job type for the given application. */ public List runs(ApplicationId id, JobType type) { - return null; + ImmutableList.Builder runs = ImmutableList.builder(); + runs.addAll(controller.curator().readHistoricRuns(id, type)); + activeRuns().stream() + .filter(run -> run.id().application().equals(id) && run.id().type() == type) + .forEach(runs::add); + return runs.build(); } - /** Returns the current status of the given job. */ - public RunStatus status(RunId id) { - return null; + List activeRuns() { + return controller.curator().readActiveRuns(); + } + + /** Returns the updated status of the given job, if it is active. */ + public Optional currentStatus(RunId id) { + try (Lock __ = controller.curator().lockActiveRuns()) { + return activeRuns().stream() // TODO jvenstad: Change these to Map. + .filter(run -> run.id().equals(id)) + .findAny(); + } + } + + public Optional update(RunId id, Step.Status status, LockedStep step) { + return currentStatus(id).map(run -> { + run = run.with(status, step); + controller.curator().writeActiveRun(run); + return run; + }); + } + + public void locked(RunId id, Step step, Consumer action) { + try (Lock lock = controller.curator().lock(id.application(), id.type(), step)) { + for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. + try (Lock __ = controller.curator().lock(id.application(), id.type(), prerequisite)) { ; } + + action.accept(new LockedStep(lock, step)); + } + catch (TimeoutException e) { + // Something else is already running that step, or a prerequisite -- try again later! + } + } + + public void finish(RunId id) { + controller.applications().lockIfPresent(id.application(), __ -> { + currentStatus(id).ifPresent(run -> { + controller.curator().writeHistoricRun(run.with(controller.clock().instant())); + }); + }); } /** Returns the details for the given job. */ public RunDetails details(RunId id) { - return null; + try (Lock __ = controller.curator().lock(id.application(), id.type())) { + return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id)); + } } /** Registers the given application, such that it may have deployment jobs run here. */ @@ -66,16 +118,23 @@ public class JobController { controller.applications().store(application.withBuiltInternally(true))); } - /** Accepts and stores a new appliaction package and test jar pair, and returns the reference these will have. */ - public ApplicationVersion submit(byte[] applicationPackage, byte[] applicationTestJar) { + /** Accepts and stores a new appliaction package and test jar pair. */ + public void submit(ApplicationId id, byte[] applicationPackage, byte[] applicationTestJar) { + controller.applications().lockOrThrow(id, application -> { + ApplicationVersion version = nextVersion(id); - // TODO jvenstad: Return versions with increasing numbers. + // TODO smorgrav: Store the pair. - return ApplicationVersion.unknown; + notifyOfNewSubmission(id); + }); } /** Orders a run of the given type, and returns the id of the created job. */ public RunId run(ApplicationId id, JobType type) { + try (Lock __ = controller.curator().lock(id, type); + Lock ___ = controller.curator().lockActiveRuns()) { + List runs = controller.curator().readHistoricRuns(id, type); + } return null; } @@ -94,4 +153,19 @@ public class JobController { ; } + + private void advanceJobs() { + activeRuns().forEach(run -> { + + }); + } + + private ApplicationVersion nextVersion(ApplicationId id) { + throw new AssertionError(); + } + + private void notifyOfNewSubmission(ApplicationId id) { + ; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index f0093d20f56..4e8495ee10b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -1,5 +1,71 @@ package com.yahoo.vespa.hosted.controller.deployment; -public class JobProfile { +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.*; + +/** + * Static profiles defining the {@link Step}s of a deployment job. + * + * @author jonmv + */ +public enum JobProfile { + + systemTest(EnumSet.of(deployReal, + installReal, + deployTester, + installTester, + runTests), + EnumSet.of(storeData, + deactivateTester, + deactivateReal)), + + stagingTest(EnumSet.of(deployInitialReal, + installInitialReal, + deployReal, + installReal, + deployTester, + installTester, + runTests), + EnumSet.of(storeData, + deactivateTester, + deactivateReal)), + + production(EnumSet.of(deployReal, + installReal, + deployTester, + installTester, + runTests), + EnumSet.of(storeData, + deactivateTester)); + + + private final Set steps; + private final Set alwaysRun; + + JobProfile(Set runWhileSuccess, Set alwaysRun) { + runWhileSuccess.addAll(alwaysRun); + this.steps = Collections.unmodifiableSet(runWhileSuccess); + this.alwaysRun = Collections.unmodifiableSet(alwaysRun); + } + + public static JobProfile of(JobType type) { + switch (type.environment()) { + case test: return systemTest; + case staging: return stagingTest; + case prod: return production; + default: throw new IllegalArgumentException("Unexpected environment " + type.environment()); + } + } + + /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */ + public Set steps() { return steps; } + + /** Returns the set of steps that should always be run, regardless of outcome. */ + public Set alwaysRun() { return alwaysRun; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index cc2e46e5132..7586b80d228 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -1,5 +1,12 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.vespa.curator.Lock; + public class LockedStep { + private final Step step; + + LockedStep(Lock lock, Step step) { this.step = step; } + public Step get() { return step; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java index 9bdf0c76d14..aaf43097908 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java @@ -13,8 +13,8 @@ public enum RunResult { /** Deployment of the real application was rejected. */ deploymentFailed, - /** Convergence of the real application timed out. */ - convergenceFailed, + /** Installation of the real application timed out. */ + installationFailed, /** Real application was deployed, but the tester application was not. */ testError, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index f200ca4f82d..6fbbe92def9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -1,28 +1,18 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.time.Instant; -import java.util.List; +import java.util.Collections; +import java.util.EnumMap; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.Stack; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.pending; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; /** - * Contains state information for a deployment job run by an {@link InternalBuildService}. - * - * Immutable. + * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}. * * @author jonmv */ @@ -30,29 +20,30 @@ public class RunStatus { private final RunId id; private final Map status; - private final Set alwaysRun; - private final RunResult result; private final Instant start; private final Instant end; - RunStatus(RunId id, Map status, Set alwaysRun, RunResult result, Instant start, Instant end) { + RunStatus(RunId id, Map status, Instant start, Instant end) { this.id = id; this.status = status; - this.alwaysRun = alwaysRun; - this.result = result; this.start = start; this.end = end; } - public static RunStatus initial(RunId id, Set runWhileSuccess, Set alwaysRun, Instant now) { - ImmutableMap.Builder status = ImmutableMap.builder(); - runWhileSuccess.forEach(step -> status.put(step, pending)); - alwaysRun.forEach(step -> status.put(step, pending)); - return new RunStatus(requireNonNull(id), status.build(), alwaysRun, null, requireNonNull(now), null); + public static RunStatus initial(RunId id, Instant now) { + Map status = new EnumMap<>(Step.class); + JobProfile.of(id.type()).steps().forEach(step -> status.put(step, unfinished)); + return new RunStatus(requireNonNull(id), status, requireNonNull(now), null); + } + + public RunStatus with(Step.Status update, LockedStep step) { + RunStatus run = new RunStatus(id, status, start, end); + run.status.put(step.get(), update); + return run; } - public RunStatus with(Step.Status update, Step step) { - return new RunStatus(id, ImmutableMap.builder().putAll(status).put(step, update).build(), alwaysRun, result, start, end); + public RunStatus with(Instant now) { + return new RunStatus(id, status, start, now); } /** Returns the id of this run. */ @@ -60,14 +51,15 @@ public class RunStatus { return id; } - /** Returns the status of all steps in this run. */ + /** Returns an unmodifiable view of the status of all steps in this run. */ public Map status() { - return status; + return Collections.unmodifiableMap(status); } /** Returns the final result of this run, if it has ended. */ public Optional result() { - return Optional.ofNullable(result); + // TODO jvenstad: To implement, or not ... If so, base on status. + throw new AssertionError(); } /** Returns the instant at which this run began. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index ccd7434a513..7eba0a4bfb5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -1,8 +1,14 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Arrays; +import java.util.Collections; import java.util.List; +/** + * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. + * + * @author jonmv + */ public enum Step { /** Download and deploy the initial real application, for staging tests. */ @@ -14,58 +20,41 @@ public enum Step { /** Download and deploy real application, restarting services if required. */ deployReal(installInitialReal), + /** Find test endpoints, download test-jar, and assemble and deploy tester application. */ + deployTester(deployReal), // TODO jvenstad: Move this up when config can be POSTed. + /** See that real application has had its nodes converge to the wanted version and generation. */ installReal(deployReal), - /** Find test endpoints, download test-jar, and assemble and deploy tester application. */ - deployTester(deployReal), - /** See that tester is done deploying, and is ready to serve. */ installTester(deployTester), /** Ask the tester to run its tests. */ runTests(installReal, installTester), - /** Download data from the tester, and store it. */ + /** Download data from the tester and store it. */ storeData(runTests), - /** Deactivate the tester, and the real deployment if test or staging environment. */ - tearDown(storeData); + /** Delete the real application -- used for test deployments. */ + deactivateReal(deployInitialReal, deployReal, runTests), + + /** Deactivate the tester. */ + deactivateTester(deployTester, storeData); private final List prerequisites; Step(Step... prerequisites) { - this.prerequisites = Arrays.asList(prerequisites); - // Hmm ... Need to pick out only the relevant prerequisites, and to allow storeData and tearDown to always run. + this.prerequisites = Collections.unmodifiableList(Arrays.asList(prerequisites)); } - - public enum Profile { - - systemTest(deployReal, installReal, deployTester, installTester, runTests), - - stagingTest, - - productionTest; - - - private final List steps; - - Profile(Step... steps) { - this.steps = Arrays.asList(steps); - } - - } + public List prerequisites() { return prerequisites; } public enum Status { - /** Step is waiting for its prerequisites to succeed. */ - pending, - - /** Step is currently running. */ - running, + /** Step is waiting for its prerequisites to succeed, or is running. */ + unfinished, /** Step failed, and subsequent steps can not start. */ failed, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java index 14acaf97fa8..2e6ff6c77d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java @@ -1,19 +1,9 @@ -package com.yahoo.vespa.hosted.controller.deployment; +package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.time.Instant; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.Map; -import java.util.Optional; -import java.util.stream.IntStream; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; /** * Advances a given job run by running the appropriate {@link Step}s, based on their current status. @@ -24,52 +14,18 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe * * @author jonmv */ -public class JobRunner { +public interface StepRunner { /** * Attempts to run the given step, and returns the new status. - * - * If the step fails, */ - RunStatus run(Step step, RunStatus run) { + default RunStatus run(Step step, RunStatus run) { switch (step) { default: throw new AssertionError(); } } - private Step.Status deployInitialReal(ApplicationId id, JobType type) { - throw new AssertionError(); - } - - /** - * Attempts to advance the given job run by running the first eligible step, and returns the new status. - * - * Only the first unfinished step is attempted, to split the jobs into the smallest possible chunks, in case - * of sudden shutdown, etc.. - */ - public RunStatus advance(RunStatus run, Instant now) { - // If the run has failed, run any remaining alwaysRun steps, and return. - if (run.status().values().contains(failed)) - return JobProfile.of(run.id().type()).alwaysRun().stream() - .filter(step -> run.status().get(step) == unfinished) - .findFirst() - .map(step -> run(step, run)) - .orElse(run.with(now)); - - // Otherwise, try to run the first unfinished step. - return run.status().entrySet().stream() - .filter(entry -> entry.getValue() == unfinished - && entry.getKey().prerequisites().stream() - .filter(run.status().keySet()::contains) - .map(run.status()::get) - .allMatch(succeeded::equals)) - .findFirst() - .map(entry -> run(entry.getKey(), run)) - .orElse(run.with(now)); - } - - RunStatus forceEnd(RunStatus run) { - // Run each pending alwaysRun step. + default Step.Status deployInitialReal(ApplicationId id, JobType type) { throw new AssertionError(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 22a6233ebe3..6ee9b15e049 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -8,12 +8,16 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; +import com.yahoo.log.event.Collection; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; @@ -55,6 +59,7 @@ public class CuratorDb { private static final Path lockRoot = root.append("locks"); private static final Path tenantRoot = root.append("tenants"); private static final Path applicationRoot = root.append("applications"); + private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -63,6 +68,7 @@ public class CuratorDb { private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final JobSerializer jobSerializer = new JobSerializer(); private final Curator curator; @@ -103,6 +109,18 @@ public class CuratorDb { return lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); } + public Lock lock(ApplicationId id, JobType type) { + return lock(lockPath(id, type), defaultLockTimeout); + } + + public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { + return tryLock(lockPath(id, type, step)); + } + + public Lock lockActiveRuns() { + return lock(lockRoot.append("activeRuns"), defaultLockTimeout); + } + public Lock lockRotations() { return lock(lockRoot.append("rotations"), defaultLockTimeout); } @@ -290,6 +308,39 @@ public class CuratorDb { curator.delete(applicationPath(application)); } + // -------------- Job Runs ------------------------------------------------ + + public void writeActiveRun(RunStatus run) { + appendRun(run, activeRunsPath()); + } + + public void writeHistoricRun(RunStatus run) { + appendRun(run, jobRunPath(run.id().application(), run.id().type())); + } + + public List readActiveRuns() { + return Collections.unmodifiableList(readRuns(activeRunsPath())); + } + + public List readHistoricRuns(ApplicationId id, JobType type) { + // TODO jvenstad: Add, somewhere, a retention filter based on age or count. + return Collections.unmodifiableList(readRuns(jobRunPath(id, type))); + } + + private void appendRun(RunStatus run, Path runsPath) { + List runs = readRuns(runsPath); + runs.add(run); + writeRuns(runsPath, runs); + } + + private List readRuns(Path runsPath) { + return readSlime(runsPath).map(jobSerializer::fromSlime).orElse(Collections.emptyList()); + } + + private void writeRuns(Path runsPaths, Iterable runs) { + curator.set(runsPaths, asJson(jobSerializer.toSlime(runs))); + } + // -------------- Provisioning (called by internal code) ------------------ @SuppressWarnings("unused") @@ -345,6 +396,27 @@ public class CuratorDb { return lockPath; } + private Path lockPath(ApplicationId application, JobType type) { + Path lockPath = lockRoot + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()) + .append(type.jobName()); + curator.create(lockPath); + return lockPath; + } + + private Path lockPath(ApplicationId application, JobType type, Step step) { + Path lockPath = lockRoot + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()) + .append(type.jobName()) + .append(step.name()); + curator.create(lockPath); + return lockPath; + } + private Path lockPath(String provisionId) { Path lockPath = lockRoot .append(provisionStatePath()) @@ -393,6 +465,14 @@ public class CuratorDb { return applicationRoot.append(application.serializedForm()); } + private static Path jobRunPath(ApplicationId id, JobType type) { + return jobRoot.append(id.serializedForm()).append(type.jobName()); + } + + private static Path activeRunsPath() { + return jobRoot.append("active"); + } + private static Path controllerPath(String hostname) { return controllerRoot.append(hostname); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index 4b13c9acc20..9f56988382c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -15,7 +15,8 @@ public class JobSerializer { List runs = new ArrayList<>(); Inspector runArray = slime.get(); runArray.traverse((ArrayTraverser) (__, runObject) -> - runs.add(runFromSlime(runObject))); + runs.add(runFromSlime(runObject))); + return runs; } -- cgit v1.2.3 From 42875888175b81c3ccfb6490fcbaa7b50e81d6e1 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 25 Jun 2018 10:44:33 +0200 Subject: Remove parenthesis --- .../java/com/yahoo/vespa/hosted/controller/ApplicationController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index d3aa051088c..7c7fb27f3eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -599,7 +599,7 @@ public class ApplicationController { * and store the application, and finally release (close) the lock. */ Lock lock(ApplicationId application) { - return curator.lock(application)); + return curator.lock(application); } /** Verify that each of the production zones listed in the deployment spec exist in this system. */ -- cgit v1.2.3 From eff8c95a57014760a3bcbcf7d6c6ed68dfefc41e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 26 Jun 2018 15:27:18 +0200 Subject: Maybe a finished job run engine? --- .../api/integration/stubs/MockLogStore.java | 24 ++-- .../controller/deployment/JobController.java | 139 ++++++++++++--------- .../hosted/controller/deployment/JobProfile.java | 6 +- .../hosted/controller/deployment/LockedStep.java | 1 - .../hosted/controller/deployment/RunStatus.java | 67 +++++++--- .../vespa/hosted/controller/deployment/Step.java | 30 +++-- .../hosted/controller/deployment/package-info.java | 5 + .../maintenance/ControllerMaintenance.java | 3 + .../controller/maintenance/DummyStepRunner.java | 14 +++ .../hosted/controller/maintenance/JobRunner.java | 94 ++++++++++++++ .../controller/maintenance/RealStepRunner.java | 76 +++++++++++ .../hosted/controller/maintenance/StepRunner.java | 16 +-- .../hosted/controller/persistence/CuratorDb.java | 46 +++---- .../controller/persistence/JobSerializer.java | 60 +++++++-- .../controller/maintenance/JobRunnerTest.java | 39 ++++++ .../restapi/controller/responses/maintenance.json | 3 + 16 files changed, 474 insertions(+), 149 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java index 9ec547c2e0e..c5dba3b8fa7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java @@ -1,12 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.util.Collections; @@ -14,42 +13,45 @@ import java.util.Collections; * @author freva */ public class MockLogStore implements LogStore { + @Override - public String getTestLog(ApplicationId applicationId, JobType jobType, long buildId) { + public String getTestLog(RunId id) { return "SUCCESS"; } @Override - public void setTestLog(ApplicationId applicationId, JobType jobType, long buildId, String testLog) { + public void setTestLog(RunId id, String testLog) { } @Override - public String getConvergenceLog(ApplicationId applicationId, JobType jobType, long buildId) { + public String getConvergenceLog(RunId id) { return "SUCCESS"; } @Override - public void setConvergenceLog(ApplicationId applicationId, JobType jobType, long buildId, String convergenceLog) { + public void setConvergenceLog(RunId id, String convergenceLog) { } @Override - public PrepareResponse getPrepareResponse(ApplicationId applicationId, JobType jobType, long buildId) { + public PrepareResponse getPrepareResponse(RunId id) { PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.message = "foo"; prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), - Collections.emptyList()); + Collections.emptyList()); prepareResponse.tenant = new TenantId("tenant"); - return prepareResponse; } + return prepareResponse; + } @Override - public void setPrepareResponse(ApplicationId applicationId, JobType jobType, long buildId, PrepareResponse prepareResponse) { + public void setPrepareResponse(RunId id, PrepareResponse prepareResponse) { } @Override - public void deleteTestData(ApplicationId applicationId, JobType jobType, long buildId) { + public void deleteTestData(RunId id) { } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 98ec77566f7..0cde83bf230 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,20 +1,26 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeoutException; import java.util.function.Consumer; +import java.util.function.UnaryOperator; import java.util.stream.Stream; +import static com.google.common.collect.ImmutableList.copyOf; + /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. * @@ -28,10 +34,12 @@ import java.util.stream.Stream; public class JobController { private final Controller controller; + private final CuratorDb curator; private final LogStore logs; public JobController(Controller controller, LogStore logStore) { this.controller = controller; + this.curator = controller.curator(); this.logs = logStore; } @@ -44,72 +52,61 @@ public class JobController { /** Returns a list of all application which have registered. */ public List applications() { - return null; + return copyOf(controller.applications().asList().stream() + .filter(application -> application.deploymentJobs().builtInternally()) + .map(Application::id) + .iterator()); } /** Returns all job types which have been run for the given application. */ public List jobs(ApplicationId id) { - return ImmutableList.copyOf(Stream.of(JobType.values()) - .filter(type -> ! runs(id, type).isEmpty()) - .iterator()); + return copyOf(Stream.of(JobType.values()) + .filter(type -> last(id, type).isPresent()) + .iterator()); + } + + public Optional last(ApplicationId id, JobType type) { + return curator.readLastRun(id, type); } /** Returns a list of meta information about all known runs of the given job type for the given application. */ - public List runs(ApplicationId id, JobType type) { - ImmutableList.Builder runs = ImmutableList.builder(); - runs.addAll(controller.curator().readHistoricRuns(id, type)); - activeRuns().stream() - .filter(run -> run.id().application().equals(id) && run.id().type() == type) - .forEach(runs::add); + public Map runs(ApplicationId id, JobType type) { + ImmutableMap.Builder runs = ImmutableMap.builder(); + runs.putAll(curator.readHistoricRuns(id, type)); + last(id, type).ifPresent(run -> runs.put(run.id(), run)); return runs.build(); } - List activeRuns() { - return controller.curator().readActiveRuns(); + public List active() { + return copyOf(applications().stream() + .flatMap(id -> Stream.of(JobType.values()) + .map(type -> last(id, type)) + .filter(Optional::isPresent).map(Optional::get) + .filter(run -> ! run.end().isPresent())) + .iterator()); } - /** Returns the updated status of the given job, if it is active. */ - public Optional currentStatus(RunId id) { - try (Lock __ = controller.curator().lockActiveRuns()) { - return activeRuns().stream() // TODO jvenstad: Change these to Map. - .filter(run -> run.id().equals(id)) - .findAny(); - } + public Optional active(RunId id) { + return last(id.application(), id.type()) + .filter(run -> ! run.end().isPresent()) + .filter(run -> run.id().equals(id)); } - public Optional update(RunId id, Step.Status status, LockedStep step) { - return currentStatus(id).map(run -> { - run = run.with(status, step); - controller.curator().writeActiveRun(run); - return run; - }); - } - - public void locked(RunId id, Step step, Consumer action) { - try (Lock lock = controller.curator().lock(id.application(), id.type(), step)) { - for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. - try (Lock __ = controller.curator().lock(id.application(), id.type(), prerequisite)) { ; } - - action.accept(new LockedStep(lock, step)); - } - catch (TimeoutException e) { - // Something else is already running that step, or a prerequisite -- try again later! - } + public void update(RunId id, Step.Status status, LockedStep step) { + modify(id, run -> run.with(status, step)); } public void finish(RunId id) { - controller.applications().lockIfPresent(id.application(), __ -> { - currentStatus(id).ifPresent(run -> { - controller.curator().writeHistoricRun(run.with(controller.clock().instant())); - }); + modify(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. + RunStatus endedRun = run.finish(controller.clock().instant()); + modify(id.application(), id.type(), runs -> runs.put(run.id(), endedRun)); + return endedRun; }); } /** Returns the details for the given job. */ public RunDetails details(RunId id) { - try (Lock __ = controller.curator().lock(id.application(), id.type())) { - return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id)); - } + return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id)); } /** Registers the given application, such that it may have deployment jobs run here. */ @@ -130,36 +127,31 @@ public class JobController { } /** Orders a run of the given type, and returns the id of the created job. */ - public RunId run(ApplicationId id, JobType type) { - try (Lock __ = controller.curator().lock(id, type); - Lock ___ = controller.curator().lockActiveRuns()) { - List runs = controller.curator().readHistoricRuns(id, type); - } - return null; + public void run(ApplicationId id, JobType type) { + modify(id, type, runs -> { + Optional last = last(id, type); + if (last.flatMap(run -> active(run.id())).isPresent()) + throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); + + RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); + curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant())); + }); } /** Unregisters the given application, and deletes all associated data. */ public void unregister(ApplicationId id) { controller.applications().lockIfPresent(id, application -> { - - // TODO jvenstad: Clean out data for jobs. - controller.applications().store(application.withBuiltInternally(false)); }); + jobs(id).forEach(type -> modify(id, type, __ -> curator.deleteRuns(id, type))); } /** Aborts the given job. */ public void abort(RunId id) { - ; - } - - - private void advanceJobs() { - activeRuns().forEach(run -> { - }); } + private ApplicationVersion nextVersion(ApplicationId id) { throw new AssertionError(); } @@ -168,4 +160,29 @@ public class JobController { ; } + private void modify(ApplicationId id, JobType type, Consumer> modifications) { + try (Lock __ = curator.lock(id, type)) { + Map runs = curator.readHistoricRuns(id, type); + modifications.accept(runs); + curator.writeHistoricRuns(id, type, runs.values()); + } + } + + private void modify(RunId id, UnaryOperator modifications) { + try (Lock __ = curator.lock(id.application(), id.type())) { + RunStatus run = active(id).orElseThrow(() -> new IllegalArgumentException(id + " is not an active run!")); + run = modifications.apply(run); + curator.writeLastRun(run); + } + } + + public void locked(RunId id, Step step, Consumer action) throws TimeoutException { + try (Lock lock = curator.lock(id.application(), id.type(), step)) { + for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. + try (Lock __ = curator.lock(id.application(), id.type(), prerequisite)) { ; } + + action.accept(new LockedStep(lock, step)); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 4e8495ee10b..d2e0a4fe705 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -19,7 +19,7 @@ public enum JobProfile { installReal, deployTester, installTester, - runTests), + startTests), EnumSet.of(storeData, deactivateTester, deactivateReal)), @@ -30,7 +30,7 @@ public enum JobProfile { installReal, deployTester, installTester, - runTests), + startTests), EnumSet.of(storeData, deactivateTester, deactivateReal)), @@ -39,7 +39,7 @@ public enum JobProfile { installReal, deployTester, installTester, - runTests), + startTests), EnumSet.of(storeData, deactivateTester)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index 7586b80d228..1a35169488a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.curator.Lock; public class LockedStep { private final Step step; - LockedStep(Lock lock, Step step) { this.step = step; } public Step get() { return step; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 6fbbe92def9..a6aa35d1220 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -1,13 +1,17 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableList; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.time.Instant; import java.util.Collections; import java.util.EnumMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; @@ -19,31 +23,32 @@ import static java.util.Objects.requireNonNull; public class RunStatus { private final RunId id; - private final Map status; + private final Map steps; private final Instant start; - private final Instant end; + private final Optional end; - RunStatus(RunId id, Map status, Instant start, Instant end) { + // For deserialisation only -- do not use! + public RunStatus(RunId id, Map steps, Instant start, Optional end) { this.id = id; - this.status = status; + this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.start = start; this.end = end; } public static RunStatus initial(RunId id, Instant now) { - Map status = new EnumMap<>(Step.class); - JobProfile.of(id.type()).steps().forEach(step -> status.put(step, unfinished)); - return new RunStatus(requireNonNull(id), status, requireNonNull(now), null); + EnumMap steps = new EnumMap<>(Step.class); + JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); + return new RunStatus(id, steps, requireNonNull(now), Optional.empty()); } - public RunStatus with(Step.Status update, LockedStep step) { - RunStatus run = new RunStatus(id, status, start, end); - run.status.put(step.get(), update); - return run; + public RunStatus with(Step.Status status, LockedStep step) { + EnumMap steps = new EnumMap<>(this.steps); + steps.put(step.get(), requireNonNull(status)); + return new RunStatus(id, steps, start, end); } - public RunStatus with(Instant now) { - return new RunStatus(id, status, start, now); + public RunStatus finish(Instant now) { + return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now)); } /** Returns the id of this run. */ @@ -52,8 +57,8 @@ public class RunStatus { } /** Returns an unmodifiable view of the status of all steps in this run. */ - public Map status() { - return Collections.unmodifiableMap(status); + public Map steps() { + return steps; } /** Returns the final result of this run, if it has ended. */ @@ -69,7 +74,37 @@ public class RunStatus { /** Returns the instant at which this run ended, if it has. */ public Optional end() { - return Optional.ofNullable(end); + return end; + } + + public boolean hasFailed() { + return steps.values().contains(failed); + } + + public List readySteps() { + return hasFailed() ? forcedSteps() : normalSteps(); + } + + private List normalSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() == unfinished + && entry.getKey().prerequisites().stream() + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); + } + + private List forcedSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() != succeeded + && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) + && entry.getKey().prerequisites().stream() + .filter(JobProfile.of(id.type()).alwaysRun()::contains) + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 7eba0a4bfb5..240f2ada242 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -7,6 +7,18 @@ import java.util.List; /** * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. * + * Each step lists its prerequisites; this serves two purposes: + * + * 1. A step may only run after its prerequisites, so these define a topological order in which + * the steps can be run. Since a job profile may list only a subset of the existing steps, + * only the prerequisites of a step which are included in a run's profile will be considered. + * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. + * When a run has failed, however, each of the always-run steps of the run's profile will be run, + * again in a topological order, and again requiring success of all their always-run prerequisites. + * + * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant + * information from a failed run is stored, and that deployment does not occur after deactivation. + * * @author jonmv */ public enum Step { @@ -20,23 +32,23 @@ public enum Step { /** Download and deploy real application, restarting services if required. */ deployReal(installInitialReal), - /** Find test endpoints, download test-jar, and assemble and deploy tester application. */ - deployTester(deployReal), // TODO jvenstad: Move this up when config can be POSTed. - /** See that real application has had its nodes converge to the wanted version and generation. */ installReal(deployReal), + /** Find test endpoints, download test-jar, and assemble and deploy tester application. */ + deployTester(installReal), // TODO jvenstad: Move this up when config can be POSTed. + /** See that tester is done deploying, and is ready to serve. */ installTester(deployTester), /** Ask the tester to run its tests. */ - runTests(installReal, installTester), + startTests(installReal, installTester), /** Download data from the tester and store it. */ - storeData(runTests), + storeData(startTests), /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, runTests), + deactivateReal(deployInitialReal, deployReal, startTests), /** Deactivate the tester. */ deactivateTester(deployTester, storeData); @@ -53,13 +65,13 @@ public enum Step { public enum Status { - /** Step is waiting for its prerequisites to succeed, or is running. */ + /** Step still has unsatisfied finish criteria -- it may not even have started. */ unfinished, - /** Step failed, and subsequent steps can not start. */ + /** Step failed and subsequent steps may not start. */ failed, - /** Step succeeded, and subsequent steps may not start. */ + /** Step succeeded and subsequent steps may now start. */ succeeded; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java new file mode 100644 index 00000000000..e8fb638bc34 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 105435f0346..5313c03cb03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -38,6 +38,7 @@ public class ControllerMaintenance extends AbstractComponent { private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; private final DnsMaintainer dnsMaintainer; private final SystemUpgrader systemUpgrader; + private final JobRunner jobRunner; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, @@ -59,6 +60,7 @@ public class ControllerMaintenance extends AbstractComponent { applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); + jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new DummyStepRunner()); } public Upgrader upgrader() { return upgrader; } @@ -81,6 +83,7 @@ public class ControllerMaintenance extends AbstractComponent { applicationOwnershipConfirmer.deconstruct(); dnsMaintainer.deconstruct(); systemUpgrader.deconstruct(); + jobRunner.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java new file mode 100644 index 00000000000..f6f418e70be --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java @@ -0,0 +1,14 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; + +public class DummyStepRunner implements StepRunner { + + @Override + public Step.Status run(LockedStep step, RunStatus run) { + return Step.Status.succeeded; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java new file mode 100644 index 00000000000..686cd3cf4ea --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -0,0 +1,94 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import org.jetbrains.annotations.TestOnly; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; + +/** + * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}. + * + * @author jonmv + */ +public class JobRunner extends Maintainer { + + private static final Logger log = Logger.getLogger(JobRunner.class.getName()); + + private final JobController jobs; + private final ExecutorService executors; + private final StepRunner runner; + + public JobRunner(Controller controller, Duration duration, JobControl jobControl, StepRunner runner) { + this(controller, duration, jobControl, Executors.newFixedThreadPool(32), runner); + } + + @TestOnly + JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) { + super(controller, duration, jobControl); + this.jobs = controller.jobController(); + this.executors = executors; + this.runner = runner; + } + + @Override + protected void maintain() { + jobs.active().forEach(this::advance); + } + + @Override + public void deconstruct() { + super.deconstruct(); + executors.shutdown(); + try { + executors.awaitTermination(50, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. */ + void advance(RunStatus run) { + List steps = run.readySteps(); + steps.forEach(step -> executors.execute(() -> advance(run.id(), step))); + if (steps.isEmpty()) + jobs.finish(run.id()); + } + + /** Attempts to advance the status of the given step, for the given run. */ + void advance(RunId id, Step step) { + try { + jobs.locked(id, step, lockedStep -> { + jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we should bail out. + if ( ! run.readySteps().contains(step)) + return; // Someone may have updated the run status, making this step obsolete, so we bail out. + + Step.Status status = runner.run(lockedStep, run); + if (run.steps().get(step) != status) { + jobs.update(run.id(), status, lockedStep); + advance(run); + } + }); + }); + } + catch (TimeoutException e) { + // Something else is already advancing this step, or a prerequisite -- try again later! + } + catch (RuntimeException e) { + log.log(LogLevel.WARNING, "Exception attempting to advance " + step + " of " + id, e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java new file mode 100644 index 00000000000..55dbda1401b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -0,0 +1,76 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; + +public class RealStepRunner implements StepRunner { + + private final ApplicationController applications; + + public RealStepRunner(ApplicationController applications) { + this.applications = applications; + } + + @Override + public Step.Status run(LockedStep step, RunStatus run) { + RunId id = run.id(); + switch (step.get()) { + case deployInitialReal: return deployInitialReal(id); + case installInitialReal: return installInitialReal(id); + case deployReal: return deployReal(id); + case deployTester: return deployTester(id); + case installReal: return installReal(id); + case installTester: return installTester(id); + case startTests: return startTests(id); + case storeData: return storeData(id); + case deactivateReal: return deactivateReal(id); + case deactivateTester: return deactivateTester(id); + default: throw new AssertionError("Unknown step '" + step + "'!"); + } + } + + private Step.Status deployInitialReal(RunId id) { + throw new AssertionError(); + } + + private Step.Status installInitialReal(RunId id) { + throw new AssertionError(); + } + + private Step.Status deployReal(RunId id) { + throw new AssertionError(); + } + + private Step.Status deployTester(RunId id) { + throw new AssertionError(); + } + + private Step.Status installReal(RunId id) { + throw new AssertionError(); + } + + private Step.Status installTester(RunId id) { + throw new AssertionError(); + } + + private Step.Status startTests(RunId id) { + throw new AssertionError(); + } + + private Step.Status storeData(RunId id) { + throw new AssertionError(); + } + + private Step.Status deactivateReal(RunId id) { + throw new AssertionError(); + } + + private Step.Status deactivateTester(RunId id) { + throw new AssertionError(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java index 2e6ff6c77d6..ebb4c91edcc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -16,17 +17,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; */ public interface StepRunner { - /** - * Attempts to run the given step, and returns the new status. - */ - default RunStatus run(Step step, RunStatus run) { - switch (step) { - default: throw new AssertionError(); - } - } - - default Step.Status deployInitialReal(ApplicationId id, JobType type) { - throw new AssertionError(); - } + /** Attempts to run the given locked step, and returns its new status. */ + Step.Status run(LockedStep step, RunStatus run); } + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 6ee9b15e049..6a2b6fb587c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -8,7 +8,6 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.event.Collection; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; @@ -16,6 +15,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -31,6 +31,7 @@ import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,10 +118,6 @@ public class CuratorDb { return tryLock(lockPath(id, type, step)); } - public Lock lockActiveRuns() { - return lock(lockRoot.append("activeRuns"), defaultLockTimeout); - } - public Lock lockRotations() { return lock(lockRoot.append("rotations"), defaultLockTimeout); } @@ -310,35 +307,26 @@ public class CuratorDb { // -------------- Job Runs ------------------------------------------------ - public void writeActiveRun(RunStatus run) { - appendRun(run, activeRunsPath()); - } - - public void writeHistoricRun(RunStatus run) { - appendRun(run, jobRunPath(run.id().application(), run.id().type())); + public void writeLastRun(RunStatus run) { + curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(jobSerializer.toSlime(run))); } - public List readActiveRuns() { - return Collections.unmodifiableList(readRuns(activeRunsPath())); + public void writeHistoricRuns(ApplicationId id, JobType type, Iterable runs) { + curator.set(jobPath(id, type), asJson(jobSerializer.toSlime(runs))); } - public List readHistoricRuns(ApplicationId id, JobType type) { - // TODO jvenstad: Add, somewhere, a retention filter based on age or count. - return Collections.unmodifiableList(readRuns(jobRunPath(id, type))); - } - - private void appendRun(RunStatus run, Path runsPath) { - List runs = readRuns(runsPath); - runs.add(run); - writeRuns(runsPath, runs); + public void deleteRuns(ApplicationId id, JobType type) { + curator.delete(jobPath(id, type)); + curator.delete(lastRunPath(id, type)); } - private List readRuns(Path runsPath) { - return readSlime(runsPath).map(jobSerializer::fromSlime).orElse(Collections.emptyList()); + public Optional readLastRun(ApplicationId id, JobType type) { + return readSlime(jobPath(id, type)).map(jobSerializer::runFromSlime); } - private void writeRuns(Path runsPaths, Iterable runs) { - curator.set(runsPaths, asJson(jobSerializer.toSlime(runs))); + public Map readHistoricRuns(ApplicationId id, JobType type) { + // TODO jvenstad: Add, somewhere, a retention filter based on age or count. + return readSlime(jobPath(id, type)).map(jobSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); } // -------------- Provisioning (called by internal code) ------------------ @@ -465,12 +453,12 @@ public class CuratorDb { return applicationRoot.append(application.serializedForm()); } - private static Path jobRunPath(ApplicationId id, JobType type) { + private static Path jobPath(ApplicationId id, JobType type) { return jobRoot.append(id.serializedForm()).append(type.jobName()); } - private static Path activeRunsPath() { - return jobRoot.append("active"); + private static Path lastRunPath(ApplicationId id, JobType type) { + return jobPath(id, type).append("last"); } private static Path controllerPath(String hostname) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index 9f56988382c..6813ed2969d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -1,27 +1,67 @@ package com.yahoo.vespa.hosted.controller.persistence; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import java.time.Instant; import java.util.ArrayList; +import java.util.EnumMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; public class JobSerializer { - public List fromSlime(Slime slime) { - List runs = new ArrayList<>(); + private static final String stepsField = "steps"; + private static final String applicationField = "id"; + private static final String jobTypeField = "type"; + private static final String numberField = "number"; + private static final String startField = "start"; + private static final String endField = "end"; + + public RunStatus runFromSlime(Slime slime) { + return runFromSlime(slime.get()); + } + + public Map runsFromSlime(Slime slime) { + Map runs = new LinkedHashMap<>(); Inspector runArray = slime.get(); - runArray.traverse((ArrayTraverser) (__, runObject) -> - runs.add(runFromSlime(runObject))); + runArray.traverse((ArrayTraverser) (__, runObject) -> { + RunStatus run = runFromSlime(runObject); + runs.put(run.id(), run); + }); return runs; } private RunStatus runFromSlime(Inspector runObject) { - throw new AssertionError(); + EnumMap steps = new EnumMap<>(Step.class); + runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { + steps.put(Step.valueOf(step), Step.Status.valueOf(status.asString())); + }); + return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.fromJobName(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()), + steps, + Instant.ofEpochMilli(runObject.field(startField).asLong()), + Optional.of(runObject.field(endField)) + .filter(Inspector::valid) + .map(end -> Instant.ofEpochMilli(end.asLong()))); + } + + public Slime toSlime(RunStatus run) { + Slime slime = new Slime(); + toSlime(run, slime.setObject()); + return slime; } public Slime toSlime(Iterable runs) { @@ -31,8 +71,14 @@ public class JobSerializer { return slime; } - private Slime toSlime(RunStatus run, Cursor runObject) { - throw new AssertionError(); + private void toSlime(RunStatus run, Cursor runObject) { + runObject.setString(applicationField, run.id().application().serializedForm()); + runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setLong(numberField, run.id().number()); + runObject.setLong(startField, run.start().toEpochMilli()); + run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); + Cursor stepsObject = runObject.setObject(stepsField); + run.steps().forEach((step, status) -> stepsObject.setString(step.name(), status.name())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java new file mode 100644 index 00000000000..2a2a6a5e1b9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -0,0 +1,39 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.ControllerTester; +import org.junit.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author jonmv + */ +public class JobRunnerTest { + + @Test + public void test() { + ControllerTester tester = new ControllerTester(); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()), + inThreadExecutor(), new DummyStepRunner()); + runner.maintain(); + } + + private static ExecutorService inThreadExecutor() { + return new AbstractExecutorService() { + AtomicBoolean shutDown = new AtomicBoolean(false); + @Override public void shutdown() { shutDown.set(true); } + @Override public List shutdownNow() { shutDown.set(true); return Collections.emptyList(); } + @Override public boolean isShutdown() { return shutDown.get(); } + @Override public boolean isTerminated() { return shutDown.get(); } + @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return true; } + @Override public void execute(Runnable command) { command.run(); } + }; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 0ad82df8db1..e3d060ee806 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -21,6 +21,9 @@ { "name": "DnsMaintainer" }, + { + "name": "JobRunner" + }, { "name": "MetricsReporter" }, -- cgit v1.2.3 From 47e0bd4d12a5f3d227c97681652d2e2e859d066f Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 27 Jun 2018 13:57:16 +0200 Subject: Some more details fleshed out --- .../deployment/InternalBuildService.java | 11 +- .../controller/deployment/JobController.java | 111 +++++++++++++-------- .../hosted/controller/deployment/RunStatus.java | 3 + .../hosted/controller/maintenance/JobRunner.java | 2 +- .../controller/maintenance/RealStepRunner.java | 10 +- .../hosted/controller/persistence/CuratorDb.java | 2 +- 6 files changed, 93 insertions(+), 46 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java index 74dffc1c4fd..f7c17c78700 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java @@ -1,7 +1,9 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Optional; /** * Wraps a JobController as a BuildService. @@ -20,17 +22,18 @@ public class InternalBuildService implements BuildService { @Override public void trigger(BuildJob buildJob) { - + jobs.run(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); } @Override public JobState stateOf(BuildJob buildJob) { - return null; + Optional run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); + return run.isPresent() && ! run.get().end().isPresent() ? JobState.running : JobState.idle; } @Override public boolean builds(BuildJob buildJob) { - return false; + return jobs.builds(buildJob.applicationId()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 0cde83bf230..9cf79c600f7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -9,12 +9,16 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.stream.Stream; @@ -43,6 +47,7 @@ public class JobController { this.logs = logStore; } + // TODO jvenstad: Remove this, and let the DeploymentTrigger trigger directly with the correct BuildService. /** Returns whether the given application has registered with this build service. */ public boolean builds(ApplicationId id) { return controller.applications().get(id) @@ -65,18 +70,26 @@ public class JobController { .iterator()); } + /** Returns an immutable map of all known runs for the given application and job type. */ + public Map runs(ApplicationId id, JobType type) { + Map runs = curator.readHistoricRuns(id, type); + last(id, type).ifPresent(run -> runs.putIfAbsent(run.id(), run)); + return ImmutableMap.copyOf(runs); + } + + /** Returns the last run of the given type, for the given application, if one has been run. */ public Optional last(ApplicationId id, JobType type) { return curator.readLastRun(id, type); } - /** Returns a list of meta information about all known runs of the given job type for the given application. */ - public Map runs(ApplicationId id, JobType type) { - ImmutableMap.Builder runs = ImmutableMap.builder(); - runs.putAll(curator.readHistoricRuns(id, type)); - last(id, type).ifPresent(run -> runs.put(run.id(), run)); - return runs.build(); + /** Returns the run with the given id, provided it is still active. */ + public Optional active(RunId id) { + return last(id.application(), id.type()) + .filter(run -> ! run.end().isPresent()) + .filter(run -> run.id().equals(id)); } + /** Returns a list of all active runs. */ public List active() { return copyOf(applications().stream() .flatMap(id -> Stream.of(JobType.values()) @@ -86,20 +99,16 @@ public class JobController { .iterator()); } - public Optional active(RunId id) { - return last(id.application(), id.type()) - .filter(run -> ! run.end().isPresent()) - .filter(run -> run.id().equals(id)); - } - + /** Changes the status of the given step, for the given run, provided it is still active. */ public void update(RunId id, Step.Status status, LockedStep step) { - modify(id, run -> run.with(status, step)); + locked(id, run -> run.with(status, step)); } + /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { - modify(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. + locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. RunStatus endedRun = run.finish(controller.clock().instant()); - modify(id.application(), id.type(), runs -> runs.put(run.id(), endedRun)); + locked(id.application(), id.type(), runs -> runs.put(run.id(), endedRun)); return endedRun; }); } @@ -115,52 +124,74 @@ public class JobController { controller.applications().store(application.withBuiltInternally(true))); } - /** Accepts and stores a new appliaction package and test jar pair. */ - public void submit(ApplicationId id, byte[] applicationPackage, byte[] applicationTestJar) { + /** Accepts and stores a new application package and test jar pair under a generated application version key. */ + public ApplicationVersion submit(ApplicationId id, SourceRevision revision, + byte[] applicationPackage, byte[] applicationTestJar) { + AtomicReference version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { - ApplicationVersion version = nextVersion(id); + if ( ! application.get().deploymentJobs().builtInternally()) + throw new IllegalArgumentException(id + " is not built here!"); + + long run = nextBuild(id); + version.set(ApplicationVersion.from(revision, run)); // TODO smorgrav: Store the pair. - notifyOfNewSubmission(id); + notifyOfNewSubmission(id, revision, run); }); + return version.get(); } /** Orders a run of the given type, and returns the id of the created job. */ public void run(ApplicationId id, JobType type) { - modify(id, type, runs -> { - Optional last = last(id, type); - if (last.flatMap(run -> active(run.id())).isPresent()) - throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); + controller.applications().lockIfPresent(id, application -> { + if ( ! application.get().deploymentJobs().builtInternally()) + throw new IllegalArgumentException(id + " is not built here!"); - RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant())); + locked(id, type, __ -> { + Optional last = last(id, type); + if (last.flatMap(run -> active(run.id())).isPresent()) + throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); + + RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); + curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant())); + }); }); } - /** Unregisters the given application, and deletes all associated data. */ + /** Unregisters the given application and deletes all associated data. */ public void unregister(ApplicationId id) { controller.applications().lockIfPresent(id, application -> { controller.applications().store(application.withBuiltInternally(false)); + for (JobType type : jobs(id)) + try (Lock __ = curator.lock(id, type)) { + curator.deleteJobData(id, type); + } }); - jobs(id).forEach(type -> modify(id, type, __ -> curator.deleteRuns(id, type))); } - /** Aborts the given job. */ - public void abort(RunId id) { - - } - - - private ApplicationVersion nextVersion(ApplicationId id) { - throw new AssertionError(); + // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. + private long nextBuild(ApplicationId id) { + return 1 + controller.applications().require(id).deploymentJobs() + .statusOf(JobType.component) + .flatMap(JobStatus::lastCompleted) + .map(JobStatus.JobRun::id) + .orElse(0L); } - private void notifyOfNewSubmission(ApplicationId id) { - ; + // TODO jvenstad: Find a more appropriate way of doing this when this is the only build service. + private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { + DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, + JobType.component, + 0, + number, + Optional.of(revision), + Optional.empty()); + controller.applications().deploymentTrigger().notifyOfCompletion(report); } - private void modify(ApplicationId id, JobType type, Consumer> modifications) { + /** Locks and modifies the list of historic runs for the given application and job type. */ + private void locked(ApplicationId id, JobType type, Consumer> modifications) { try (Lock __ = curator.lock(id, type)) { Map runs = curator.readHistoricRuns(id, type); modifications.accept(runs); @@ -168,7 +199,8 @@ public class JobController { } } - private void modify(RunId id, UnaryOperator modifications) { + /** Locks and modifies the run with the given id, provided it is still active. */ + private void locked(RunId id, UnaryOperator modifications) { try (Lock __ = curator.lock(id.application(), id.type())) { RunStatus run = active(id).orElseThrow(() -> new IllegalArgumentException(id + " is not an active run!")); run = modifications.apply(run); @@ -176,6 +208,7 @@ public class JobController { } } + /** Locks the given step, and checks none of its prerequisites are running, then performs the given actions. */ public void locked(RunId id, Step step, Consumer action) throws TimeoutException { try (Lock lock = curator.lock(id.application(), id.type(), step)) { for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index a6aa35d1220..3406931739f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -48,6 +48,9 @@ public class RunStatus { } public RunStatus finish(Instant now) { + if (end.isPresent()) + throw new IllegalStateException("This step ended at " + end.get() + " -- it can't be ended again!"); + return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 686cd3cf4ea..eacf9c72992 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -71,7 +71,7 @@ public class JobRunner extends Maintainer { void advance(RunId id, Step step) { try { jobs.locked(id, step, lockedStep -> { - jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we should bail out. + jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we bail out. if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index 55dbda1401b..ae956c2c5c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; public class RealStepRunner implements StepRunner { + private static final String prefix = "-test-"; + private final ApplicationController applications; public RealStepRunner(ApplicationController applications) { @@ -34,7 +36,7 @@ public class RealStepRunner implements StepRunner { } private Step.Status deployInitialReal(RunId id) { - throw new AssertionError(); + } private Step.Status installInitialReal(RunId id) { @@ -73,4 +75,10 @@ public class RealStepRunner implements StepRunner { throw new AssertionError(); } + private static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + prefix + id.instance().value()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 6a2b6fb587c..15ddcdace5b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -315,7 +315,7 @@ public class CuratorDb { curator.set(jobPath(id, type), asJson(jobSerializer.toSlime(runs))); } - public void deleteRuns(ApplicationId id, JobType type) { + public void deleteJobData(ApplicationId id, JobType type) { curator.delete(jobPath(id, type)); curator.delete(lastRunPath(id, type)); } -- cgit v1.2.3 From 14a3187b6c7c89ed47298d6566e1ac6905569404 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 27 Jun 2018 14:03:33 +0200 Subject: RunStatus.hasEnded() for convenience --- .../vespa/hosted/controller/deployment/InternalBuildService.java | 2 +- .../com/yahoo/vespa/hosted/controller/deployment/JobController.java | 4 ++-- .../java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java | 4 ++++ .../com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java index f7c17c78700..381a4712ec8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java @@ -28,7 +28,7 @@ public class InternalBuildService implements BuildService { @Override public JobState stateOf(BuildJob buildJob) { Optional run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); - return run.isPresent() && ! run.get().end().isPresent() ? JobState.running : JobState.idle; + return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 9cf79c600f7..40d20e1e100 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -85,7 +85,7 @@ public class JobController { /** Returns the run with the given id, provided it is still active. */ public Optional active(RunId id) { return last(id.application(), id.type()) - .filter(run -> ! run.end().isPresent()) + .filter(run -> ! run.hasEnded()) .filter(run -> run.id().equals(id)); } @@ -95,7 +95,7 @@ public class JobController { .flatMap(id -> Stream.of(JobType.values()) .map(type -> last(id, type)) .filter(Optional::isPresent).map(Optional::get) - .filter(run -> ! run.end().isPresent())) + .filter(run -> ! run.hasEnded())) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 3406931739f..8fa463d3f1b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -84,6 +84,10 @@ public class RunStatus { return steps.values().contains(failed); } + public boolean hasEnded() { + return end.isPresent(); + } + public List readySteps() { return hasFailed() ? forcedSteps() : normalSteps(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index ae956c2c5c8..da80df59655 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -36,7 +36,7 @@ public class RealStepRunner implements StepRunner { } private Step.Status deployInitialReal(RunId id) { - + throw new AssertionError(); } private Step.Status installInitialReal(RunId id) { -- cgit v1.2.3 From 48417d7a59599c7c8149a19421a35f1d74ca8290 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 27 Jun 2018 16:45:16 +0200 Subject: Persistent (de)serialisation, and some minors --- .../controller/deployment/JobController.java | 9 ++- .../vespa/hosted/controller/deployment/Step.java | 5 +- .../controller/maintenance/RealStepRunner.java | 65 ++++++++++++++---- .../controller/persistence/JobSerializer.java | 77 ++++++++++++++++++++-- 4 files changed, 134 insertions(+), 22 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 40d20e1e100..466a658b49c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -47,6 +47,10 @@ public class JobController { this.logs = logStore; } + public LogStore logs() { + return logs; + } + // TODO jvenstad: Remove this, and let the DeploymentTrigger trigger directly with the correct BuildService. /** Returns whether the given application has registered with this build service. */ public boolean builds(ApplicationId id) { @@ -113,11 +117,6 @@ public class JobController { }); } - /** Returns the details for the given job. */ - public RunDetails details(RunId id) { - return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id)); - } - /** Registers the given application, such that it may have deployment jobs run here. */ void register(ApplicationId id) { controller.applications().lockIfPresent(id, application -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 240f2ada242..a1d07001331 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -51,7 +51,10 @@ public enum Step { deactivateReal(deployInitialReal, deployReal, startTests), /** Deactivate the tester. */ - deactivateTester(deployTester, storeData); + deactivateTester(deployTester, storeData), + + /** Report completion to deployment orchestration machinery. */ + report; private final List prerequisites; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index da80df59655..9f539e854e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -1,20 +1,37 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; + +/** + * Runs steps of a deployment job against its provided controller. + * + * @author jonmv + */ public class RealStepRunner implements StepRunner { - private static final String prefix = "-test-"; + private static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + "-test-" + id.instance().value()); + } - private final ApplicationController applications; + private final Controller controller; - public RealStepRunner(ApplicationController applications) { - this.applications = applications; + public RealStepRunner(Controller controller) { + this.controller = controller; } @Override @@ -31,54 +48,80 @@ public class RealStepRunner implements StepRunner { case storeData: return storeData(id); case deactivateReal: return deactivateReal(id); case deactivateTester: return deactivateTester(id); + case report: return report(id); default: throw new AssertionError("Unknown step '" + step + "'!"); } } private Step.Status deployInitialReal(RunId id) { - throw new AssertionError(); + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, Optional.empty(), false, true)); + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if (id.type().isTest() && e.getErrorCode() == ConfigServerException.ErrorCode.OUT_OF_CAPACITY) + return unfinished; + } + return failed; } private Step.Status installInitialReal(RunId id) { - throw new AssertionError(); + // If converged and serviceconverged: succeeded + // If timeout, failed + return unfinished; } private Step.Status deployReal(RunId id) { + // Separate out deploy logic from above, and reuse. throw new AssertionError(); } private Step.Status deployTester(RunId id) { + // Find endpoints of real application. This will move down at a later time. + // See above. throw new AssertionError(); } private Step.Status installReal(RunId id) { + // See three above. throw new AssertionError(); } private Step.Status installTester(RunId id) { + // See above. throw new AssertionError(); } private Step.Status startTests(RunId id) { + // Empty for now, but will be: find endpoints and post them. throw new AssertionError(); } private Step.Status storeData(RunId id) { + // Update test logs. + // If tests are done, return test results. throw new AssertionError(); } private Step.Status deactivateReal(RunId id) { + // Try to deactivate, and if deactivated, finished. throw new AssertionError(); } private Step.Status deactivateTester(RunId id) { + // See above. throw new AssertionError(); } - private static ApplicationId testerOf(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value(), - prefix + id.instance().value()); + private Step.Status report(RunId id) { + // Easy squeezy. + throw new AssertionError(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index 6813ed2969d..ba66bb67636 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -10,15 +10,30 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import java.time.Instant; -import java.util.ArrayList; import java.util.EnumMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; +import static java.util.Objects.requireNonNull; + public class JobSerializer { private static final String stepsField = "steps"; @@ -44,9 +59,9 @@ public class JobSerializer { } private RunStatus runFromSlime(Inspector runObject) { - EnumMap steps = new EnumMap<>(Step.class); + EnumMap steps = new EnumMap<>(Step.class); runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(Step.valueOf(step), Step.Status.valueOf(status.asString())); + steps.put(stepOf(step), statusOf(status.asString())); }); return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), JobType.fromJobName(runObject.field(jobTypeField).asString()), @@ -78,7 +93,59 @@ public class JobSerializer { runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, status) -> stepsObject.setString(step.name(), status.name())); + run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + } + + private static String valueOf(Step step) { + switch (step) { + case deployInitialReal : return "DIR"; + case installInitialReal : return "IIR"; + case deployReal : return "DR" ; + case installReal : return "ID" ; + case deactivateReal : return "DAR"; + case deployTester : return "DT" ; + case installTester : return "IR" ; + case deactivateTester : return "DAT"; + case startTests : return "ST" ; + case storeData : return "SD" ; + case report : return "R" ; + default : throw new AssertionError("No value defined for '" + step + "'!"); + } + } + + private static Step stepOf(String step) { + switch (step) { + case "DIR" : return deployInitialReal ; + case "IIR" : return installInitialReal; + case "DR" : return deployReal ; + case "ID" : return installReal ; + case "DAR" : return deactivateReal ; + case "DT" : return deployTester ; + case "IR" : return installTester ; + case "DAT" : return deactivateTester ; + case "ST" : return startTests ; + case "SD" : return storeData ; + case "R" : return report ; + default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + } + } + + private static String valueOf(Status status) { + switch (status) { + case unfinished : return "U"; + case failed : return "F"; + case succeeded : return "S"; + default : throw new AssertionError("No value defined for '" + status + "'!"); + } + } + + private static Status statusOf(String status) { + switch (status) { + case "U" : return unfinished; + case "F" : return failed ; + case "S" : return succeeded ; + default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + } } } -- cgit v1.2.3 From ce5876ae157f5ac56bb51cd8e2ad76ed3b4b1f35 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jun 2018 14:10:06 +0200 Subject: Job serialisation and log store changes --- .../controller/api/integration/LogStore.java | 4 +- .../api/integration/stubs/MockLogStore.java | 11 +-- .../hosted/controller/ApplicationController.java | 1 - .../controller/deployment/JobController.java | 42 ++++++++++- .../hosted/controller/deployment/JobProfile.java | 2 +- .../hosted/controller/deployment/RunDetails.java | 21 ++++-- .../hosted/controller/deployment/RunStatus.java | 50 ++++++++++--- .../controller/maintenance/RealStepRunner.java | 45 +++++++----- .../controller/persistence/JobSerializer.java | 16 ++--- .../controller/persistence/JobSerializerTest.java | 84 ++++++++++++++++++++++ .../persistence/testdata/run-status.json | 21 ++++++ 11 files changed, 244 insertions(+), 53 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java index 959ac2b8680..918047edca1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -21,10 +21,10 @@ public interface LogStore { void setConvergenceLog(RunId id, String convergenceLog); /** @return the result of prepare of the test application for the given deployment job. */ - PrepareResponse getPrepareResponse(RunId id); + String getDeploymentLog(RunId id); /** Stores the given result of prepare of the test application for the given deployment job. */ - void setPrepareResponse(RunId id, PrepareResponse prepareResponse); + void setDeploymentLog(RunId id, String deploymentLog); /** Deletes all data associated with test of a given deployment job */ void deleteTestData(RunId id); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java index c5dba3b8fa7..bc9f6247055 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java @@ -35,17 +35,12 @@ public class MockLogStore implements LogStore { } @Override - public PrepareResponse getPrepareResponse(RunId id) { - PrepareResponse prepareResponse = new PrepareResponse(); - prepareResponse.message = "foo"; - prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), - Collections.emptyList()); - prepareResponse.tenant = new TenantId("tenant"); - return prepareResponse; + public String getDeploymentLog(RunId id) { + return "SUCCESS"; } @Override - public void setPrepareResponse(RunId id, PrepareResponse prepareResponse) { + public void setDeploymentLog(RunId id, String deploymentLog) { } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7c7fb27f3eb..0dd80d54919 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -55,7 +55,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.time.Clock; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 466a658b49c..8093c4d7a18 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -30,9 +30,12 @@ import static com.google.common.collect.ImmutableList.copyOf; * * Keys are the {@link ApplicationId} of the real application, for which the deployment job is run, and the * {@link JobType} of the real deployment to test. - * * Although the deployment jobs are themselves applications, their IDs are not to be referenced. * + * Jobs consist of sets of {@link Step}s, defined in {@link JobProfile}s. + * Each run is represented by a {@link RunStatus}, which holds the status of each step of the run, as well as + * some other meta data. + * * @author jonmv */ public class JobController { @@ -45,10 +48,43 @@ public class JobController { this.controller = controller; this.curator = controller.curator(); this.logs = logStore; + + } + + /** Rewrite all job data with the newest format. */ + public void updateStorage() { + for (ApplicationId id : applications()) + for (JobType type : jobs(id)) { + locked(id, type, runs -> { + }); + last(id, type).ifPresent(last -> locked(last.id(), run -> run)); + } } - public LogStore logs() { - return logs; + /** Returns the details currently logged for the given run. */ + public RunDetails details(RunId id) { + return new RunDetails(logs.getDeploymentLog(id), logs.getConvergenceLog(id), logs.getTestLog(id)); + } + + /** Appends the given string to the currently stored deployment logs for the given run. */ + public void logDeployment(RunId id, String appendage) { + try (Lock __ = curator.lock(id.application(), id.type())) { + logs.setDeploymentLog(id, logs.getDeploymentLog(id).concat(appendage)); + } + } + + /** Appends the given string to the currently stored convergence logs for the given run. */ + public void logConvergence(RunId id, String appendage) { + try (Lock __ = curator.lock(id.application(), id.type())) { + logs.setConvergenceLog(id, logs.getConvergenceLog(id).concat(appendage)); + } + } + + /** Appends the given string to the currently stored test logs for the given run. */ + public void logTest(RunId id, String appendage) { + try (Lock __ = curator.lock(id.application(), id.type())) { + logs.setTestLog(id, logs.getTestLog(id).concat(appendage)); + } } // TODO jvenstad: Remove this, and let the DeploymentTrigger trigger directly with the correct BuildService. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index d2e0a4fe705..82464820d9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -58,7 +58,7 @@ public enum JobProfile { case test: return systemTest; case staging: return stagingTest; case prod: return production; - default: throw new IllegalArgumentException("Unexpected environment " + type.environment()); + default: throw new AssertionError("Unexpected environment '" + type.environment() + "'!"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java index 300452ac0a8..98969bd4508 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java @@ -1,8 +1,5 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; - /** * Contains details about a deployment job run. * @@ -10,14 +7,26 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes */ public class RunDetails { - private final PrepareResponse deploymentResult; + private final String deploymentLog; private final String convergenceLog; private final String testLog; - public RunDetails(PrepareResponse deploymentResult, String convergenceLog, String testLog) { - this.deploymentResult = deploymentResult; + public RunDetails(String deploymentLog, String convergenceLog, String testLog) { + this.deploymentLog = deploymentLog; this.convergenceLog = convergenceLog; this.testLog = testLog; } + public String getDeploymentLog() { + return deploymentLog; + } + + public String getConvergenceLog() { + return convergenceLog; + } + + public String getTestLog() { + return testLog; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 8fa463d3f1b..ba94b21b8bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,6 +26,7 @@ public class RunStatus { private final Map steps; private final Instant start; private final Optional end; + // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs. // For deserialisation only -- do not use! public RunStatus(RunId id, Map steps, Instant start, Optional end) { @@ -42,6 +43,9 @@ public class RunStatus { } public RunStatus with(Step.Status status, LockedStep step) { + if (end.isPresent()) + throw new IllegalStateException("This step ended at " + end.get() + " -- it can't be further modified!"); + EnumMap steps = new EnumMap<>(this.steps); steps.put(step.get(), requireNonNull(status)); return new RunStatus(id, steps, start, end); @@ -80,36 +84,66 @@ public class RunStatus { return end; } + /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { return steps.values().contains(failed); } + /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ public boolean hasEnded() { return end.isPresent(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof RunStatus)) return false; + + RunStatus status = (RunStatus) o; + + return id.equals(status.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public String toString() { + return "RunStatus{" + + "id=" + id + + ", start=" + start + + ", end=" + end + + ", steps=" + steps + + '}'; + } + + /** Returns the list of steps to run for this job right now, depending on whether the job has failed. */ public List readySteps() { return hasFailed() ? forcedSteps() : normalSteps(); } + /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ private List normalSteps() { return ImmutableList.copyOf(steps.entrySet().stream() .filter(entry -> entry.getValue() == unfinished - && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + && entry.getKey().prerequisites().stream() + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) .map(Map.Entry::getKey) .iterator()); } + /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ private List forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() .filter(entry -> entry.getValue() != succeeded - && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) - && entry.getKey().prerequisites().stream() - .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) + && entry.getKey().prerequisites().stream() + .filter(JobProfile.of(id.type()).alwaysRun()::contains) + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) .map(Map.Entry::getKey) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index 9f539e854e8..1ed55548ee7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -1,7 +1,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -54,21 +56,7 @@ public class RealStepRunner implements StepRunner { } private Step.Status deployInitialReal(RunId id) { - try { - // TODO jvenstad: Do whatever is required based on the result, and log all of this. - controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), - Optional.empty(), - new DeployOptions(false, Optional.empty(), false, true)); - return succeeded; - } - catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. - if (id.type().isTest() && e.getErrorCode() == ConfigServerException.ErrorCode.OUT_OF_CAPACITY) - return unfinished; - } - return failed; + return deployReal(id, true); } private Step.Status installInitialReal(RunId id) { @@ -79,7 +67,7 @@ public class RealStepRunner implements StepRunner { private Step.Status deployReal(RunId id) { // Separate out deploy logic from above, and reuse. - throw new AssertionError(); + return deployReal(id,false); } private Step.Status deployTester(RunId id) { @@ -124,4 +112,29 @@ public class RealStepRunner implements StepRunner { throw new AssertionError(); } + private Step.Status deployReal(RunId id, boolean setTheStage) { + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + ActivateResult result = controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, + Optional.empty(), + false, + setTheStage)); + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if (id.type().isTest() && e.getErrorCode() == ConfigServerException.ErrorCode.OUT_OF_CAPACITY) + return unfinished; + } + return failed; + } + + private Application application(ApplicationId id) { + return controller.applications().require(id); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index ba66bb67636..f34cde002d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -96,15 +96,15 @@ public class JobSerializer { run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); } - private static String valueOf(Step step) { + static String valueOf(Step step) { switch (step) { case deployInitialReal : return "DIR"; case installInitialReal : return "IIR"; case deployReal : return "DR" ; - case installReal : return "ID" ; + case installReal : return "IR" ; case deactivateReal : return "DAR"; case deployTester : return "DT" ; - case installTester : return "IR" ; + case installTester : return "IT" ; case deactivateTester : return "DAT"; case startTests : return "ST" ; case storeData : return "SD" ; @@ -113,15 +113,15 @@ public class JobSerializer { } } - private static Step stepOf(String step) { + static Step stepOf(String step) { switch (step) { case "DIR" : return deployInitialReal ; case "IIR" : return installInitialReal; case "DR" : return deployReal ; - case "ID" : return installReal ; + case "IR" : return installReal ; case "DAR" : return deactivateReal ; case "DT" : return deployTester ; - case "IR" : return installTester ; + case "IT" : return installTester ; case "DAT" : return deactivateTester ; case "ST" : return startTests ; case "SD" : return storeData ; @@ -130,7 +130,7 @@ public class JobSerializer { } } - private static String valueOf(Status status) { + static String valueOf(Status status) { switch (status) { case unfinished : return "U"; case failed : return "F"; @@ -139,7 +139,7 @@ public class JobSerializer { } } - private static Status statusOf(String status) { + static Status statusOf(String status) { switch (status) { case "U" : return unfinished; case "F" : return failed ; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java new file mode 100644 index 00000000000..e6e726bec5e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java @@ -0,0 +1,84 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.Collections; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class JobSerializerTest { + + private static final JobSerializer serializer = new JobSerializer(); + private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); + private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), + JobType.productionUsEast3, + (long) 112358); + private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); + + @Test + public void testSerialization() throws IOException { + for (Step step : Step.values()) + assertEquals(step, JobSerializer.stepOf(JobSerializer.valueOf(step))); + + for (Step.Status status : Step.Status.values()) + assertEquals(status, JobSerializer.statusOf(JobSerializer.valueOf(status))); + + // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! + RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); + for (Step step : Step.values()) + assertTrue(run.steps().containsKey(step)); + + assertEquals(id, run.id()); + assertEquals(start, run.start()); + assertFalse(run.hasEnded()); + assertEquals(ImmutableMap.builder() + .put(deployInitialReal, unfinished) + .put(installInitialReal, failed) + .put(deployReal, succeeded) + .put(installReal, unfinished) + .put(deactivateReal, failed) + .put(deployTester, succeeded) + .put(installTester, unfinished) + .put(deactivateTester, failed) + .put(startTests, succeeded) + .put(storeData, unfinished) + .put(report, failed) + .build(), + run.steps()); + + RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); + assertEquals(run.id(), phoenix.id()); + assertEquals(run.start(), phoenix.start()); + assertEquals(run.end(), phoenix.end()); + assertEquals(run.steps(), phoenix.steps()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json new file mode 100644 index 00000000000..4db94eb429a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -0,0 +1,21 @@ +[ + { + "id": "tenant:application:default", + "type": "production-us-east-3", + "number": 112358, + "start": 1196676930000, + "steps": { + "DIR": "U", + "IIR": "F", + "DR": "S", + "IR": "U", + "DAR": "F", + "DT": "S", + "IT": "U", + "DAT": "F", + "ST": "S", + "SD": "U", + "R": "F" + } + } +] \ No newline at end of file -- cgit v1.2.3 From 02780c2eb0d2bc1d5962ee3c09f4ea63fb6388c3 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jun 2018 16:37:30 +0200 Subject: Some more wiring and stubs --- .../integration/deployment/ArtifactRepository.java | 13 +- .../hosted/controller/ApplicationController.java | 12 +- .../hosted/controller/deployment/RunStatus.java | 4 +- .../controller/maintenance/RealStepRunner.java | 137 ++++++++++++++------- .../controller/persistence/JobSerializer.java | 1 - .../integration/ArtifactRepositoryMock.java | 17 ++- 6 files changed, 129 insertions(+), 55 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java index b77e085e733..b44e7f6f5e7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java @@ -12,10 +12,19 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; */ public interface ArtifactRepository { - /** Get tenant application package of given version */ + /** Returns the tenant application package of the given version. */ byte[] getApplicationPackage(ApplicationId application, String applicationVersion); - /** Get system application package of given version */ + /** Stores the given tenant application package of the given version. */ + void putApplicationPackage(ApplicationId application, String applicationVersion, byte[] applicationPackage); + + /** Returns the system application package of the given version. */ byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version); + /** Stores the given tester application fat jar of the given version. */ + void putTesterJar(ApplicationId tester, String applicationVersion, byte[] fatTestJar); + + /** Returns the tester application fat jar of the given version. */ + byte[] getTesterJar(ApplicationId tester, String applicationVersion); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0dd80d54919..c43249f01bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -231,8 +231,8 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(ApplicationId id, Optional token) { - if ( ! (id.instance().isDefault() || id.instance().value().matches("\\d+"))) // TODO: Support instances properly - throw new UnsupportedOperationException("Only the instance names 'default' and names which are just the PR number are supported at the moment"); + if ( ! (id.instance().isDefault())) // TODO: Support instances properly + throw new UnsupportedOperationException("Only the instance name 'default' is supported at the moment"); try (Lock lock = lock(id)) { // Validate only application names which do not already exist. if (asList(id.tenant()).stream().noneMatch(application -> application.id().application().equals(id.application()))) @@ -353,6 +353,14 @@ public class ApplicationController { } } + /** Assembles and deploys a tester application to the given zone. */ + public ActivateResult deployTester(ApplicationId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { + if ( ! tester.instance().value().endsWith("-t")) + throw new IllegalArgumentException("'" + tester + "' is not a tester application!"); + + return deploy(tester, applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + } + private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, Set rotationNames, Set cnames) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index ba94b21b8bd..2ecac588d54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -44,7 +44,7 @@ public class RunStatus { public RunStatus with(Step.Status status, LockedStep step) { if (end.isPresent()) - throw new IllegalStateException("This step ended at " + end.get() + " -- it can't be further modified!"); + throw new AssertionError("This step ended at " + end.get() + " -- it can't be further modified!"); EnumMap steps = new EnumMap<>(this.steps); steps.put(step.get(), requireNonNull(status)); @@ -53,7 +53,7 @@ public class RunStatus { public RunStatus finish(Instant now) { if (end.isPresent()) - throw new IllegalStateException("This step ended at " + end.get() + " -- it can't be ended again!"); + throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index 1ed55548ee7..9e64ec7def9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -6,13 +6,21 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import java.util.Optional; +import java.util.function.Supplier; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -27,7 +35,7 @@ public class RealStepRunner implements StepRunner { private static ApplicationId testerOf(ApplicationId id) { return ApplicationId.from(id.tenant().value(), id.application().value(), - "-test-" + id.instance().value()); + id.instance().value() + "-t"); } private final Controller controller; @@ -37,7 +45,7 @@ public class RealStepRunner implements StepRunner { } @Override - public Step.Status run(LockedStep step, RunStatus run) { + public Status run(LockedStep step, RunStatus run) { RunId id = run.id(); switch (step.get()) { case deployInitialReal: return deployInitialReal(id); @@ -55,86 +63,121 @@ public class RealStepRunner implements StepRunner { } } - private Step.Status deployInitialReal(RunId id) { + private Status deployInitialReal(RunId id) { return deployReal(id, true); } - private Step.Status installInitialReal(RunId id) { - // If converged and serviceconverged: succeeded - // If timeout, failed - return unfinished; + private Status deployReal(RunId id) { + // Separate out deploy logic from above, and reuse. + return deployReal(id, false); } - private Step.Status deployReal(RunId id) { - // Separate out deploy logic from above, and reuse. - return deployReal(id,false); + private Status deployReal(RunId id, boolean setTheStage) { + return deploy(id.application(), + id.type(), + () -> controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, + Optional.empty(), + false, + setTheStage))); } - private Step.Status deployTester(RunId id) { + private Status deployTester(RunId id) { // Find endpoints of real application. This will move down at a later time. // See above. - throw new AssertionError(); + return deploy(testerOf(id.application()), + id.type(), + () -> controller.applications().deployTester(testerOf(id.application()), + testerPackage(id), + id.type().zone(controller.system()).get(), + new DeployOptions(true, + Optional.of(controller.systemVersion()), + false, + false))); + } + + private Status deploy(ApplicationId id, JobType type, Supplier deploy) { + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + ActivateResult result = deploy.get(); + + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() + || e.getErrorCode() == ACTIVATION_CONFLICT + || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { + + return unfinished; + } + } + return failed; } - private Step.Status installReal(RunId id) { - // See three above. - throw new AssertionError(); + private Status installInitialReal(RunId id) { + return install(id.application(), id.type()); } - private Step.Status installTester(RunId id) { - // See above. - throw new AssertionError(); + private Status installReal(RunId id) { + return install(id.application(), id.type()); + } + + private Status installTester(RunId id) { + return install(testerOf(id.application()), id.type()); + } + + private Status install(ApplicationId id, JobType type) { + // If converged and serviceconverged: succeeded + // If timeout, failed + return unfinished; } - private Step.Status startTests(RunId id) { + private Status startTests(RunId id) { // Empty for now, but will be: find endpoints and post them. throw new AssertionError(); } - private Step.Status storeData(RunId id) { + private Status storeData(RunId id) { // Update test logs. // If tests are done, return test results. throw new AssertionError(); } - private Step.Status deactivateReal(RunId id) { - // Try to deactivate, and if deactivated, finished. - throw new AssertionError(); + private Status deactivateReal(RunId id) { + return deactivate(id.application(), id.type()); } - private Step.Status deactivateTester(RunId id) { - // See above. - throw new AssertionError(); + private Status deactivateTester(RunId id) { + return deactivate(testerOf(id.application()), id.type()); } - private Step.Status report(RunId id) { - // Easy squeezy. + private Status deactivate(ApplicationId id, JobType type) { + // Try to deactivate, and if deactivated, finished. throw new AssertionError(); } - private Step.Status deployReal(RunId id, boolean setTheStage) { - try { - // TODO jvenstad: Do whatever is required based on the result, and log all of this. - ActivateResult result = controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), - Optional.empty(), - new DeployOptions(false, - Optional.empty(), - false, - setTheStage)); - return succeeded; - } - catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. - if (id.type().isTest() && e.getErrorCode() == ConfigServerException.ErrorCode.OUT_OF_CAPACITY) - return unfinished; - } - return failed; + private Status report(RunId id) { + // Easy squeezy. + throw new AssertionError(); } private Application application(ApplicationId id) { return controller.applications().require(id); } + private ApplicationPackage testerPackage(RunId id) { + ApplicationVersion version = application(id.application()).deploymentJobs() + .statusOf(id.type()).get() + .lastTriggered().get() + .application(); + + + // TODO hakonhall: Fetch! + throw new AssertionError(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index f34cde002d0..503f6fd990e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -32,7 +32,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; -import static java.util.Objects.requireNonNull; public class JobSerializer { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index 04c670cf136..cf546d8b075 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -46,9 +46,24 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac return artifact.data; } + @Override + public void putApplicationPackage(ApplicationId application, String applicationVersion, byte[] applicationPackage) { + throw new AssertionError(); + } + + @Override + public byte[] getTesterJar(ApplicationId tester, String applicationVersion) { + throw new AssertionError(); + } + + @Override + public void putTesterJar(ApplicationId tester, String applicationVersion, byte[] fatTestJar) { + throw new AssertionError(); + } + @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { - return new byte[0]; + throw new AssertionError(); } private static int artifactHash(ApplicationId applicationId, String applicationVersion) { -- cgit v1.2.3 From 801d98f6c1919bf8fd49340ac3c9e371f3c1f1d3 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jun 2018 16:50:36 +0200 Subject: Remove outdated tests --- .../controller/maintenance/RealStepRunner.java | 2 +- .../vespa/hosted/controller/ControllerTest.java | 75 ---------------------- .../integration/ArtifactRepositoryMock.java | 2 +- .../controller/versions/VersionStatusTest.java | 5 -- 4 files changed, 2 insertions(+), 82 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index 9e64ec7def9..14402e389df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -175,7 +175,7 @@ public class RealStepRunner implements StepRunner { .lastTriggered().get() .application(); - + // TODO hakonhall: Fetch! throw new AssertionError(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 5b818288b06..9980ddfc359 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -210,52 +210,6 @@ public class ControllerTest { runUpgrade(tester, app.id(), applicationVersion); } - @Test - public void testPullRequestDeployment() { - // Setup system - ControllerTester tester = new ControllerTester(); - ApplicationController applications = tester.controller().applications(); - - // staging deployment - long app1ProjectId = 22; - ApplicationId app1 = tester.createAndDeploy("tenant1", "domain1", - "application1", Environment.staging, - app1ProjectId).id(); - - // pull-request deployment - uses different instance id - ApplicationId app1pr = tester.createAndDeploy("tenant1", "domain1", - "application1", "1", - Environment.staging, app1ProjectId, null).id(); - - assertTrue(applications.get(app1).isPresent()); - assertEquals(app1, applications.get(app1).get().id()); - assertTrue(applications.get(app1pr).isPresent()); - assertEquals(app1pr, applications.get(app1pr).get().id()); - - // Simulate restart - tester.createNewController(); - applications = tester.controller().applications(); - - assertTrue(applications.get(app1).isPresent()); - assertEquals(app1, applications.get(app1).get().id()); - assertTrue(applications.get(app1pr).isPresent()); - assertEquals(app1pr, applications.get(app1pr).get().id()); - - // Deleting application also removes PR instance - ApplicationId app2 = tester.createAndDeploy("tenant1", "domain1", - "application2", Environment.staging, - 33).id(); - tester.controller().applications().deleteApplication(app1, Optional.of(new NToken("ntoken"))); - assertEquals("All instances deleted", 0, - tester.controller().applications().asList(app1.tenant()).stream() - .filter(app -> app.id().application().equals(app1.application())) - .count()); - assertEquals("Other application survives", 1, - tester.controller().applications().asList(app1.tenant()).stream() - .filter(app -> app.id().application().equals(app2.application())) - .count()); - } - @Test public void testGlobalRotations() throws IOException { // Setup tester and app def @@ -539,35 +493,6 @@ public class ControllerTest { tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); } - @Test - public void testDeploymentOfNewInstanceWithIllegalApplicationName() { - ControllerTester tester = new ControllerTester(); - String application = "this_application_name_is_far_too_long_and_has_underscores"; - ZoneId zone = ZoneId.from("test", "us-east-1"); - DeployOptions options = new DeployOptions(false, - Optional.empty(), - false, - false); - - tester.createTenant("tenant", "domain", null); - - // Deploy an application which doesn't yet exist, and which has an illegal application name. - try { - tester.controller().applications().deploy(ApplicationId.from("tenant", application, "123"), zone, Optional.empty(), options); - fail("Illegal application name should cause validation exception."); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid id")); - } - - // Sneak an illegal application in the back door. - tester.createApplication(new ApplicationSerializer().toSlime(new Application(ApplicationId.from("tenant", application, "default")))); - - // Deploy a PR instance for the application, with no NToken. - tester.controller().applications().deploy(ApplicationId.from("tenant", application, "456"), zone, Optional.empty(), options); - assertTrue(tester.controller().applications().get(ApplicationId.from("tenant", application, "456")).isPresent()); - } - private void runUpgrade(DeploymentTester tester, ApplicationId application, ApplicationVersion version) { Version next = Version.fromString("6.2"); tester.upgradeSystem(next); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index cf546d8b075..c722d30c885 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -63,7 +63,7 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { - throw new AssertionError(); + return new byte[0]; } private static int artifactHash(ApplicationId applicationId, String applicationVersion) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 09216eec3c7..291e6899a7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -170,11 +170,6 @@ public class VersionStatusTest { // Application without deployment Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L); - // Pull request builds - tester.controllerTester().createApplication(TenantName.from("tenant1"), - "ignored1", - "43", 1000); - assertEquals("All applications running on this version: High", Confidence.high, confidence(tester.controller(), version0)); -- cgit v1.2.3 From 9f13fe27df1f60c4c3da9ba23dc5ad3e395ae69a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 29 Jun 2018 14:01:51 +0200 Subject: Support aborting runs, and some JobRunner tests --- .../controller/deployment/JobController.java | 19 ++- .../hosted/controller/deployment/JobProfile.java | 15 +- .../hosted/controller/deployment/RunStatus.java | 31 +++- .../vespa/hosted/controller/deployment/Step.java | 12 +- .../controller/maintenance/DummyStepRunner.java | 3 +- .../hosted/controller/maintenance/JobRunner.java | 9 +- .../controller/maintenance/RealStepRunner.java | 183 --------------------- .../hosted/controller/maintenance/StepRunner.java | 5 +- .../hosted/controller/persistence/CuratorDb.java | 20 +-- .../controller/persistence/JobSerializer.java | 150 ----------------- .../controller/maintenance/JobRunnerTest.java | 163 +++++++++++++++++- .../controller/persistence/JobSerializerTest.java | 84 ---------- .../persistence/testdata/run-status.json | 2 +- 13 files changed, 232 insertions(+), 464 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 8093c4d7a18..1ad89cb7a72 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -147,12 +147,17 @@ public class JobController { /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. - RunStatus endedRun = run.finish(controller.clock().instant()); - locked(id.application(), id.type(), runs -> runs.put(run.id(), endedRun)); - return endedRun; + RunStatus finishedRun = run.finished(controller.clock().instant()); + locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun)); + return finishedRun; }); } + /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */ + public void abort(RunId id) { + locked(id, run -> run.aborted()); + } + /** Registers the given application, such that it may have deployment jobs run here. */ void register(ApplicationId id) { controller.applications().lockIfPresent(id, application -> @@ -164,8 +169,7 @@ public class JobController { byte[] applicationPackage, byte[] applicationTestJar) { AtomicReference version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) - throw new IllegalArgumentException(id + " is not built here!"); + controller.applications().store(application.withBuiltInternally(true)); long run = nextBuild(id); version.set(ApplicationVersion.from(revision, run)); @@ -177,7 +181,7 @@ public class JobController { return version.get(); } - /** Orders a run of the given type, and returns the id of the created job. */ + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void run(ApplicationId id, JobType type) { controller.applications().lockIfPresent(id, application -> { if ( ! application.get().deploymentJobs().builtInternally()) @@ -201,6 +205,7 @@ public class JobController { for (JobType type : jobs(id)) try (Lock __ = curator.lock(id, type)) { curator.deleteJobData(id, type); + // TODO jvenstad: Deactivate tester applications? } }); } @@ -218,7 +223,7 @@ public class JobController { private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, JobType.component, - 0, + Long.MAX_VALUE, // TODO jvenstad: Clean up this! number, Optional.of(revision), Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 82464820d9b..c36e0b3a39f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -20,9 +20,10 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(storeData, + EnumSet.of(runTests, deactivateTester, - deactivateReal)), + deactivateReal, + report)), stagingTest(EnumSet.of(deployInitialReal, installInitialReal, @@ -31,17 +32,19 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(storeData, + EnumSet.of(runTests, deactivateTester, - deactivateReal)), + deactivateReal, + report)), production(EnumSet.of(deployReal, installReal, deployTester, installTester, startTests), - EnumSet.of(storeData, - deactivateTester)); + EnumSet.of(runTests, + deactivateTester, + report)); private final Set steps; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 2ecac588d54..a91cc905add 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,36 +26,45 @@ public class RunStatus { private final Map steps; private final Instant start; private final Optional end; + private final boolean aborted; // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs. // For deserialisation only -- do not use! - public RunStatus(RunId id, Map steps, Instant start, Optional end) { + public RunStatus(RunId id, Map steps, Instant start, Optional end, boolean aborted) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.start = start; this.end = end; + this.aborted = aborted; } public static RunStatus initial(RunId id, Instant now) { EnumMap steps = new EnumMap<>(Step.class); JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); - return new RunStatus(id, steps, requireNonNull(now), Optional.empty()); + return new RunStatus(id, steps, requireNonNull(now), Optional.empty(), false); } public RunStatus with(Step.Status status, LockedStep step) { - if (end.isPresent()) + if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be further modified!"); EnumMap steps = new EnumMap<>(this.steps); steps.put(step.get(), requireNonNull(status)); - return new RunStatus(id, steps, start, end); + return new RunStatus(id, steps, start, end, aborted); } - public RunStatus finish(Instant now) { - if (end.isPresent()) + public RunStatus finished(Instant now) { + if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); - return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now)); + return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now), aborted); + } + + public RunStatus aborted() { + if (hasEnded()) + throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!"); + + return new RunStatus(id, new EnumMap<>(steps), start, end, true); } /** Returns the id of this run. */ @@ -86,7 +95,12 @@ public class RunStatus { /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { - return steps.values().contains(failed); + return aborted || steps.values().contains(failed); + } + + /** Returns whether the run has been forcefully aborted. */ + public boolean isAborted() { + return aborted; } /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ @@ -115,6 +129,7 @@ public class RunStatus { "id=" + id + ", start=" + start + ", end=" + end + + ", aborted=" + aborted + ", steps=" + steps + '}'; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index a1d07001331..2048c5ab353 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; /** @@ -19,6 +20,7 @@ import java.util.List; * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant * information from a failed run is stored, and that deployment does not occur after deactivation. * + * @see JobController * @author jonmv */ public enum Step { @@ -44,17 +46,17 @@ public enum Step { /** Ask the tester to run its tests. */ startTests(installReal, installTester), - /** Download data from the tester and store it. */ - storeData(startTests), + /** See that the tests are done running and store the test results. */ + runTests(startTests), /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, startTests), + deactivateReal(deployInitialReal, deployReal, runTests), /** Deactivate the tester. */ - deactivateTester(deployTester, storeData), + deactivateTester(deployTester, runTests), /** Report completion to deployment orchestration machinery. */ - report; + report(deactivateReal, deactivateTester); private final List prerequisites; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java index f6f418e70be..94a86d713b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -7,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; public class DummyStepRunner implements StepRunner { @Override - public Step.Status run(LockedStep step, RunStatus run) { + public Step.Status run(LockedStep step, RunId id) { return Step.Status.succeeded; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index eacf9c72992..1434c40bdee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -15,11 +15,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}. * + * @see JobController * @author jonmv */ public class JobRunner extends Maintainer { @@ -70,18 +72,21 @@ public class JobRunner extends Maintainer { /** Attempts to advance the status of the given step, for the given run. */ void advance(RunId id, Step step) { try { + AtomicBoolean changed = new AtomicBoolean(false); jobs.locked(id, step, lockedStep -> { jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we bail out. if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. - Step.Status status = runner.run(lockedStep, run); + Step.Status status = runner.run(lockedStep, run.id()); if (run.steps().get(step) != status) { jobs.update(run.id(), status, lockedStep); - advance(run); + changed.set(true); } }); }); + if (changed.get()) + jobs.active(id).ifPresent(this::advance); } catch (TimeoutException e) { // Something else is already advancing this step, or a prerequisite -- try again later! diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java deleted file mode 100644 index 14402e389df..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ /dev/null @@ -1,183 +0,0 @@ -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; - -import java.util.Optional; -import java.util.function.Supplier; - -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; - -/** - * Runs steps of a deployment job against its provided controller. - * - * @author jonmv - */ -public class RealStepRunner implements StepRunner { - - private static ApplicationId testerOf(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value(), - id.instance().value() + "-t"); - } - - private final Controller controller; - - public RealStepRunner(Controller controller) { - this.controller = controller; - } - - @Override - public Status run(LockedStep step, RunStatus run) { - RunId id = run.id(); - switch (step.get()) { - case deployInitialReal: return deployInitialReal(id); - case installInitialReal: return installInitialReal(id); - case deployReal: return deployReal(id); - case deployTester: return deployTester(id); - case installReal: return installReal(id); - case installTester: return installTester(id); - case startTests: return startTests(id); - case storeData: return storeData(id); - case deactivateReal: return deactivateReal(id); - case deactivateTester: return deactivateTester(id); - case report: return report(id); - default: throw new AssertionError("Unknown step '" + step + "'!"); - } - } - - private Status deployInitialReal(RunId id) { - return deployReal(id, true); - } - - private Status deployReal(RunId id) { - // Separate out deploy logic from above, and reuse. - return deployReal(id, false); - } - - private Status deployReal(RunId id, boolean setTheStage) { - return deploy(id.application(), - id.type(), - () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), - Optional.empty(), - new DeployOptions(false, - Optional.empty(), - false, - setTheStage))); - } - - private Status deployTester(RunId id) { - // Find endpoints of real application. This will move down at a later time. - // See above. - return deploy(testerOf(id.application()), - id.type(), - () -> controller.applications().deployTester(testerOf(id.application()), - testerPackage(id), - id.type().zone(controller.system()).get(), - new DeployOptions(true, - Optional.of(controller.systemVersion()), - false, - false))); - } - - private Status deploy(ApplicationId id, JobType type, Supplier deploy) { - try { - // TODO jvenstad: Do whatever is required based on the result, and log all of this. - ActivateResult result = deploy.get(); - - return succeeded; - } - catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. - if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() - || e.getErrorCode() == ACTIVATION_CONFLICT - || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { - - return unfinished; - } - } - return failed; - } - - private Status installInitialReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installTester(RunId id) { - return install(testerOf(id.application()), id.type()); - } - - private Status install(ApplicationId id, JobType type) { - // If converged and serviceconverged: succeeded - // If timeout, failed - return unfinished; - } - - private Status startTests(RunId id) { - // Empty for now, but will be: find endpoints and post them. - throw new AssertionError(); - } - - private Status storeData(RunId id) { - // Update test logs. - // If tests are done, return test results. - throw new AssertionError(); - } - - private Status deactivateReal(RunId id) { - return deactivate(id.application(), id.type()); - } - - private Status deactivateTester(RunId id) { - return deactivate(testerOf(id.application()), id.type()); - } - - private Status deactivate(ApplicationId id, JobType type) { - // Try to deactivate, and if deactivated, finished. - throw new AssertionError(); - } - - private Status report(RunId id) { - // Easy squeezy. - throw new AssertionError(); - } - - private Application application(ApplicationId id) { - return controller.applications().require(id); - } - - private ApplicationPackage testerPackage(RunId id) { - ApplicationVersion version = application(id.application()).deploymentJobs() - .statusOf(id.type()).get() - .lastTriggered().get() - .application(); - - - // TODO hakonhall: Fetch! - throw new AssertionError(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java index ebb4c91edcc..400e9b4f74b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -17,8 +18,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; */ public interface StepRunner { - /** Attempts to run the given locked step, and returns its new status. */ - Step.Status run(LockedStep step, RunStatus run); + /** Attempts to run the given locked step in the given run, and returns its new status. */ + Step.Status run(LockedStep step, RunId id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 15ddcdace5b..dcbc9479845 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -69,7 +69,7 @@ public class CuratorDb { private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); - private final JobSerializer jobSerializer = new JobSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); private final Curator curator; @@ -308,25 +308,25 @@ public class CuratorDb { // -------------- Job Runs ------------------------------------------------ public void writeLastRun(RunStatus run) { - curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(jobSerializer.toSlime(run))); + curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); } public void writeHistoricRuns(ApplicationId id, JobType type, Iterable runs) { - curator.set(jobPath(id, type), asJson(jobSerializer.toSlime(runs))); - } - - public void deleteJobData(ApplicationId id, JobType type) { - curator.delete(jobPath(id, type)); - curator.delete(lastRunPath(id, type)); + curator.set(jobPath(id, type), asJson(runSerializer.toSlime(runs))); } public Optional readLastRun(ApplicationId id, JobType type) { - return readSlime(jobPath(id, type)).map(jobSerializer::runFromSlime); + return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime); } public Map readHistoricRuns(ApplicationId id, JobType type) { // TODO jvenstad: Add, somewhere, a retention filter based on age or count. - return readSlime(jobPath(id, type)).map(jobSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); + return readSlime(jobPath(id, type)).map(runSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); + } + + public void deleteJobData(ApplicationId id, JobType type) { + curator.delete(jobPath(id, type)); + curator.delete(lastRunPath(id, type)); } // -------------- Provisioning (called by internal code) ------------------ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java deleted file mode 100644 index 503f6fd990e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; - -import java.time.Instant; -import java.util.EnumMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; - -public class JobSerializer { - - private static final String stepsField = "steps"; - private static final String applicationField = "id"; - private static final String jobTypeField = "type"; - private static final String numberField = "number"; - private static final String startField = "start"; - private static final String endField = "end"; - - public RunStatus runFromSlime(Slime slime) { - return runFromSlime(slime.get()); - } - - public Map runsFromSlime(Slime slime) { - Map runs = new LinkedHashMap<>(); - Inspector runArray = slime.get(); - runArray.traverse((ArrayTraverser) (__, runObject) -> { - RunStatus run = runFromSlime(runObject); - runs.put(run.id(), run); - }); - - return runs; - } - - private RunStatus runFromSlime(Inspector runObject) { - EnumMap steps = new EnumMap<>(Step.class); - runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(stepOf(step), statusOf(status.asString())); - }); - return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()), - steps, - Instant.ofEpochMilli(runObject.field(startField).asLong()), - Optional.of(runObject.field(endField)) - .filter(Inspector::valid) - .map(end -> Instant.ofEpochMilli(end.asLong()))); - } - - public Slime toSlime(RunStatus run) { - Slime slime = new Slime(); - toSlime(run, slime.setObject()); - return slime; - } - - public Slime toSlime(Iterable runs) { - Slime slime = new Slime(); - Cursor runArray = slime.setArray(); - runs.forEach(run -> toSlime(run, runArray.addObject())); - return slime; - } - - private void toSlime(RunStatus run, Cursor runObject) { - runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().jobName()); - runObject.setLong(numberField, run.id().number()); - runObject.setLong(startField, run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); - Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); - } - - static String valueOf(Step step) { - switch (step) { - case deployInitialReal : return "DIR"; - case installInitialReal : return "IIR"; - case deployReal : return "DR" ; - case installReal : return "IR" ; - case deactivateReal : return "DAR"; - case deployTester : return "DT" ; - case installTester : return "IT" ; - case deactivateTester : return "DAT"; - case startTests : return "ST" ; - case storeData : return "SD" ; - case report : return "R" ; - default : throw new AssertionError("No value defined for '" + step + "'!"); - } - } - - static Step stepOf(String step) { - switch (step) { - case "DIR" : return deployInitialReal ; - case "IIR" : return installInitialReal; - case "DR" : return deployReal ; - case "IR" : return installReal ; - case "DAR" : return deactivateReal ; - case "DT" : return deployTester ; - case "IT" : return installTester ; - case "DAT" : return deactivateTester ; - case "ST" : return startTests ; - case "SD" : return storeData ; - case "R" : return report ; - default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); - } - } - - static String valueOf(Status status) { - switch (status) { - case unfinished : return "U"; - case failed : return "F"; - case succeeded : return "S"; - default : throw new AssertionError("No value defined for '" + status + "'!"); - } - } - - static Status statusOf(String status) { - switch (status) { - case "U" : return unfinished; - case "F" : return failed ; - case "S" : return succeeded ; - default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); - } - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 2a2a6a5e1b9..67c1854e15a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,15 +1,46 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import org.junit.Test; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.List; +import java.util.Map; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author jonmv @@ -17,11 +48,116 @@ import java.util.concurrent.atomic.AtomicBoolean; public class JobRunnerTest { @Test - public void test() { - ControllerTester tester = new ControllerTester(); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()), - inThreadExecutor(), new DummyStepRunner()); + public void testMultiThreadedExecutionFinishes() throws InterruptedException { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + Executors.newFixedThreadPool(32), sleepy(new DummyStepRunner())); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + + jobs.run(id, systemTest); + try { + jobs.run(id, systemTest); + fail("Job is already running, so this should not be allowed!"); + } + catch (IllegalStateException e) { } + jobs.run(id, stagingTest); + + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); runner.maintain(); + assertFalse(jobs.last(id, systemTest).get().hasEnded()); + Thread.sleep(1000); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine". + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); + } + + @Test + public void testStepLogic() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + Map outcomes = new EnumMap<>(Step.class); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + inThreadExecutor(), mappedRunner(outcomes)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + Supplier run = () -> jobs.last(id, systemTest).get(); + + jobs.run(id, systemTest); + RunId first = run.get().id(); + + // Unfinished steps change nothing. + Map steps = run.get().steps(); + runner.maintain(); + assertEquals(steps, run.get().steps()); + assertEquals(Arrays.asList(deployReal), run.get().readySteps()); + + // Deployment allows installation. + outcomes.put(deployReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installReal), run.get().readySteps()); + + // Installation allows tester deployment. + outcomes.put(installReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deployTester), run.get().readySteps()); + + // Tester deployment allows tester installation. + outcomes.put(deployTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installTester), run.get().readySteps()); + + // Tester installation allows starting tests. + outcomes.put(installTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(startTests), run.get().readySteps()); + + // Starting tests allows storing data. + outcomes.put(startTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(runTests), run.get().readySteps()); + + // Storing data allows deactivating tester. + outcomes.put(runTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Failure deactivating real fails the run, but run-always steps continue. + outcomes.put(deactivateReal, failed); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Aborting the run now does nothing, as only run-always steps are left. + jobs.abort(run.get().id()); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Success of the remaining run-always steps ends the run. + outcomes.put(deactivateReal, succeeded); + outcomes.put(deactivateTester, succeeded); + outcomes.put(report, succeeded); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().isAborted()); + + // A new run is attempted. + jobs.run(id, systemTest); + assertEquals(first.number() + 1, run.get().id().number()); + + // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. + outcomes.put(deployTester, failed); + runner.maintain(); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().hasFailed()); + assertFalse(run.get().isAborted()); + assertEquals(failed, run.get().steps().get(deployTester)); + assertEquals(unfinished, run.get().steps().get(installTester)); + assertEquals(succeeded, run.get().steps().get(report)); } private static ExecutorService inThreadExecutor() { @@ -36,4 +172,21 @@ public class JobRunnerTest { }; } + + private static StepRunner sleepy(StepRunner runner) { + return (step, id) -> { + try { + Thread.sleep(10); + } + catch (InterruptedException e) { + throw new AssertionError("Not supposed to happen."); + } + return runner.run(step, id); + }; + } + + private static StepRunner mappedRunner(Map outcomes) { + return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java deleted file mode 100644 index e6e726bec5e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.yahoo.vespa.hosted.controller.persistence; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import org.junit.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.Collections; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class JobSerializerTest { - - private static final JobSerializer serializer = new JobSerializer(); - private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); - private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), - JobType.productionUsEast3, - (long) 112358); - private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); - - @Test - public void testSerialization() throws IOException { - for (Step step : Step.values()) - assertEquals(step, JobSerializer.stepOf(JobSerializer.valueOf(step))); - - for (Step.Status status : Step.Status.values()) - assertEquals(status, JobSerializer.statusOf(JobSerializer.valueOf(status))); - - // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! - RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); - for (Step step : Step.values()) - assertTrue(run.steps().containsKey(step)); - - assertEquals(id, run.id()); - assertEquals(start, run.start()); - assertFalse(run.hasEnded()); - assertEquals(ImmutableMap.builder() - .put(deployInitialReal, unfinished) - .put(installInitialReal, failed) - .put(deployReal, succeeded) - .put(installReal, unfinished) - .put(deactivateReal, failed) - .put(deployTester, succeeded) - .put(installTester, unfinished) - .put(deactivateTester, failed) - .put(startTests, succeeded) - .put(storeData, unfinished) - .put(report, failed) - .build(), - run.steps()); - - RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); - assertEquals(run.id(), phoenix.id()); - assertEquals(run.start(), phoenix.start()); - assertEquals(run.end(), phoenix.end()); - assertEquals(run.steps(), phoenix.steps()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 4db94eb429a..e31d14ac181 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -14,7 +14,7 @@ "IT": "U", "DAT": "F", "ST": "S", - "SD": "U", + "RT": "U", "R": "F" } } -- cgit v1.2.3 From 54258144705529ba2bdecc46322b10682262db7a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 29 Jun 2018 14:02:24 +0200 Subject: Re-add files that were lost by rename + reset --- .../controller/maintenance/InternalStepRunner.java | 180 +++++++++++++++++++++ .../controller/persistence/RunSerializer.java | 157 ++++++++++++++++++ .../controller/persistence/RunSerializerTest.java | 90 +++++++++++ 3 files changed, 427 insertions(+) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java new file mode 100644 index 00000000000..41d719f37ed --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java @@ -0,0 +1,180 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; + +import java.util.Optional; +import java.util.function.Supplier; + +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; + +/** + * Runs steps of a deployment job against its provided controller. + * + * @author jonmv + */ +public class InternalStepRunner implements StepRunner { + + private static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + id.instance().value() + "-t"); + } + + private final Controller controller; + + public InternalStepRunner(Controller controller) { + this.controller = controller; + } + + @Override + public Status run(LockedStep step, RunId id) { + switch (step.get()) { + case deployInitialReal: return deployInitialReal(id); + case installInitialReal: return installInitialReal(id); + case deployReal: return deployReal(id); + case deployTester: return deployTester(id); + case installReal: return installReal(id); + case installTester: return installTester(id); + case startTests: return startTests(id); + case runTests: return storeData(id); + case deactivateReal: return deactivateReal(id); + case deactivateTester: return deactivateTester(id); + case report: return report(id); + default: throw new AssertionError("Unknown step '" + step + "'!"); + } + } + + private Status deployInitialReal(RunId id) { + return deployReal(id, true); + } + + private Status deployReal(RunId id) { + // Separate out deploy logic from above, and reuse. + return deployReal(id, false); + } + + private Status deployReal(RunId id, boolean setTheStage) { + return deploy(id.application(), + id.type(), + () -> controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, + Optional.empty(), + false, + setTheStage))); + } + + private Status deployTester(RunId id) { + // Find endpoints of real application. This will move down at a later time. + // See above. + return deploy(testerOf(id.application()), + id.type(), + () -> controller.applications().deployTester(testerOf(id.application()), + testerPackage(id), + id.type().zone(controller.system()).get(), + new DeployOptions(true, + Optional.of(controller.systemVersion()), + false, + false))); + } + + private Status deploy(ApplicationId id, JobType type, Supplier deploy) { + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + ActivateResult result = deploy.get(); + + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() + || e.getErrorCode() == ACTIVATION_CONFLICT + || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { + + return unfinished; + } + } + return failed; + } + + private Status installInitialReal(RunId id) { + return install(id.application(), id.type()); + } + + private Status installReal(RunId id) { + return install(id.application(), id.type()); + } + + private Status installTester(RunId id) { + return install(testerOf(id.application()), id.type()); + } + + private Status install(ApplicationId id, JobType type) { + // If converged and serviceconverged: succeeded + // If timeout, failed + return unfinished; + } + + private Status startTests(RunId id) { + // Empty for now, but will be: find endpoints and post them. + throw new AssertionError(); + } + + private Status storeData(RunId id) { + // Update test logs. + // If tests are done, return test results. + throw new AssertionError(); + } + + private Status deactivateReal(RunId id) { + return deactivate(id.application(), id.type()); + } + + private Status deactivateTester(RunId id) { + return deactivate(testerOf(id.application()), id.type()); + } + + private Status deactivate(ApplicationId id, JobType type) { + // Try to deactivate, and if deactivated, finished. + throw new AssertionError(); + } + + private Status report(RunId id) { + // Easy squeezy. + throw new AssertionError(); + } + + private Application application(ApplicationId id) { + return controller.applications().require(id); + } + + private ApplicationPackage testerPackage(RunId id) { + ApplicationVersion version = application(id.application()).deploymentJobs() + .statusOf(id.type()).get() + .lastTriggered().get() + .application(); + + + // TODO hakonhall: Fetch! + throw new AssertionError(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java new file mode 100644 index 00000000000..c7d782c8302 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -0,0 +1,157 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; + +import java.time.Instant; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; + +/** + * Serialises and deserialises RunStatus objects for persistent storage. + * + * @author jonmv + */ +public class RunSerializer { + + private static final String stepsField = "steps"; + private static final String applicationField = "id"; + private static final String jobTypeField = "type"; + private static final String numberField = "number"; + private static final String startField = "start"; + private static final String endField = "end"; + private static final String abortedField = "aborted"; + + RunStatus runFromSlime(Slime slime) { + return runFromSlime(slime.get()); + } + + Map runsFromSlime(Slime slime) { + Map runs = new LinkedHashMap<>(); + Inspector runArray = slime.get(); + runArray.traverse((ArrayTraverser) (__, runObject) -> { + RunStatus run = runFromSlime(runObject); + runs.put(run.id(), run); + }); + return runs; + } + + private RunStatus runFromSlime(Inspector runObject) { + EnumMap steps = new EnumMap<>(Step.class); + runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { + steps.put(stepOf(step), statusOf(status.asString())); + }); + return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.fromJobName(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()), + steps, + Instant.ofEpochMilli(runObject.field(startField).asLong()), + Optional.of(runObject.field(endField)) + .filter(Inspector::valid) + .map(end -> Instant.ofEpochMilli(end.asLong())), + runObject.field(abortedField).asBool()); + } + + Slime toSlime(Iterable runs) { + Slime slime = new Slime(); + Cursor runArray = slime.setArray(); + runs.forEach(run -> toSlime(run, runArray.addObject())); + return slime; + } + + Slime toSlime(RunStatus run) { + Slime slime = new Slime(); + toSlime(run, slime.setObject()); + return slime; + } + + private void toSlime(RunStatus run, Cursor runObject) { + runObject.setString(applicationField, run.id().application().serializedForm()); + runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setLong(numberField, run.id().number()); + runObject.setLong(startField, run.start().toEpochMilli()); + run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); + if (run.isAborted()) runObject.setBool(abortedField, true); + Cursor stepsObject = runObject.setObject(stepsField); + run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + } + + static String valueOf(Step step) { + switch (step) { + case deployInitialReal : return "DIR"; + case installInitialReal : return "IIR"; + case deployReal : return "DR" ; + case installReal : return "IR" ; + case deactivateReal : return "DAR"; + case deployTester : return "DT" ; + case installTester : return "IT" ; + case deactivateTester : return "DAT"; + case startTests : return "ST" ; + case runTests : return "RT" ; + case report : return "R" ; + default : throw new AssertionError("No value defined for '" + step + "'!"); + } + } + + static Step stepOf(String step) { + switch (step) { + case "DIR" : return deployInitialReal ; + case "IIR" : return installInitialReal; + case "DR" : return deployReal ; + case "IR" : return installReal ; + case "DAR" : return deactivateReal ; + case "DT" : return deployTester ; + case "IT" : return installTester ; + case "DAT" : return deactivateTester ; + case "ST" : return startTests ; + case "RT" : return runTests; + case "R" : return report ; + default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + } + } + + static String valueOf(Status status) { + switch (status) { + case unfinished : return "U"; + case failed : return "F"; + case succeeded : return "S"; + default : throw new AssertionError("No value defined for '" + status + "'!"); + } + } + + static Status statusOf(String status) { + switch (status) { + case "U" : return unfinished; + case "F" : return failed ; + case "S" : return succeeded ; + default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java new file mode 100644 index 00000000000..01acc401a1d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -0,0 +1,90 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.Collections; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class RunSerializerTest { + + private static final RunSerializer serializer = new RunSerializer(); + private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); + private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), + JobType.productionUsEast3, + (long) 112358); + private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); + + @Test + public void testSerialization() throws IOException { + for (Step step : Step.values()) + assertEquals(step, RunSerializer.stepOf(RunSerializer.valueOf(step))); + + for (Step.Status status : Step.Status.values()) + assertEquals(status, RunSerializer.statusOf(RunSerializer.valueOf(status))); + + // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! + RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); + for (Step step : Step.values()) + assertTrue(run.steps().containsKey(step)); + + assertEquals(id, run.id()); + assertEquals(start, run.start()); + assertFalse(run.hasEnded()); + assertFalse(run.isAborted()); + assertEquals(ImmutableMap.builder() + .put(deployInitialReal, unfinished) + .put(installInitialReal, failed) + .put(deployReal, succeeded) + .put(installReal, unfinished) + .put(deactivateReal, failed) + .put(deployTester, succeeded) + .put(installTester, unfinished) + .put(deactivateTester, failed) + .put(startTests, succeeded) + .put(runTests, unfinished) + .put(report, failed) + .build(), + run.steps()); + + run = run.aborted().finished(Instant.now()); + assertTrue(run.isAborted()); + assertTrue(run.hasEnded()); + + RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); + assertEquals(run.id(), phoenix.id()); + assertEquals(run.start(), phoenix.start()); + assertEquals(run.end(), phoenix.end()); + assertEquals(run.isAborted(), phoenix.isAborted()); + assertEquals(run.steps(), phoenix.steps()); + } + +} -- cgit v1.2.3 From bb7e81007fcf13bf83ab37e63c34cc88a5fae3eb Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 29 Jun 2018 14:41:45 +0200 Subject: Collect garbage on controller restart --- .../yahoo/vespa/hosted/controller/Controller.java | 3 ++ .../controller/deployment/JobController.java | 34 ++++++++++++++++++---- .../controller/maintenance/InternalStepRunner.java | 3 +- .../controller/maintenance/JobRunnerTest.java | 10 +++++-- 4 files changed, 41 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 631125d9368..05d5c737c9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -133,6 +133,9 @@ public class Controller extends AbstractComponent { // Record the version of this controller curator().writeControllerVersion(this.hostname(), Vtag.currentVersion); + + jobController.collectGarbage(); + jobController.updateStorage(); } /** Returns the instance controlling tenants */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 1ad89cb7a72..329f7ac62a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -5,7 +5,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; @@ -24,6 +26,7 @@ import java.util.function.UnaryOperator; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; +import static com.yahoo.vespa.hosted.controller.maintenance.InternalStepRunner.testerOf; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -198,18 +201,37 @@ public class JobController { }); } - /** Unregisters the given application and deletes all associated data. */ + /** Unregisters the given application and makes all associated data eligible for garbage collection. */ public void unregister(ApplicationId id) { controller.applications().lockIfPresent(id, application -> { controller.applications().store(application.withBuiltInternally(false)); - for (JobType type : jobs(id)) - try (Lock __ = curator.lock(id, type)) { - curator.deleteJobData(id, type); - // TODO jvenstad: Deactivate tester applications? - } }); } + public void collectGarbage() { + controller.applications().asList().stream() + .filter(application -> ! application.deploymentJobs().builtInternally()) + .map(Application::id) + .forEach(id -> { + for (JobType type : jobs(id)) + try (Lock __ = curator.lock(id, type)) { + if ( ! active(last(id, type).get().id()).isPresent()) + deactivateTester(id, type); + curator.deleteJobData(id, type); + } + }); + } + + // TODO jvenstad: Urgh, clean this up somehow? + public void deactivateTester(ApplicationId id, JobType type) { + try { + controller.configServer().deactivate(new DeploymentId(testerOf(id), type.zone(controller.system()).get())); + } + catch (NoInstanceException ignored) { + // ok; already gone + } + } + // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. private long nextBuild(ApplicationId id) { return 1 + controller.applications().require(id).deploymentJobs() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java index 41d719f37ed..3ac880e436b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java @@ -30,7 +30,8 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe */ public class InternalStepRunner implements StepRunner { - private static ApplicationId testerOf(ApplicationId id) { + // TODO jvenstad: Move this tester logic to the application controller, perhaps? + public static ApplicationId testerOf(ApplicationId id) { return ApplicationId.from(id.tenant().value(), id.application().value(), id.instance().value() + "-t"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 67c1854e15a..a43b0e05f11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -32,6 +32,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; @@ -51,8 +52,10 @@ public class JobRunnerTest { public void testMultiThreadedExecutionFinishes() throws InterruptedException { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); + // Fail the installation of the initial version of the real application in staging tests, and succeed everything else. + StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == installInitialReal ? failed : succeeded; JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), - Executors.newFixedThreadPool(32), sleepy(new DummyStepRunner())); + Executors.newFixedThreadPool(32), sleepy(stepRunner)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); @@ -68,8 +71,11 @@ public class JobRunnerTest { assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); runner.maintain(); assertFalse(jobs.last(id, systemTest).get().hasEnded()); - Thread.sleep(1000); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine". + assertFalse(jobs.last(id, stagingTest).get().hasEnded()); + Thread.sleep(500); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine". assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); + assertTrue(jobs.last(id, stagingTest).get().hasEnded()); + assertTrue(jobs.last(id, stagingTest).get().hasFailed()); } @Test -- cgit v1.2.3 From 160f13236f76b73618cc331bc7b4980f0564e004 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 2 Jul 2018 12:54:51 +0200 Subject: Thorough GC of job resources --- .../yahoo/vespa/hosted/controller/Controller.java | 1 - .../controller/deployment/JobController.java | 38 +++++++++++++++------- .../hosted/controller/deployment/JobProfile.java | 7 ++-- .../vespa/hosted/controller/deployment/Step.java | 11 +++---- .../controller/maintenance/InternalStepRunner.java | 4 +-- .../hosted/controller/maintenance/JobRunner.java | 1 + .../hosted/controller/persistence/CuratorDb.java | 10 ++++++ .../controller/persistence/RunSerializer.java | 6 ++-- .../controller/maintenance/JobRunnerTest.java | 10 +++--- .../controller/persistence/RunSerializerTest.java | 4 +-- .../persistence/testdata/run-status.json | 2 +- 11 files changed, 59 insertions(+), 35 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 05d5c737c9b..790d6d00035 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -134,7 +134,6 @@ public class Controller extends AbstractComponent { // Record the version of this controller curator().writeControllerVersion(this.hostname(), Vtag.currentVersion); - jobController.collectGarbage(); jobController.updateStorage(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 329f7ac62a4..abcde079271 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -16,9 +16,11 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -59,8 +61,8 @@ public class JobController { for (ApplicationId id : applications()) for (JobType type : jobs(id)) { locked(id, type, runs -> { + last(id, type).ifPresent(last -> locked(last.id(), run -> run)); }); - last(id, type).ifPresent(last -> locked(last.id(), run -> run)); } } @@ -205,21 +207,33 @@ public class JobController { public void unregister(ApplicationId id) { controller.applications().lockIfPresent(id, application -> { controller.applications().store(application.withBuiltInternally(false)); + jobs(id).forEach(type -> { + try (Lock __ = curator.lock(id, type)) { + last(id, type).ifPresent(last -> active(last.id()).ifPresent(active -> finish(active.id()))); + } + }); }); } + /** Deletes stale data and tester deployments for applications which are unknown, or no longer built internally. */ public void collectGarbage() { - controller.applications().asList().stream() - .filter(application -> ! application.deploymentJobs().builtInternally()) - .map(Application::id) - .forEach(id -> { - for (JobType type : jobs(id)) - try (Lock __ = curator.lock(id, type)) { - if ( ! active(last(id, type).get().id()).isPresent()) - deactivateTester(id, type); - curator.deleteJobData(id, type); - } - }); + Set applicationsToBuild = new HashSet<>(applications()); + curator.applicationsWithJobs().stream() + .filter(id -> ! applicationsToBuild.contains(id)) + .forEach(id -> { + try { + for (JobType type : jobs(id)) + try (Lock __ = curator.lock(id, type); + Lock ___ = curator.lock(id, type, Step.deactivateTester)) { + deactivateTester(id, type); + curator.deleteJobData(id, type); + } + } + catch (TimeoutException e) { + return; // Don't remove the data if we couldn't deactivate all testers. + } + curator.deleteJobData(id); + }); } // TODO jvenstad: Urgh, clean this up somehow? diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index c36e0b3a39f..120c4f282ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -15,12 +15,13 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.*; */ public enum JobProfile { + // TODO jvenstad: runTests is not a run-always step, as it really means: check if tests are done, and store whatever is ready. systemTest(EnumSet.of(deployReal, installReal, deployTester, installTester, startTests), - EnumSet.of(runTests, + EnumSet.of(endTests, deactivateTester, deactivateReal, report)), @@ -32,7 +33,7 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(runTests, + EnumSet.of(endTests, deactivateTester, deactivateReal, report)), @@ -42,7 +43,7 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(runTests, + EnumSet.of(endTests, deactivateTester, report)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 2048c5ab353..1ef49ee7499 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.List; /** @@ -46,16 +45,16 @@ public enum Step { /** Ask the tester to run its tests. */ startTests(installReal, installTester), - /** See that the tests are done running and store the test results. */ - runTests(startTests), + /** See that the tests are done running. */ + endTests(startTests), /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, runTests), + deactivateReal(deployInitialReal, deployReal, endTests), /** Deactivate the tester. */ - deactivateTester(deployTester, runTests), + deactivateTester(deployTester, endTests), - /** Report completion to deployment orchestration machinery. */ + /** Report completion to the deployment orchestration machinery. */ report(deactivateReal, deactivateTester); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java index 3ac880e436b..cf20084a1d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java @@ -53,7 +53,7 @@ public class InternalStepRunner implements StepRunner { case installReal: return installReal(id); case installTester: return installTester(id); case startTests: return startTests(id); - case runTests: return storeData(id); + case endTests: return endTests(id); case deactivateReal: return deactivateReal(id); case deactivateTester: return deactivateTester(id); case report: return report(id); @@ -139,7 +139,7 @@ public class InternalStepRunner implements StepRunner { throw new AssertionError(); } - private Status storeData(RunId id) { + private Status endTests(RunId id) { // Update test logs. // If tests are done, return test results. throw new AssertionError(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 1434c40bdee..6649f705821 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -47,6 +47,7 @@ public class JobRunner extends Maintainer { @Override protected void maintain() { jobs.active().forEach(this::advance); + jobs.collectGarbage(); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index dcbc9479845..49e8d7498ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -329,6 +329,16 @@ public class CuratorDb { curator.delete(lastRunPath(id, type)); } + public void deleteJobData(ApplicationId id) { + curator.delete(jobRoot.append(id.serializedForm())); + } + + public List applicationsWithJobs() { + return curator.getChildren(jobRoot).stream() + .map(ApplicationId::fromSerializedForm) + .collect(Collectors.toList()); + } + // -------------- Provisioning (called by internal code) ------------------ @SuppressWarnings("unused") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index c7d782c8302..7df60278390 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -31,7 +31,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; /** * Serialises and deserialises RunStatus objects for persistent storage. @@ -113,7 +113,7 @@ public class RunSerializer { case installTester : return "IT" ; case deactivateTester : return "DAT"; case startTests : return "ST" ; - case runTests : return "RT" ; + case endTests : return "ET" ; case report : return "R" ; default : throw new AssertionError("No value defined for '" + step + "'!"); } @@ -130,7 +130,7 @@ public class RunSerializer { case "IT" : return installTester ; case "DAT" : return deactivateTester ; case "ST" : return startTests ; - case "RT" : return runTests; + case "ET" : return endTests ; case "R" : return report ; default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index a43b0e05f11..406e64feb24 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -37,7 +37,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -49,7 +49,7 @@ import static org.junit.Assert.fail; public class JobRunnerTest { @Test - public void testMultiThreadedExecutionFinishes() throws InterruptedException { + public void multiThreadedExecutionFinishes() throws InterruptedException { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); // Fail the installation of the initial version of the real application in staging tests, and succeed everything else. @@ -79,7 +79,7 @@ public class JobRunnerTest { } @Test - public void testStepLogic() { + public void stepLogic() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); Map outcomes = new EnumMap<>(Step.class); @@ -122,10 +122,10 @@ public class JobRunnerTest { // Starting tests allows storing data. outcomes.put(startTests, succeeded); runner.maintain(); - assertEquals(Arrays.asList(runTests), run.get().readySteps()); + assertEquals(Arrays.asList(endTests), run.get().readySteps()); // Storing data allows deactivating tester. - outcomes.put(runTests, succeeded); + outcomes.put(endTests, succeeded); runner.maintain(); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 01acc401a1d..12640a5e8fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -29,7 +29,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -70,7 +70,7 @@ public class RunSerializerTest { .put(installTester, unfinished) .put(deactivateTester, failed) .put(startTests, succeeded) - .put(runTests, unfinished) + .put(endTests, unfinished) .put(report, failed) .build(), run.steps()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index e31d14ac181..d659bd9fff0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -14,7 +14,7 @@ "IT": "U", "DAT": "F", "ST": "S", - "RT": "U", + "ET": "U", "R": "F" } } -- cgit v1.2.3 From 75afc0fcee7ff40db9dcf3421d27fbd85f6a11fe Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 2 Jul 2018 13:40:56 +0200 Subject: endTests is not run-always, and avoid Thread.sleep in unit test --- .../hosted/controller/deployment/JobProfile.java | 18 +++++------ .../vespa/hosted/controller/deployment/Step.java | 4 ++- .../controller/maintenance/JobRunnerTest.java | 36 +++++++++------------- 3 files changed, 27 insertions(+), 31 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 120c4f282ec..0cad9e98d5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -20,9 +20,9 @@ public enum JobProfile { installReal, deployTester, installTester, - startTests), - EnumSet.of(endTests, - deactivateTester, + startTests, + endTests), + EnumSet.of(deactivateTester, deactivateReal, report)), @@ -32,9 +32,9 @@ public enum JobProfile { installReal, deployTester, installTester, - startTests), - EnumSet.of(endTests, - deactivateTester, + startTests, + endTests), + EnumSet.of(deactivateTester, deactivateReal, report)), @@ -42,9 +42,9 @@ public enum JobProfile { installReal, deployTester, installTester, - startTests), - EnumSet.of(endTests, - deactivateTester, + startTests, + endTests), + EnumSet.of(deactivateTester, report)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 1ef49ee7499..98b4294d47a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -1,5 +1,7 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableList; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -61,7 +63,7 @@ public enum Step { private final List prerequisites; Step(Step... prerequisites) { - this.prerequisites = Collections.unmodifiableList(Arrays.asList(prerequisites)); + this.prerequisites = ImmutableList.copyOf(prerequisites); } public List prerequisites() { return prerequisites; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 406e64feb24..4d789ba13cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -17,6 +17,7 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -54,8 +55,9 @@ public class JobRunnerTest { JobController jobs = tester.controller().jobController(); // Fail the installation of the initial version of the real application in staging tests, and succeed everything else. StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == installInitialReal ? failed : succeeded; + CountDownLatch latch = new CountDownLatch(14); // Number of steps that will run, below. JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), - Executors.newFixedThreadPool(32), sleepy(stepRunner)); + Executors.newFixedThreadPool(32), notifying(stepRunner, latch)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); @@ -72,7 +74,11 @@ public class JobRunnerTest { runner.maintain(); assertFalse(jobs.last(id, systemTest).get().hasEnded()); assertFalse(jobs.last(id, stagingTest).get().hasEnded()); - Thread.sleep(500); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine". + + latch.await(1, TimeUnit.SECONDS); + assertEquals(0, latch.getCount()); + + jobs.active().forEach(run -> jobs.active(run.id())); // Wait for locks of jobs which haven't yet been written through. assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); @@ -93,38 +99,31 @@ public class JobRunnerTest { jobs.run(id, systemTest); RunId first = run.get().id(); - // Unfinished steps change nothing. Map steps = run.get().steps(); runner.maintain(); assertEquals(steps, run.get().steps()); assertEquals(Arrays.asList(deployReal), run.get().readySteps()); - // Deployment allows installation. outcomes.put(deployReal, succeeded); runner.maintain(); assertEquals(Arrays.asList(installReal), run.get().readySteps()); - // Installation allows tester deployment. outcomes.put(installReal, succeeded); runner.maintain(); assertEquals(Arrays.asList(deployTester), run.get().readySteps()); - // Tester deployment allows tester installation. outcomes.put(deployTester, succeeded); runner.maintain(); assertEquals(Arrays.asList(installTester), run.get().readySteps()); - // Tester installation allows starting tests. outcomes.put(installTester, succeeded); runner.maintain(); assertEquals(Arrays.asList(startTests), run.get().readySteps()); - // Starting tests allows storing data. outcomes.put(startTests, succeeded); runner.maintain(); assertEquals(Arrays.asList(endTests), run.get().readySteps()); - // Storing data allows deactivating tester. outcomes.put(endTests, succeeded); runner.maintain(); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); @@ -134,15 +133,12 @@ public class JobRunnerTest { runner.maintain(); assertTrue(run.get().hasFailed()); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); - runner.maintain(); - assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); - // Aborting the run now does nothing, as only run-always steps are left. + // Abortion does nothing, as the run has already failed. jobs.abort(run.get().id()); runner.maintain(); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); - // Success of the remaining run-always steps ends the run. outcomes.put(deactivateReal, succeeded); outcomes.put(deactivateTester, succeeded); outcomes.put(report, succeeded); @@ -178,16 +174,14 @@ public class JobRunnerTest { }; } - - private static StepRunner sleepy(StepRunner runner) { + private static StepRunner notifying(StepRunner runner, CountDownLatch latch) { return (step, id) -> { - try { - Thread.sleep(10); - } - catch (InterruptedException e) { - throw new AssertionError("Not supposed to happen."); + Status status = runner.run(step, id); + synchronized (latch) { + assertTrue(latch.getCount() > 0); + latch.countDown(); } - return runner.run(step, id); + return status; }; } -- cgit v1.2.3 From 45180ef9556ceea8df516dc239033f064128f46c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 2 Jul 2018 16:59:34 +0200 Subject: More thorough step and GC tests and some more wiring --- .../hosted/controller/ApplicationController.java | 2 + .../controller/deployment/DummyStepRunner.java | 12 ++ .../controller/deployment/InternalStepRunner.java | 231 +++++++++++++++++++++ .../controller/deployment/JobController.java | 24 ++- .../hosted/controller/deployment/StepRunner.java | 25 +++ .../maintenance/ControllerMaintenance.java | 1 + .../controller/maintenance/DummyStepRunner.java | 15 -- .../controller/maintenance/InternalStepRunner.java | 181 ---------------- .../hosted/controller/maintenance/JobRunner.java | 5 +- .../hosted/controller/maintenance/StepRunner.java | 25 --- .../controller/maintenance/JobRunnerTest.java | 63 +++++- 11 files changed, 349 insertions(+), 235 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c43249f01bd..7ae21e21f99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -140,6 +140,8 @@ public class ApplicationController { return sort(curator.readApplications(tenant)); } + public ArtifactRepository artifacts() { return artifactRepository; } + /** * Set the rotations marked as 'global' either 'in' or 'out of' service. * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java new file mode 100644 index 00000000000..17b523c60bf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java @@ -0,0 +1,12 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; + +public class DummyStepRunner implements StepRunner { + + @Override + public Step.Status run(LockedStep step, RunId id) { + return Step.Status.succeeded; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java new file mode 100644 index 00000000000..712a5421e7c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -0,0 +1,231 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; + +/** + * Runs steps of a deployment job against its provided controller. + * + * @author jonmv + */ +public class InternalStepRunner implements StepRunner { + + // TODO jvenstad: Move this tester logic to the application controller, perhaps? + public static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + id.instance().value() + "-t"); + } + + private final Controller controller; + + public InternalStepRunner(Controller controller) { + this.controller = controller; + } + + @Override + public Status run(LockedStep step, RunId id) { + switch (step.get()) { + case deployInitialReal: return deployInitialReal(id); + case installInitialReal: return installInitialReal(id); + case deployReal: return deployReal(id); + case deployTester: return deployTester(id); + case installReal: return installReal(id); + case installTester: return installTester(id); + case startTests: return startTests(id); + case endTests: return endTests(id); + case deactivateReal: return deactivateReal(id); + case deactivateTester: return deactivateTester(id); + case report: return report(id); + default: throw new AssertionError("Unknown step '" + step + "'!"); + } + } + + private Status deployInitialReal(RunId id) { + return deployReal(id, true); + } + + private Status deployReal(RunId id) { + return deployReal(id, false); + } + + private Status deployReal(RunId id, boolean setTheStage) { + return deploy(id.application(), + id.type(), + () -> controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, + Optional.empty(), + false, + setTheStage))); + } + + private Status deployTester(RunId id) { + Map> endpoints = deploymentEndpoints(id.application()); + if ( ! endpoints.containsKey(id.type().zone(controller.system()).get())) + return unfinished; + + return deploy(testerOf(id.application()), + id.type(), + () -> controller.applications().deployTester(testerOf(id.application()), + testerPackage(id, endpoints), + id.type().zone(controller.system()).get(), + new DeployOptions(true, + Optional.of(controller.systemVersion()), + false, + false))); + } + + private Status deploy(ApplicationId id, JobType type, Supplier deploy) { + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + ActivateResult result = deploy.get(); + + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() + || e.getErrorCode() == ACTIVATION_CONFLICT + || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { + + return unfinished; + } + } + return failed; + } + + private Status installInitialReal(RunId id) { + return install(id.application(), id.type()); + } + + private Status installReal(RunId id) { + return install(id.application(), id.type()); + } + + private Status installTester(RunId id) { + return install(testerOf(id.application()), id.type()); + } + + private Status install(ApplicationId id, JobType type) { + // If converged and serviceconverged: succeeded + // If timeout, failed + return unfinished; + } + + private Status startTests(RunId id) { + // Empty for now, but will be: find endpoints and post them. + throw new AssertionError(); + } + + private Status endTests(RunId id) { + // Update test logs. + // If tests are done, return test results. + throw new AssertionError(); + } + + private Status deactivateReal(RunId id) { + return deactivate(id.application(), id.type()); + } + + private Status deactivateTester(RunId id) { + return deactivate(testerOf(id.application()), id.type()); + } + + private Status deactivate(ApplicationId id, JobType type) { + // Try to deactivate, and if deactivated, finished. + throw new AssertionError(); + } + + private Status report(RunId id) { + // Easy squeezy. + throw new AssertionError(); + } + + private Application application(ApplicationId id) { + return controller.applications().require(id); + } + + private ApplicationPackage testerPackage(RunId id, Map> endpoints) { + ApplicationVersion version = application(id.application()).deploymentJobs() + .statusOf(id.type()).get() + .lastTriggered().get() + .application(); + + byte[] testConfig = testConfig(id.application(), id.type().zone(controller.system()).get(), controller.system(), endpoints); + byte[] testJar = controller.applications().artifacts().getTesterJar(testerOf(id.application()), version.id()); + byte[] servicesXml = servicesXml(); + + // TODO hakonhall: Assemble! + + throw new AssertionError(); + } + + private Map> deploymentEndpoints(ApplicationId id) { + ImmutableMap.Builder> deployments = ImmutableMap.builder(); + controller.applications().require(id).deployments().keySet() + .forEach(zone -> controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) + .ifPresent(endpoints -> deployments.put(zone, endpoints))); + return deployments.build(); + } + + private byte[] servicesXml() { + //TODO hakonhall: Create! + return "".getBytes(); + } + + /** Returns the config for the tests to run for the given job. */ + private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, Map> deployments) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("application", id.serializedForm()); + root.setString("zone", testerZone.value()); + root.setString("system", system.name()); + Cursor endpointsObject = root.setObject("endpoints"); + deployments.forEach((zone, endpoints) -> { + Cursor endpointArray = endpointsObject.setArray(zone.value()); + for (URI endpoint : endpoints) + endpointArray.addString(endpoint.toString()); + }); + try { + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index abcde079271..9366a678e88 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -28,7 +28,8 @@ import java.util.function.UnaryOperator; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; -import static com.yahoo.vespa.hosted.controller.maintenance.InternalStepRunner.testerOf; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.testerOf; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -61,7 +62,7 @@ public class JobController { for (ApplicationId id : applications()) for (JobType type : jobs(id)) { locked(id, type, runs -> { - last(id, type).ifPresent(last -> locked(last.id(), run -> run)); + curator.readLastRun(id, type).ifPresent(curator::writeLastRun); }); } } @@ -209,7 +210,7 @@ public class JobController { controller.applications().store(application.withBuiltInternally(false)); jobs(id).forEach(type -> { try (Lock __ = curator.lock(id, type)) { - last(id, type).ifPresent(last -> active(last.id()).ifPresent(active -> finish(active.id()))); + last(id, type).ifPresent(last -> active(last.id()).ifPresent(active -> abort(active.id()))); } }); }); @@ -223,10 +224,11 @@ public class JobController { .forEach(id -> { try { for (JobType type : jobs(id)) - try (Lock __ = curator.lock(id, type); - Lock ___ = curator.lock(id, type, Step.deactivateTester)) { - deactivateTester(id, type); - curator.deleteJobData(id, type); + try (Lock __ = curator.lock(id, type)) { + locked(id, type, deactivateTester, ___ -> { + deactivateTester(id, type); + curator.deleteJobData(id, type); + }); } } catch (TimeoutException e) { @@ -284,11 +286,11 @@ public class JobController { } } - /** Locks the given step, and checks none of its prerequisites are running, then performs the given actions. */ - public void locked(RunId id, Step step, Consumer action) throws TimeoutException { - try (Lock lock = curator.lock(id.application(), id.type(), step)) { + /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ + public void locked(ApplicationId id, JobType type, Step step, Consumer action) throws TimeoutException { + try (Lock lock = curator.lock(id, type, step)) { for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. - try (Lock __ = curator.lock(id.application(), id.type(), prerequisite)) { ; } + try (Lock __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java new file mode 100644 index 00000000000..cf024064cc4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java @@ -0,0 +1,25 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; + +/** + * Advances a given job run by running the appropriate {@link Step}s, based on their current status. + * + * When an attempt is made to advance a given job, a lock for that job (application and type) is + * taken, and released again only when the attempt finishes. Multiple other attempts may be made in + * the meantime, but they should give up unless the lock is promptly acquired. + * + * @author jonmv + */ +public interface StepRunner { + + /** Attempts to run the given locked step in the given run, and returns its new status. */ + Step.Status run(LockedStep step, RunId id); + +} + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 5313c03cb03..a18af6a9064 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepo import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.deployment.DummyStepRunner; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java deleted file mode 100644 index 94a86d713b8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; - -public class DummyStepRunner implements StepRunner { - - @Override - public Step.Status run(LockedStep step, RunId id) { - return Step.Status.succeeded; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java deleted file mode 100644 index cf20084a1d3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java +++ /dev/null @@ -1,181 +0,0 @@ -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; - -import java.util.Optional; -import java.util.function.Supplier; - -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; - -/** - * Runs steps of a deployment job against its provided controller. - * - * @author jonmv - */ -public class InternalStepRunner implements StepRunner { - - // TODO jvenstad: Move this tester logic to the application controller, perhaps? - public static ApplicationId testerOf(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value(), - id.instance().value() + "-t"); - } - - private final Controller controller; - - public InternalStepRunner(Controller controller) { - this.controller = controller; - } - - @Override - public Status run(LockedStep step, RunId id) { - switch (step.get()) { - case deployInitialReal: return deployInitialReal(id); - case installInitialReal: return installInitialReal(id); - case deployReal: return deployReal(id); - case deployTester: return deployTester(id); - case installReal: return installReal(id); - case installTester: return installTester(id); - case startTests: return startTests(id); - case endTests: return endTests(id); - case deactivateReal: return deactivateReal(id); - case deactivateTester: return deactivateTester(id); - case report: return report(id); - default: throw new AssertionError("Unknown step '" + step + "'!"); - } - } - - private Status deployInitialReal(RunId id) { - return deployReal(id, true); - } - - private Status deployReal(RunId id) { - // Separate out deploy logic from above, and reuse. - return deployReal(id, false); - } - - private Status deployReal(RunId id, boolean setTheStage) { - return deploy(id.application(), - id.type(), - () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), - Optional.empty(), - new DeployOptions(false, - Optional.empty(), - false, - setTheStage))); - } - - private Status deployTester(RunId id) { - // Find endpoints of real application. This will move down at a later time. - // See above. - return deploy(testerOf(id.application()), - id.type(), - () -> controller.applications().deployTester(testerOf(id.application()), - testerPackage(id), - id.type().zone(controller.system()).get(), - new DeployOptions(true, - Optional.of(controller.systemVersion()), - false, - false))); - } - - private Status deploy(ApplicationId id, JobType type, Supplier deploy) { - try { - // TODO jvenstad: Do whatever is required based on the result, and log all of this. - ActivateResult result = deploy.get(); - - return succeeded; - } - catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. - if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() - || e.getErrorCode() == ACTIVATION_CONFLICT - || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { - - return unfinished; - } - } - return failed; - } - - private Status installInitialReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installTester(RunId id) { - return install(testerOf(id.application()), id.type()); - } - - private Status install(ApplicationId id, JobType type) { - // If converged and serviceconverged: succeeded - // If timeout, failed - return unfinished; - } - - private Status startTests(RunId id) { - // Empty for now, but will be: find endpoints and post them. - throw new AssertionError(); - } - - private Status endTests(RunId id) { - // Update test logs. - // If tests are done, return test results. - throw new AssertionError(); - } - - private Status deactivateReal(RunId id) { - return deactivate(id.application(), id.type()); - } - - private Status deactivateTester(RunId id) { - return deactivate(testerOf(id.application()), id.type()); - } - - private Status deactivate(ApplicationId id, JobType type) { - // Try to deactivate, and if deactivated, finished. - throw new AssertionError(); - } - - private Status report(RunId id) { - // Easy squeezy. - throw new AssertionError(); - } - - private Application application(ApplicationId id) { - return controller.applications().require(id); - } - - private ApplicationPackage testerPackage(RunId id) { - ApplicationVersion version = application(id.application()).deploymentJobs() - .statusOf(id.type()).get() - .lastTriggered().get() - .application(); - - - // TODO hakonhall: Fetch! - throw new AssertionError(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 6649f705821..7dbf1a2c05e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import org.jetbrains.annotations.TestOnly; import java.time.Duration; @@ -74,8 +75,8 @@ public class JobRunner extends Maintainer { void advance(RunId id, Step step) { try { AtomicBoolean changed = new AtomicBoolean(false); - jobs.locked(id, step, lockedStep -> { - jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we bail out. + jobs.locked(id.application(), id.type(), step, lockedStep -> { + jobs.active(id).ifPresent(run -> { // The run may have become inactive, so we bail out. if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java deleted file mode 100644 index 400e9b4f74b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; - -/** - * Advances a given job run by running the appropriate {@link Step}s, based on their current status. - * - * When an attempt is made to advance a given job, a lock for that job (application and type) is - * taken, and released again only when the attempt finishes. Multiple other attempts may be made in - * the meantime, but they should give up unless the lock is promptly acquired. - * - * @author jonmv - */ -public interface StepRunner { - - /** Attempts to run the given locked step in the given run, and returns its new status. */ - Step.Status run(LockedStep step, RunId id); - -} - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 4d789ba13cb..49cfa129249 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -8,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import org.junit.Test; import java.time.Duration; @@ -16,13 +18,18 @@ import java.util.Collections; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; @@ -78,7 +85,7 @@ public class JobRunnerTest { latch.await(1, TimeUnit.SECONDS); assertEquals(0, latch.getCount()); - jobs.active().forEach(run -> jobs.active(run.id())); // Wait for locks of jobs which haven't yet been written through. + jobs.updateStorage(); // Holding the lock of each job ensures data read below is fresh. assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); @@ -162,6 +169,44 @@ public class JobRunnerTest { assertEquals(succeeded, run.get().steps().get(report)); } + @Test + public void stepLocking() throws InterruptedException, BrokenBarrierException { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + // Hang during tester deployment, until notified. + CyclicBarrier barrier = new CyclicBarrier(2); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + Executors.newFixedThreadPool(32), waitingRunner(barrier)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + + RunId runId = new RunId(id, systemTest, 1); + jobs.run(id, systemTest); + runner.maintain(); + barrier.await(); + try { + jobs.locked(id, systemTest, deactivateTester, step -> { }); + fail("deployTester step should still be locked!"); + } + catch (TimeoutException e) { } + + // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. + assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); + tester.controller().applications().deleteApplication(id, Optional.of(TestIdentities.userNToken)); + assertEquals(Collections.emptyList(), jobs.active()); + assertEquals(runId, jobs.last(id, systemTest).get().id()); + + // Deployment still ongoing, so garbage is not yet collected. + runner.maintain(); + assertEquals(runId, jobs.last(id, systemTest).get().id()); + + // Deployment lets go, deactivation may now run, and trash is thrown out. + barrier.await(); + runner.maintain(); + assertEquals(Optional.empty(), jobs.last(id, systemTest)); + } + private static ExecutorService inThreadExecutor() { return new AbstractExecutorService() { AtomicBoolean shutDown = new AtomicBoolean(false); @@ -189,4 +234,20 @@ public class JobRunnerTest { return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished); } + private static StepRunner waitingRunner(CyclicBarrier barrier) { + return (step, id) -> { + try { + if (step.get() == deployTester) { + barrier.await(); // Wake up the main thread, which waits for this step to be locked. + barrier.reset(); + barrier.await(); // Then wait while holding the lock for this step, until the main thread wakes us up. + } + } + catch (InterruptedException | BrokenBarrierException e) { + throw new AssertionError(e); + } + return succeeded; + }; + } + } -- cgit v1.2.3 From 98feb2edfde11e8964af1d7902a4d14f436fdb34 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Jul 2018 15:34:44 +0200 Subject: Store one object per deployment job step --- .../controller/api/integration/LogStore.java | 24 ++++-------- .../api/integration/stubs/MockLogStore.java | 41 ++++++--------------- .../controller/deployment/JobController.java | 43 ++++++++++++---------- .../hosted/controller/deployment/RunDetails.java | 27 +++++--------- 4 files changed, 53 insertions(+), 82 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java index 918047edca1..ef2d9077892 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -3,30 +3,20 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import java.util.Optional; + /** * @author freva */ public interface LogStore { - /** @return the test log of the given deployment job. */ - String getTestLog(RunId id); - - /** Stores the given test log for the given deployment job. */ - void setTestLog(RunId id, String testLog); - - /** @return the convergence log of the given deployment job. */ - String getConvergenceLog(RunId id); - - /** Stores the given convergence log for the given deployment job. */ - void setConvergenceLog(RunId id, String convergenceLog); - - /** @return the result of prepare of the test application for the given deployment job. */ - String getDeploymentLog(RunId id); + /** @return the log of the given step of the given deployment job, or an empty byte array if non-existent. */ + byte[] getLog(RunId id, String step); - /** Stores the given result of prepare of the test application for the given deployment job. */ - void setDeploymentLog(RunId id, String deploymentLog); + /** Stores the given log for the given step of the given deployment job. */ + void setLog(RunId id, String step, byte[] log); - /** Deletes all data associated with test of a given deployment job */ + /** Deletes all data associated with the given deployment job */ void deleteTestData(RunId id); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java index bc9f6247055..61e6ac1004b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java @@ -1,52 +1,35 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; -import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** - * @author freva + * @author jonmv */ public class MockLogStore implements LogStore { - @Override - public String getTestLog(RunId id) { - return "SUCCESS"; - } - - @Override - public void setTestLog(RunId id, String testLog) { - - } + private final Map> storage = new ConcurrentHashMap<>(); @Override - public String getConvergenceLog(RunId id) { - return "SUCCESS"; + public byte[] getLog(RunId id, String step) { + return storage.containsKey(id) && storage.get(id).containsKey(step) + ? storage.get(id).get(step) + : new byte[0]; } @Override - public void setConvergenceLog(RunId id, String convergenceLog) { - - } - - @Override - public String getDeploymentLog(RunId id) { - return "SUCCESS"; - } - - @Override - public void setDeploymentLog(RunId id, String deploymentLog) { - + public void setLog(RunId id, String step, byte[] log) { + storage.putIfAbsent(id, new ConcurrentHashMap<>()); + storage.get(id).put(step, log); } @Override public void deleteTestData(RunId id) { - + storage.remove(id); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 9366a678e88..056dcba2cd5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -16,6 +16,8 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import java.io.ByteArrayInputStream; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -67,29 +69,32 @@ public class JobController { } } - /** Returns the details currently logged for the given run. */ - public RunDetails details(RunId id) { - return new RunDetails(logs.getDeploymentLog(id), logs.getConvergenceLog(id), logs.getTestLog(id)); - } - - /** Appends the given string to the currently stored deployment logs for the given run. */ - public void logDeployment(RunId id, String appendage) { - try (Lock __ = curator.lock(id.application(), id.type())) { - logs.setDeploymentLog(id, logs.getDeploymentLog(id).concat(appendage)); - } - } - - /** Appends the given string to the currently stored convergence logs for the given run. */ - public void logConvergence(RunId id, String appendage) { - try (Lock __ = curator.lock(id.application(), id.type())) { - logs.setConvergenceLog(id, logs.getConvergenceLog(id).concat(appendage)); + /** Returns the details currently logged for the given run, if known. */ + public Optional details(RunId id) { + RunStatus run = runs(id.application(), id.type()).get(id); + if (run == null) + return Optional.empty(); + + Map details = new HashMap<>(); + for (Step step : run.steps().keySet()) { + byte[] log = logs.getLog(id, step.name()); + if (log.length > 0) + details.put(step, log); } + return Optional.of(new RunDetails(details)); } - /** Appends the given string to the currently stored test logs for the given run. */ - public void logTest(RunId id, String appendage) { + /** Appends the given log bytes to the currently stored bytes for the given run and step. */ + public void log(RunId id, Step step, byte[] log) { try (Lock __ = curator.lock(id.application(), id.type())) { - logs.setTestLog(id, logs.getTestLog(id).concat(appendage)); + byte[] stored = logs.getLog(id, step.name()); + if (stored.length > 0) { + byte[] addition = log; + log = new byte[stored.length + addition.length]; + System.arraycopy(stored, 0, log, 0, stored.length); + System.arraycopy(addition, 0, log, stored.length, addition.length); + } + logs.setLog(id, step.name(), log); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java index 98969bd4508..ebe2b920d0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java @@ -1,5 +1,10 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Optional; + /** * Contains details about a deployment job run. * @@ -7,26 +12,14 @@ package com.yahoo.vespa.hosted.controller.deployment; */ public class RunDetails { - private final String deploymentLog; - private final String convergenceLog; - private final String testLog; - - public RunDetails(String deploymentLog, String convergenceLog, String testLog) { - this.deploymentLog = deploymentLog; - this.convergenceLog = convergenceLog; - this.testLog = testLog; - } - - public String getDeploymentLog() { - return deploymentLog; - } + private final Map logs; - public String getConvergenceLog() { - return convergenceLog; + public RunDetails(Map logs) { + this.logs = ImmutableMap.copyOf(logs); } - public String getTestLog() { - return testLog; + public Optional get(Step step) { + return Optional.ofNullable(logs.get(step)); } } -- cgit v1.2.3 From 18df18885a485d337f828c1b585932b0a6041ed3 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 3 Jul 2018 16:39:19 +0200 Subject: Some stupid logger thing for StepRunner, and more wiring there --- .../controller/deployment/InternalStepRunner.java | 179 ++++++++++++++++++--- 1 file changed, 155 insertions(+), 24 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 712a5421e7c..8c90d27653a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -11,7 +11,9 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -19,28 +21,48 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import java.io.UncheckedIOException; import java.net.URI; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TimeZone; import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import static com.yahoo.log.LogLevel.DEBUG; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; /** * Runs steps of a deployment job against its provided controller. * + * A dual-purpose logger is set up for each thread that runs a step here: + * 1. All messages are logged to a buffer which is stored in an external log storage at the end of execution, and + * 2. all messages are also logged through the usual logging framework; thus, by default, any messages of level + * {@code Level.INFO} or higher end up in the Vespa log, and all messages may be sent there by means of log-control. + * * @author jonmv */ public class InternalStepRunner implements StepRunner { + static final Duration endpointTimeout = Duration.ofMinutes(15); + // TODO jvenstad: Move this tester logic to the application controller, perhaps? public static ApplicationId testerOf(ApplicationId id) { return ApplicationId.from(id.tenant().value(), @@ -49,6 +71,8 @@ public class InternalStepRunner implements StepRunner { } private final Controller controller; + // Wraps loggers which additionally write all records to byte arrays which are stored as the deployment job logs. + private final ThreadLocal logger = new ThreadLocal<>(); public InternalStepRunner(Controller controller) { this.controller = controller; @@ -56,27 +80,36 @@ public class InternalStepRunner implements StepRunner { @Override public Status run(LockedStep step, RunId id) { - switch (step.get()) { - case deployInitialReal: return deployInitialReal(id); - case installInitialReal: return installInitialReal(id); - case deployReal: return deployReal(id); - case deployTester: return deployTester(id); - case installReal: return installReal(id); - case installTester: return installTester(id); - case startTests: return startTests(id); - case endTests: return endTests(id); - case deactivateReal: return deactivateReal(id); - case deactivateTester: return deactivateTester(id); - case report: return report(id); - default: throw new AssertionError("Unknown step '" + step + "'!"); + try { + logger.set(ByteArrayLogger.of(id.application(), id.type(), step.get())); + switch (step.get()) { + case deployInitialReal: return deployInitialReal(id); + case installInitialReal: return installInitialReal(id); + case deployReal: return deployReal(id); + case deployTester: return deployTester(id); + case installReal: return installReal(id); + case installTester: return installTester(id); + case startTests: return startTests(id); + case endTests: return endTests(id); + case deactivateReal: return deactivateReal(id); + case deactivateTester: return deactivateTester(id); + case report: return report(id); + default: throw new AssertionError("Unknown step '" + step + "'!"); + } + } + finally { + controller.jobController().log(id, step.get(), logger.get().getLog()); + logger.remove(); } } private Status deployInitialReal(RunId id) { + logger.get().log(DEBUG, "Deploying the current version of " + id.application() + " ..."); return deployReal(id, true); } private Status deployReal(RunId id) { + logger.get().log(DEBUG, "Deploying the version to test of " + id.application() + " ..."); return deployReal(id, false); } @@ -84,7 +117,7 @@ public class InternalStepRunner implements StepRunner { return deploy(id.application(), id.type(), () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), + zone(id.type()), Optional.empty(), new DeployOptions(false, Optional.empty(), @@ -93,39 +126,89 @@ public class InternalStepRunner implements StepRunner { } private Status deployTester(RunId id) { + logger.get().log(DEBUG, "Attempting to find endpoints for " + id + " ..."); Map> endpoints = deploymentEndpoints(id.application()); - if ( ! endpoints.containsKey(id.type().zone(controller.system()).get())) + logger.get().log(DEBUG, "Found endpoints:\n" + + endpoints.entrySet().stream() + .map(zoneEndpoints -> "- " + zoneEndpoints.getKey() + ":\n" + + zoneEndpoints.getValue().stream() + .map(uri -> " |-- " + uri) + .collect(Collectors.joining("\n")))); + if ( ! endpoints.containsKey(zone(id.type()))) { + if (application(id.application()).deployments().get(zone(id.type())).at() + .isBefore(controller.clock().instant().minus(endpointTimeout))) { + logger.get().log(WARNING, "Endpoints for " + id.application() + " in " + zone(id.type()) + + " failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return failed; + } + + logger.get().log(DEBUG, "Endpoints for the deployment to test are not yet ready."); return unfinished; + } + logger.get().log(DEBUG, "Deploying the tester container for " + id.application() + " ..."); return deploy(testerOf(id.application()), id.type(), () -> controller.applications().deployTester(testerOf(id.application()), testerPackage(id, endpoints), - id.type().zone(controller.system()).get(), + zone(id.type()), new DeployOptions(true, Optional.of(controller.systemVersion()), false, false))); } - private Status deploy(ApplicationId id, JobType type, Supplier deploy) { + private Status deploy(ApplicationId id, JobType type, Supplier deployment) { try { // TODO jvenstad: Do whatever is required based on the result, and log all of this. - ActivateResult result = deploy.get(); + PrepareResponse prepareResponse = deployment.get().prepareResponse(); + if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { + logger.get().log(DEBUG, "Deploy failed due to non-compatible changes that require re-feed. " + + "Your options are: \n" + + "1. Revert the incompatible changes.\n" + + "2. If you think it is safe in your case, you can override this validation, see\n" + + " http://docs.vespa.ai/documentation/reference/validation-overrides.html\n" + + "3. Deploy as a new application under a different name.\n" + + "Illegal actions:\n" + + prepareResponse.configChangeActions.refeedActions.stream() + .filter(action -> ! action.allowed) + .flatMap(action -> action.messages.stream()) + .collect(Collectors.joining("\n")) + "\n" + + "Details:\n" + + prepareResponse.log.stream() + .map(entry -> entry.message) + .collect(Collectors.joining("\n"))); + return failed; + } + if (prepareResponse.configChangeActions.restartActions.isEmpty()) + logger.get().log(DEBUG, "No services requiring restart."); + else + prepareResponse.configChangeActions.restartActions.stream() + .flatMap(action -> action.services.stream()) + .map(service -> service.hostName) + .sorted().distinct() + .map(Hostname::new) + .forEach(hostname -> { + controller.applications().restart(new DeploymentId(id, zone(type)), Optional.of(hostname)); + logger.get().log(DEBUG, "Restarting services on host " + hostname.id() + "."); + }); + logger.get().log(DEBUG, "Deployment of " + id + " in " + zone(type) + " was successful!"); return succeeded; } catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() || e.getErrorCode() == ACTIVATION_CONFLICT || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { - + logger.get().log(DEBUG, "Exception of type '" + e.getErrorCode() + "' attempting to deploy:\n" + + e.getMessage() + "\n"); return unfinished; } + + logger.get().log(INFO, "Exception of type '" + e.getErrorCode() + "' attempting to deploy:\n" + + e.getMessage() + "\n"); + return failed; } - return failed; } private Status installInitialReal(RunId id) { @@ -179,13 +262,17 @@ public class InternalStepRunner implements StepRunner { return controller.applications().require(id); } + private ZoneId zone(JobType type) { + return type.zone(controller.system()).get(); + } + private ApplicationPackage testerPackage(RunId id, Map> endpoints) { ApplicationVersion version = application(id.application()).deploymentJobs() .statusOf(id.type()).get() .lastTriggered().get() .application(); - byte[] testConfig = testConfig(id.application(), id.type().zone(controller.system()).get(), controller.system(), endpoints); + byte[] testConfig = testConfig(id.application(), zone(id.type()), controller.system(), endpoints); byte[] testJar = controller.applications().artifacts().getTesterJar(testerOf(id.application()), version.id()); byte[] servicesXml = servicesXml(); @@ -196,7 +283,7 @@ public class InternalStepRunner implements StepRunner { private Map> deploymentEndpoints(ApplicationId id) { ImmutableMap.Builder> deployments = ImmutableMap.builder(); - controller.applications().require(id).deployments().keySet() + application(id).deployments().keySet() .forEach(zone -> controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) .ifPresent(endpoints -> deployments.put(zone, endpoints))); return deployments.build(); @@ -228,4 +315,48 @@ public class InternalStepRunner implements StepRunner { } } + + /** Logger which logs all records to a private byte array, as well as to its parent. */ + static class ByteArrayLogger extends Logger { + + private static final Logger parent = Logger.getLogger(InternalStepRunner.class.getName()); + private static final SimpleDateFormat timestampFormat = new SimpleDateFormat("[HH:mm:ss.SSS] "); + static { timestampFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } + + private final ByteArrayOutputStream bytes; + private final PrintStream out; + + private ByteArrayLogger(Logger parent, String suffix) { + super(parent.getName() + suffix, null); + setParent(parent); + + bytes = new ByteArrayOutputStream(); + out = new PrintStream(bytes); + } + + static ByteArrayLogger of(ApplicationId id, JobType type, Step step) { + return new ByteArrayLogger(parent, String.format(".%s.%s.%s", id.serializedForm(), type.jobName(), step)); + } + + @Override + public void log(LogRecord record) { + String timestamp = timestampFormat.format(new Date(record.getMillis())); + for (String line : record.getMessage().split("\n")) + out.println(timestamp + ": " + line); + + getParent().log(record); + } + + @Override + public boolean isLoggable(Level __) { + return true; + } + + public byte[] getLog() { + out.flush(); + return bytes.toByteArray(); + } + + } + } -- cgit v1.2.3 From 7b10c6fab21895ddd635c9a221f3da23b99a21bc Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 4 Jul 2018 11:45:23 +0200 Subject: Append, rather than set, logs, and fix unstable unit test --- .../controller/api/integration/LogStore.java | 6 +++--- .../api/integration/stubs/MockLogStore.java | 23 ++++++++++++---------- .../controller/deployment/JobController.java | 12 ++--------- .../controller/maintenance/JobRunnerTest.java | 4 ++-- 4 files changed, 20 insertions(+), 25 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java index ef2d9077892..23da48b4aad 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogStore.java @@ -11,12 +11,12 @@ import java.util.Optional; public interface LogStore { /** @return the log of the given step of the given deployment job, or an empty byte array if non-existent. */ - byte[] getLog(RunId id, String step); + byte[] get(RunId id, String step); /** Stores the given log for the given step of the given deployment job. */ - void setLog(RunId id, String step, byte[] log); + void append(RunId id, String step, byte[] log); /** Deletes all data associated with the given deployment job */ - void deleteTestData(RunId id); + void delete(RunId id); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java index 61e6ac1004b..330b967c1b5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockLogStore.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -12,24 +13,26 @@ import java.util.concurrent.ConcurrentHashMap; */ public class MockLogStore implements LogStore { - private final Map> storage = new ConcurrentHashMap<>(); + private final Map> logs = new ConcurrentHashMap<>(); @Override - public byte[] getLog(RunId id, String step) { - return storage.containsKey(id) && storage.get(id).containsKey(step) - ? storage.get(id).get(step) - : new byte[0]; + public byte[] get(RunId id, String step) { + return logs.getOrDefault(id, Collections.emptyMap()).getOrDefault(step, new byte[0]); } @Override - public void setLog(RunId id, String step, byte[] log) { - storage.putIfAbsent(id, new ConcurrentHashMap<>()); - storage.get(id).put(step, log); + public void append(RunId id, String step, byte[] log) { + logs.putIfAbsent(id, new ConcurrentHashMap<>()); + byte[] old = get(id, step); + byte[] union = new byte[old.length + log.length]; + System.arraycopy(old, 0, union, 0, old.length); + System.arraycopy(log, 0, union, old.length, log.length); + logs.get(id).put(step, union); } @Override - public void deleteTestData(RunId id) { - storage.remove(id); + public void delete(RunId id) { + logs.remove(id); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 056dcba2cd5..29895066525 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import java.io.ByteArrayInputStream; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -77,7 +76,7 @@ public class JobController { Map details = new HashMap<>(); for (Step step : run.steps().keySet()) { - byte[] log = logs.getLog(id, step.name()); + byte[] log = logs.get(id, step.name()); if (log.length > 0) details.put(step, log); } @@ -87,14 +86,7 @@ public class JobController { /** Appends the given log bytes to the currently stored bytes for the given run and step. */ public void log(RunId id, Step step, byte[] log) { try (Lock __ = curator.lock(id.application(), id.type())) { - byte[] stored = logs.getLog(id, step.name()); - if (stored.length > 0) { - byte[] addition = log; - log = new byte[stored.length + addition.length]; - System.arraycopy(stored, 0, log, 0, stored.length); - System.arraycopy(addition, 0, log, stored.length, addition.length); - } - logs.setLog(id, step.name(), log); + logs.append(id, step.name(), log); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 49cfa129249..a1436b61203 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -85,7 +85,7 @@ public class JobRunnerTest { latch.await(1, TimeUnit.SECONDS); assertEquals(0, latch.getCount()); - jobs.updateStorage(); // Holding the lock of each job ensures data read below is fresh. + runner.deconstruct(); // Ensures all workers have finished writing to the curator. assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); @@ -170,7 +170,7 @@ public class JobRunnerTest { } @Test - public void stepLocking() throws InterruptedException, BrokenBarrierException { + public void locksAndGarbage() throws InterruptedException, BrokenBarrierException { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); // Hang during tester deployment, until notified. -- cgit v1.2.3