diff options
13 files changed, 232 insertions, 464 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 8093c4d7a18..1ad89cb7a72 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -147,12 +147,17 @@ public class JobController { /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. - RunStatus endedRun = run.finish(controller.clock().instant()); - locked(id.application(), id.type(), runs -> runs.put(run.id(), endedRun)); - return endedRun; + RunStatus finishedRun = run.finished(controller.clock().instant()); + locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun)); + return finishedRun; }); } + /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */ + public void abort(RunId id) { + locked(id, run -> run.aborted()); + } + /** Registers the given application, such that it may have deployment jobs run here. */ void register(ApplicationId id) { controller.applications().lockIfPresent(id, application -> @@ -164,8 +169,7 @@ public class JobController { byte[] applicationPackage, byte[] applicationTestJar) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) - throw new IllegalArgumentException(id + " is not built here!"); + controller.applications().store(application.withBuiltInternally(true)); long run = nextBuild(id); version.set(ApplicationVersion.from(revision, run)); @@ -177,7 +181,7 @@ public class JobController { return version.get(); } - /** Orders a run of the given type, and returns the id of the created job. */ + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void run(ApplicationId id, JobType type) { controller.applications().lockIfPresent(id, application -> { if ( ! application.get().deploymentJobs().builtInternally()) @@ -201,6 +205,7 @@ public class JobController { for (JobType type : jobs(id)) try (Lock __ = curator.lock(id, type)) { curator.deleteJobData(id, type); + // TODO jvenstad: Deactivate tester applications? } }); } @@ -218,7 +223,7 @@ public class JobController { private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, JobType.component, - 0, + Long.MAX_VALUE, // TODO jvenstad: Clean up this! number, Optional.of(revision), Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 82464820d9b..c36e0b3a39f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -20,9 +20,10 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(storeData, + EnumSet.of(runTests, deactivateTester, - deactivateReal)), + deactivateReal, + report)), stagingTest(EnumSet.of(deployInitialReal, installInitialReal, @@ -31,17 +32,19 @@ public enum JobProfile { deployTester, installTester, startTests), - EnumSet.of(storeData, + EnumSet.of(runTests, deactivateTester, - deactivateReal)), + deactivateReal, + report)), production(EnumSet.of(deployReal, installReal, deployTester, installTester, startTests), - EnumSet.of(storeData, - deactivateTester)); + EnumSet.of(runTests, + deactivateTester, + report)); private final Set<Step> steps; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 2ecac588d54..a91cc905add 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -26,36 +26,45 @@ public class RunStatus { private final Map<Step, Step.Status> steps; private final Instant start; private final Optional<Instant> end; + private final boolean aborted; // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs. // For deserialisation only -- do not use! - public RunStatus(RunId id, Map<Step, Step.Status> steps, Instant start, Optional<Instant> end) { + public RunStatus(RunId id, Map<Step, Step.Status> steps, Instant start, Optional<Instant> end, boolean aborted) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.start = start; this.end = end; + this.aborted = aborted; } public static RunStatus initial(RunId id, Instant now) { EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); - return new RunStatus(id, steps, requireNonNull(now), Optional.empty()); + return new RunStatus(id, steps, requireNonNull(now), Optional.empty(), false); } public RunStatus with(Step.Status status, LockedStep step) { - if (end.isPresent()) + if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be further modified!"); EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); steps.put(step.get(), requireNonNull(status)); - return new RunStatus(id, steps, start, end); + return new RunStatus(id, steps, start, end, aborted); } - public RunStatus finish(Instant now) { - if (end.isPresent()) + public RunStatus finished(Instant now) { + if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); - return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now)); + return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now), aborted); + } + + public RunStatus aborted() { + if (hasEnded()) + throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!"); + + return new RunStatus(id, new EnumMap<>(steps), start, end, true); } /** Returns the id of this run. */ @@ -86,7 +95,12 @@ public class RunStatus { /** Returns whether the run has failed, and should switch to its run-always steps. */ public boolean hasFailed() { - return steps.values().contains(failed); + return aborted || steps.values().contains(failed); + } + + /** Returns whether the run has been forcefully aborted. */ + public boolean isAborted() { + return aborted; } /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ @@ -115,6 +129,7 @@ public class RunStatus { "id=" + id + ", start=" + start + ", end=" + end + + ", aborted=" + aborted + ", steps=" + steps + '}'; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index a1d07001331..2048c5ab353 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; /** @@ -19,6 +20,7 @@ import java.util.List; * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant * information from a failed run is stored, and that deployment does not occur after deactivation. * + * @see JobController * @author jonmv */ public enum Step { @@ -44,17 +46,17 @@ public enum Step { /** Ask the tester to run its tests. */ startTests(installReal, installTester), - /** Download data from the tester and store it. */ - storeData(startTests), + /** See that the tests are done running and store the test results. */ + runTests(startTests), /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, startTests), + deactivateReal(deployInitialReal, deployReal, runTests), /** Deactivate the tester. */ - deactivateTester(deployTester, storeData), + deactivateTester(deployTester, runTests), /** Report completion to deployment orchestration machinery. */ - report; + report(deactivateReal, deactivateTester); private final List<Step> prerequisites; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java index f6f418e70be..94a86d713b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DummyStepRunner.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -7,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; public class DummyStepRunner implements StepRunner { @Override - public Step.Status run(LockedStep step, RunStatus run) { + public Step.Status run(LockedStep step, RunId id) { return Step.Status.succeeded; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index eacf9c72992..1434c40bdee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -15,11 +15,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}. * + * @see JobController * @author jonmv */ public class JobRunner extends Maintainer { @@ -70,18 +72,21 @@ public class JobRunner extends Maintainer { /** Attempts to advance the status of the given step, for the given run. */ void advance(RunId id, Step step) { try { + AtomicBoolean changed = new AtomicBoolean(false); jobs.locked(id, step, lockedStep -> { jobs.active(id).ifPresent(run -> { // The run may have become inactive, which means we bail out. if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. - Step.Status status = runner.run(lockedStep, run); + Step.Status status = runner.run(lockedStep, run.id()); if (run.steps().get(step) != status) { jobs.update(run.id(), status, lockedStep); - advance(run); + changed.set(true); } }); }); + if (changed.get()) + jobs.active(id).ifPresent(this::advance); } catch (TimeoutException e) { // Something else is already advancing this step, or a prerequisite -- try again later! diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java deleted file mode 100644 index 14402e389df..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ /dev/null @@ -1,183 +0,0 @@ -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; - -import java.util.Optional; -import java.util.function.Supplier; - -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; -import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; - -/** - * Runs steps of a deployment job against its provided controller. - * - * @author jonmv - */ -public class RealStepRunner implements StepRunner { - - private static ApplicationId testerOf(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value(), - id.instance().value() + "-t"); - } - - private final Controller controller; - - public RealStepRunner(Controller controller) { - this.controller = controller; - } - - @Override - public Status run(LockedStep step, RunStatus run) { - RunId id = run.id(); - switch (step.get()) { - case deployInitialReal: return deployInitialReal(id); - case installInitialReal: return installInitialReal(id); - case deployReal: return deployReal(id); - case deployTester: return deployTester(id); - case installReal: return installReal(id); - case installTester: return installTester(id); - case startTests: return startTests(id); - case storeData: return storeData(id); - case deactivateReal: return deactivateReal(id); - case deactivateTester: return deactivateTester(id); - case report: return report(id); - default: throw new AssertionError("Unknown step '" + step + "'!"); - } - } - - private Status deployInitialReal(RunId id) { - return deployReal(id, true); - } - - private Status deployReal(RunId id) { - // Separate out deploy logic from above, and reuse. - return deployReal(id, false); - } - - private Status deployReal(RunId id, boolean setTheStage) { - return deploy(id.application(), - id.type(), - () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()).get(), - Optional.empty(), - new DeployOptions(false, - Optional.empty(), - false, - setTheStage))); - } - - private Status deployTester(RunId id) { - // Find endpoints of real application. This will move down at a later time. - // See above. - return deploy(testerOf(id.application()), - id.type(), - () -> controller.applications().deployTester(testerOf(id.application()), - testerPackage(id), - id.type().zone(controller.system()).get(), - new DeployOptions(true, - Optional.of(controller.systemVersion()), - false, - false))); - } - - private Status deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deploy) { - try { - // TODO jvenstad: Do whatever is required based on the result, and log all of this. - ActivateResult result = deploy.get(); - - return succeeded; - } - catch (ConfigServerException e) { - // TODO jvenstad: Consider retrying different things as well. - // TODO jvenstad: Log error information. - if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() - || e.getErrorCode() == ACTIVATION_CONFLICT - || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { - - return unfinished; - } - } - return failed; - } - - private Status installInitialReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installReal(RunId id) { - return install(id.application(), id.type()); - } - - private Status installTester(RunId id) { - return install(testerOf(id.application()), id.type()); - } - - private Status install(ApplicationId id, JobType type) { - // If converged and serviceconverged: succeeded - // If timeout, failed - return unfinished; - } - - private Status startTests(RunId id) { - // Empty for now, but will be: find endpoints and post them. - throw new AssertionError(); - } - - private Status storeData(RunId id) { - // Update test logs. - // If tests are done, return test results. - throw new AssertionError(); - } - - private Status deactivateReal(RunId id) { - return deactivate(id.application(), id.type()); - } - - private Status deactivateTester(RunId id) { - return deactivate(testerOf(id.application()), id.type()); - } - - private Status deactivate(ApplicationId id, JobType type) { - // Try to deactivate, and if deactivated, finished. - throw new AssertionError(); - } - - private Status report(RunId id) { - // Easy squeezy. - throw new AssertionError(); - } - - private Application application(ApplicationId id) { - return controller.applications().require(id); - } - - private ApplicationPackage testerPackage(RunId id) { - ApplicationVersion version = application(id.application()).deploymentJobs() - .statusOf(id.type()).get() - .lastTriggered().get() - .application(); - - - // TODO hakonhall: Fetch! - throw new AssertionError(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java index ebb4c91edcc..400e9b4f74b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/StepRunner.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -17,8 +18,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; */ public interface StepRunner { - /** Attempts to run the given locked step, and returns its new status. */ - Step.Status run(LockedStep step, RunStatus run); + /** Attempts to run the given locked step in the given run, and returns its new status. */ + Step.Status run(LockedStep step, RunId id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 15ddcdace5b..dcbc9479845 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -69,7 +69,7 @@ public class CuratorDb { private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); - private final JobSerializer jobSerializer = new JobSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); private final Curator curator; @@ -308,25 +308,25 @@ public class CuratorDb { // -------------- Job Runs ------------------------------------------------ public void writeLastRun(RunStatus run) { - curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(jobSerializer.toSlime(run))); + curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); } public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<RunStatus> runs) { - curator.set(jobPath(id, type), asJson(jobSerializer.toSlime(runs))); - } - - public void deleteJobData(ApplicationId id, JobType type) { - curator.delete(jobPath(id, type)); - curator.delete(lastRunPath(id, type)); + curator.set(jobPath(id, type), asJson(runSerializer.toSlime(runs))); } public Optional<RunStatus> readLastRun(ApplicationId id, JobType type) { - return readSlime(jobPath(id, type)).map(jobSerializer::runFromSlime); + return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime); } public Map<RunId, RunStatus> readHistoricRuns(ApplicationId id, JobType type) { // TODO jvenstad: Add, somewhere, a retention filter based on age or count. - return readSlime(jobPath(id, type)).map(jobSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); + return readSlime(jobPath(id, type)).map(runSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); + } + + public void deleteJobData(ApplicationId id, JobType type) { + curator.delete(jobPath(id, type)); + curator.delete(lastRunPath(id, type)); } // -------------- Provisioning (called by internal code) ------------------ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java deleted file mode 100644 index 503f6fd990e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.deployment.Step.Status; - -import java.time.Instant; -import java.util.EnumMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; - -public class JobSerializer { - - private static final String stepsField = "steps"; - private static final String applicationField = "id"; - private static final String jobTypeField = "type"; - private static final String numberField = "number"; - private static final String startField = "start"; - private static final String endField = "end"; - - public RunStatus runFromSlime(Slime slime) { - return runFromSlime(slime.get()); - } - - public Map<RunId, RunStatus> runsFromSlime(Slime slime) { - Map<RunId, RunStatus> runs = new LinkedHashMap<>(); - Inspector runArray = slime.get(); - runArray.traverse((ArrayTraverser) (__, runObject) -> { - RunStatus run = runFromSlime(runObject); - runs.put(run.id(), run); - }); - - return runs; - } - - private RunStatus runFromSlime(Inspector runObject) { - EnumMap<Step, Status> steps = new EnumMap<>(Step.class); - runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(stepOf(step), statusOf(status.asString())); - }); - return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()), - steps, - Instant.ofEpochMilli(runObject.field(startField).asLong()), - Optional.of(runObject.field(endField)) - .filter(Inspector::valid) - .map(end -> Instant.ofEpochMilli(end.asLong()))); - } - - public Slime toSlime(RunStatus run) { - Slime slime = new Slime(); - toSlime(run, slime.setObject()); - return slime; - } - - public Slime toSlime(Iterable<RunStatus> runs) { - Slime slime = new Slime(); - Cursor runArray = slime.setArray(); - runs.forEach(run -> toSlime(run, runArray.addObject())); - return slime; - } - - private void toSlime(RunStatus run, Cursor runObject) { - runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().jobName()); - runObject.setLong(numberField, run.id().number()); - runObject.setLong(startField, run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); - Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); - } - - static String valueOf(Step step) { - switch (step) { - case deployInitialReal : return "DIR"; - case installInitialReal : return "IIR"; - case deployReal : return "DR" ; - case installReal : return "IR" ; - case deactivateReal : return "DAR"; - case deployTester : return "DT" ; - case installTester : return "IT" ; - case deactivateTester : return "DAT"; - case startTests : return "ST" ; - case storeData : return "SD" ; - case report : return "R" ; - default : throw new AssertionError("No value defined for '" + step + "'!"); - } - } - - static Step stepOf(String step) { - switch (step) { - case "DIR" : return deployInitialReal ; - case "IIR" : return installInitialReal; - case "DR" : return deployReal ; - case "IR" : return installReal ; - case "DAR" : return deactivateReal ; - case "DT" : return deployTester ; - case "IT" : return installTester ; - case "DAT" : return deactivateTester ; - case "ST" : return startTests ; - case "SD" : return storeData ; - case "R" : return report ; - default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); - } - } - - static String valueOf(Status status) { - switch (status) { - case unfinished : return "U"; - case failed : return "F"; - case succeeded : return "S"; - default : throw new AssertionError("No value defined for '" + status + "'!"); - } - } - - static Status statusOf(String status) { - switch (status) { - case "U" : return unfinished; - case "F" : return failed ; - case "S" : return succeeded ; - default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); - } - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 2a2a6a5e1b9..67c1854e15a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,15 +1,46 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import org.junit.Test; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.List; +import java.util.Map; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author jonmv @@ -17,11 +48,116 @@ import java.util.concurrent.atomic.AtomicBoolean; public class JobRunnerTest { @Test - public void test() { - ControllerTester tester = new ControllerTester(); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()), - inThreadExecutor(), new DummyStepRunner()); + public void testMultiThreadedExecutionFinishes() throws InterruptedException { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + Executors.newFixedThreadPool(32), sleepy(new DummyStepRunner())); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + + jobs.run(id, systemTest); + try { + jobs.run(id, systemTest); + fail("Job is already running, so this should not be allowed!"); + } + catch (IllegalStateException e) { } + jobs.run(id, stagingTest); + + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); runner.maintain(); + assertFalse(jobs.last(id, systemTest).get().hasEnded()); + Thread.sleep(1000); // I'm so sorry, but I want to test this. Takes ~100ms "on my machine". + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); + } + + @Test + public void testStepLogic() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + Map<Step, Status> outcomes = new EnumMap<>(Step.class); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + inThreadExecutor(), mappedRunner(outcomes)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get(); + + jobs.run(id, systemTest); + RunId first = run.get().id(); + + // Unfinished steps change nothing. + Map<Step, Status> steps = run.get().steps(); + runner.maintain(); + assertEquals(steps, run.get().steps()); + assertEquals(Arrays.asList(deployReal), run.get().readySteps()); + + // Deployment allows installation. + outcomes.put(deployReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installReal), run.get().readySteps()); + + // Installation allows tester deployment. + outcomes.put(installReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deployTester), run.get().readySteps()); + + // Tester deployment allows tester installation. + outcomes.put(deployTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installTester), run.get().readySteps()); + + // Tester installation allows starting tests. + outcomes.put(installTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(startTests), run.get().readySteps()); + + // Starting tests allows storing data. + outcomes.put(startTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(runTests), run.get().readySteps()); + + // Storing data allows deactivating tester. + outcomes.put(runTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Failure deactivating real fails the run, but run-always steps continue. + outcomes.put(deactivateReal, failed); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Aborting the run now does nothing, as only run-always steps are left. + jobs.abort(run.get().id()); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Success of the remaining run-always steps ends the run. + outcomes.put(deactivateReal, succeeded); + outcomes.put(deactivateTester, succeeded); + outcomes.put(report, succeeded); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().isAborted()); + + // A new run is attempted. + jobs.run(id, systemTest); + assertEquals(first.number() + 1, run.get().id().number()); + + // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. + outcomes.put(deployTester, failed); + runner.maintain(); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().hasFailed()); + assertFalse(run.get().isAborted()); + assertEquals(failed, run.get().steps().get(deployTester)); + assertEquals(unfinished, run.get().steps().get(installTester)); + assertEquals(succeeded, run.get().steps().get(report)); } private static ExecutorService inThreadExecutor() { @@ -36,4 +172,21 @@ public class JobRunnerTest { }; } + + private static StepRunner sleepy(StepRunner runner) { + return (step, id) -> { + try { + Thread.sleep(10); + } + catch (InterruptedException e) { + throw new AssertionError("Not supposed to happen."); + } + return runner.run(step, id); + }; + } + + private static StepRunner mappedRunner(Map<Step, Status> outcomes) { + return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java deleted file mode 100644 index e6e726bec5e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializerTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.yahoo.vespa.hosted.controller.persistence; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; -import org.junit.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.Collections; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class JobSerializerTest { - - private static final JobSerializer serializer = new JobSerializer(); - private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); - private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), - JobType.productionUsEast3, - (long) 112358); - private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); - - @Test - public void testSerialization() throws IOException { - for (Step step : Step.values()) - assertEquals(step, JobSerializer.stepOf(JobSerializer.valueOf(step))); - - for (Step.Status status : Step.Status.values()) - assertEquals(status, JobSerializer.statusOf(JobSerializer.valueOf(status))); - - // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! - RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); - for (Step step : Step.values()) - assertTrue(run.steps().containsKey(step)); - - assertEquals(id, run.id()); - assertEquals(start, run.start()); - assertFalse(run.hasEnded()); - assertEquals(ImmutableMap.<Step, Step.Status>builder() - .put(deployInitialReal, unfinished) - .put(installInitialReal, failed) - .put(deployReal, succeeded) - .put(installReal, unfinished) - .put(deactivateReal, failed) - .put(deployTester, succeeded) - .put(installTester, unfinished) - .put(deactivateTester, failed) - .put(startTests, succeeded) - .put(storeData, unfinished) - .put(report, failed) - .build(), - run.steps()); - - RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); - assertEquals(run.id(), phoenix.id()); - assertEquals(run.start(), phoenix.start()); - assertEquals(run.end(), phoenix.end()); - assertEquals(run.steps(), phoenix.steps()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 4db94eb429a..e31d14ac181 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -14,7 +14,7 @@ "IT": "U", "DAT": "F", "ST": "S", - "SD": "U", + "RT": "U", "R": "F" } } |