diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2019-12-20 10:31:43 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-12-20 10:31:43 +0100 |
commit | d1ef9ba58add96123213fd8fe65a6dcc6ffdb5a6 (patch) | |
tree | fa954f7e9db7edae5aef50c4aaef1aa611a775da /controller-server | |
parent | e1016d0a2cb537b53662ec6cc3fc851175ad4572 (diff) | |
parent | d3334d6f6b72bba9e58a0a12bf9a8aa8a8ddd069 (diff) |
Merge pull request #11581 from vespa-engine/hakonhall/add-start-timestamp-for-step-try-2
Add start timestamp for step, try 2
Diffstat (limited to 'controller-server')
14 files changed, 279 insertions, 109 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 81b33557044..8a6c2fa88f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; @@ -33,7 +31,6 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -48,7 +45,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -148,7 +144,7 @@ public class JobController { /** Fetches any new Vespa log entries, and records the timestamp of the last of these, for continuation. */ public void updateVespaLog(RunId id) { locked(id, run -> { - if ( ! run.steps().containsKey(copyVespaLogs)) + if ( ! run.hasStep(copyVespaLogs)) return run; ZoneId zone = id.type().zone(controller.system()); @@ -311,6 +307,11 @@ public class JobController { locked(id, run -> run.with(status, step)); } + /** Invoked when starting the step */ + public void setStartTimestamp(RunId id, Instant timestamp, LockedStep step) { + locked(id, run -> run.with(timestamp, step)); + } + /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 7bccbc0ebc9..0ebdc2aa1d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -7,7 +7,18 @@ import java.util.Collections; import java.util.EnumSet; import java.util.Set; -import static com.yahoo.vespa.hosted.controller.deployment.Step.*; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; /** * Static profiles defining the {@link Step}s of a deployment job. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 2f9c5ea9e08..00021c4765a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -11,11 +11,11 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; @@ -28,7 +28,7 @@ import static java.util.Objects.requireNonNull; public class Run { private final RunId id; - private final Map<Step, Step.Status> steps; + private final Map<Step, StepInfo> steps; private final Versions versions; private final Instant start; private final Optional<Instant> end; @@ -38,7 +38,7 @@ public class Run { private final Optional<X509Certificate> testerCertificate; // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, Step.Status> steps, Versions versions, Instant start, + public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<X509Certificate> testerCertificate) { this.id = id; @@ -53,52 +53,67 @@ public class Run { } public static Run initial(RunId id, Versions versions, Instant now) { - EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); - JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); + EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); + JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty()); } - /** Returns a new Run with the new status, and with the status of the given, completed step set accordingly. */ + /** Returns a new Run with the status of the given completed step set accordingly. */ public Run with(RunStatus status, LockedStep step) { requireActive(); - if (steps.get(step.get()) != unfinished) + StepInfo stepInfo = getRequiredStepInfo(step.get()); + if (stepInfo.status() != unfinished) throw new IllegalStateException("Step '" + step.get() + "' can't be set to '" + status + "'" + - " -- it already completed with status '" + steps.get(step.get()) + "'!"); + " -- it already completed with status '" + stepInfo.status() + "'!"); - EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); - steps.put(step.get(), Step.Status.of(status)); + EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); + steps.put(step.get(), stepInfo.with(Step.Status.of(status))); return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); } + /** Returns a new Run with a new start time*/ + public Run with(Instant startTime, LockedStep step) { + requireActive(); + StepInfo stepInfo = getRequiredStepInfo(step.get()); + if (stepInfo.status() != unfinished) + throw new IllegalStateException("Unable to set start timestamp of step " + step.get() + + ": it has already completed with status " + stepInfo.status() + "!"); + + EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); + steps.put(step.get(), stepInfo.with(startTime)); + + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); + } + public Run finished(Instant now) { requireActive(); - return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status, + return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); } public Run aborted() { requireActive(); - return new Run(id, new EnumMap<>(steps), versions, start, end, aborted, + return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, testerCertificate); } public Run with(long lastTestRecord) { requireActive(); - return new Run(id, new EnumMap<>(steps), versions, start, end, status, + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, new EnumMap<>(steps), versions, start, end, status, + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, new EnumMap<>(steps), versions, start, end, status, + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, Optional.of(testerCertificate)); } @@ -107,11 +122,35 @@ public class Run { return id; } - /** Returns an unmodifiable view of the status of all steps in this run. */ - public Map<Step, Step.Status> steps() { + /** Whether this run contains this step. */ + public boolean hasStep(Step step) { + return steps.containsKey(step); + } + + /** Returns info on step. */ + public Optional<StepInfo> stepInfo(Step step) { + return Optional.ofNullable(steps.get(step)); + } + + private StepInfo getRequiredStepInfo(Step step) { + return stepInfo(step).orElseThrow(() -> new IllegalArgumentException("There is no such step " + step + " for run " + id)); + } + + /** Returns status of step. */ + public Optional<Step.Status> stepStatus(Step step) { + return stepInfo(step).map(StepInfo::status); + } + + /** Returns an unmodifiable view of all step information in this run. */ + public Map<Step, StepInfo> steps() { return steps; } + /** Returns an unmodifiable view of the status of all steps in this run. */ + public Map<Step, Step.Status> stepStatuses() { + return Collections.unmodifiableMap(steps.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().status()))); + } + public RunStatus status() { return status; } @@ -190,10 +229,10 @@ public class Run { /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ private List<Step> normalSteps() { return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() == unfinished + .filter(entry -> entry.getValue().status() == unfinished && entry.getKey().prerequisites().stream() .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + || steps.get(step).status() == succeeded)) .map(Map.Entry::getKey) .iterator()); } @@ -201,11 +240,12 @@ public class Run { /** Returns the list of not-yet-run run-always steps whose run-always prerequisites have all run. */ private List<Step> forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() == unfinished + .filter(entry -> entry.getValue().status() == unfinished && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) && entry.getKey().prerequisites().stream() .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) != unfinished)) + .allMatch(step -> steps.get(step) == null + || steps.get(step).status() != unfinished)) .map(Map.Entry::getKey) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 8870425c82b..fc0b99ef9aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -68,6 +68,7 @@ public enum Step { this.prerequisites = ImmutableList.copyOf(prerequisites); } + /** Returns the prerequisite steps that must be successfully completed before this, assuming the job contains these steps. */ public List<Step> prerequisites() { return prerequisites; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java new file mode 100644 index 00000000000..d2b6dacd426 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepInfo.java @@ -0,0 +1,60 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * Information about a step. + * + * @author hakonhall + */ +// @Immutable +public class StepInfo { + private final Step step; + private final Step.Status status; + private final Optional<Instant> startTime; + + public static StepInfo initial(Step step) { return new StepInfo(step, Step.Status.unfinished, Optional.empty()); } + + public StepInfo(Step step, Step.Status status, Optional<Instant> startTime) { + this.step = step; + this.status = status; + this.startTime = startTime; + } + + public Step step() { return step; } + public Step.Status status() { return status; } + public Optional<Instant> startTime() { return startTime; } + + /** Returns a copy of this, but with the given status. */ + public StepInfo with(Step.Status status) { return new StepInfo(step, status, startTime); } + + /** Returns a copy of this, but with the given start timestamp. */ + public StepInfo with(Instant startTimestamp) { return new StepInfo(step, status, Optional.of(startTimestamp)); } + + @Override + public String toString() { + return "StepInfo{" + + "step=" + step + + ", status=" + status + + ", startTime=" + startTime + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StepInfo stepInfo = (StepInfo) o; + return step == stepInfo.step && + status == stepInfo.status && + Objects.equals(startTime, stepInfo.startTime); + } + + @Override + public int hashCode() { + return Objects.hash(step, status, startTime); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 5d0567b7eac..4c61b341c20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -8,10 +8,12 @@ import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import org.jetbrains.annotations.TestOnly; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -98,6 +100,11 @@ public class JobRunner extends Maintainer { if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. + StepInfo stepInfo = run.stepInfo(lockedStep.get()).orElseThrow(); + if (stepInfo.startTime().isEmpty()) { + jobs.setStartTimestamp(run.id(), Instant.now(), lockedStep); + } + runner.run(lockedStep, run.id()).ifPresent(status -> { jobs.update(run.id(), status, lockedStep); changed.set(true); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index b84df02e583..b777d6f52ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -9,14 +9,15 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import com.yahoo.vespa.hosted.controller.deployment.Versions; import java.time.Instant; @@ -24,16 +25,15 @@ import java.time.temporal.ChronoUnit; import java.util.EnumMap; import java.util.NavigableMap; import java.util.Optional; -import java.util.SortedMap; import java.util.TreeMap; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -44,12 +44,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static java.util.Comparator.comparing; /** @@ -66,7 +66,10 @@ class RunSerializer { // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // TODO: Remove "steps" when there are no traces of it in the controllers private static final String stepsField = "steps"; + private static final String stepDetailsField = "stepDetails"; + private static final String startTimeField = "startTime"; private static final String applicationField = "id"; private static final String jobTypeField = "type"; private static final String numberField = "number"; @@ -102,9 +105,19 @@ class RunSerializer { } private Run runFromSlime(Inspector runObject) { - EnumMap<Step, Status> steps = new EnumMap<>(Step.class); + var steps = new EnumMap<Step, StepInfo>(Step.class); + Inspector detailsField = runObject.field(stepDetailsField); runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(stepOf(step), stepStatusOf(status.asString())); + Step typedStep = stepOf(step); + + // For historical reasons are the step details stored in a separate JSON structure from the step statuses. + Inspector stepDetailsField = detailsField.field(step); + Inspector startTimeValue = stepDetailsField.field(startTimeField); + Optional<Instant> startTime = startTimeValue.valid() ? + Optional.of(instantOf(startTimeValue.asLong())) : + Optional.empty(); + + steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); return new Run(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), JobType.fromJobName(runObject.field(jobTypeField).asString()), @@ -182,7 +195,13 @@ class RunSerializer { run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + run.steps().forEach((step, statusInfo) -> stepsObject.setString(valueOf(step), valueOf(statusInfo.status()))); + + // For historical reasons are the step details stored in a different field from the step statuses. + Cursor stepDetailsObject = runObject.setObject(stepDetailsField); + run.steps().forEach((step, statusInfo) -> + statusInfo.startTime().ifPresent(startTime -> + stepDetailsObject.setObject(valueOf(step)).setLong(startTimeField, valueOf(startTime)))); Cursor versionsObject = runObject.setObject(versionsField); toSlime(run.versions().targetPlatform(), run.versions().targetApplication(), versionsObject); @@ -265,6 +284,14 @@ class RunSerializer { } } + static Long valueOf(Instant instant) { + return instant.toEpochMilli(); + } + + static Instant instantOf(Long epochMillis) { + return Instant.ofEpochMilli(epochMillis); + } + static String valueOf(RunStatus status) { switch (status) { case running : return "running"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 5406aeb0c61..415768bdea0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -258,7 +258,7 @@ class JobControllerApiHandlerHelper { && run.targetApplication().equals(deployment.applicationVersion())) .isPresent()); if (running.containsKey(type)) - deploymentObject.setString("status", running.get(type).steps().get(deployReal) == unfinished ? "deploying" : "verifying"); + deploymentObject.setString("status", running.get(type).stepStatus(deployReal).equals(Optional.of(unfinished)) ? "deploying" : "verifying"); else if (change.hasTargets()) deploymentObject.setString("status", pendingProduction.containsKey(type) ? "pending" : "completed"); } @@ -378,7 +378,7 @@ class JobControllerApiHandlerHelper { versionsToSlime(runObject, run.versions()); Cursor stepsObject = runObject.setObject("steps"); - run.steps().forEach((step, status) -> stepsObject.setString(step.name(), status.name())); + run.steps().forEach((step, info) -> stepsObject.setString(step.name(), info.status().name())); Cursor tasksObject = runObject.setObject("tasks"); taskStatus(deployReal, run).ifPresent(status -> tasksObject.setString("deploy", status)); taskStatus(Step.installReal, run).ifPresent(status -> tasksObject.setString("install", status)); @@ -391,8 +391,8 @@ class JobControllerApiHandlerHelper { private static Optional<String> taskStatus(Step step, Run run) { return run.readySteps().contains(step) ? Optional.of("running") : Optional.ofNullable(run.steps().get(step)) - .filter(status -> status != unfinished) - .map(Step.Status::name); + .filter(info -> info.status() != unfinished) + .map(info -> info.status().name()); } /** Returns a response with the runs for the given job type. */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java index 1bb3120dd8e..79e4072c2e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java @@ -4,16 +4,10 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import org.junit.Assert; import org.junit.Test; -import java.io.UnsupportedEncodingException; import java.net.URI; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.Instant; import java.util.Collections; -import java.util.EnumMap; import java.util.List; import java.util.Optional; @@ -23,7 +17,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.time.Instant.EPOCH; import static java.time.Instant.now; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; /** * @author jonmv @@ -31,13 +24,13 @@ import static org.junit.Assert.fail; public class BadgesTest { private static final ApplicationId id = ApplicationId.from("tenant", "application", "default"); - private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, Step.Status.succeeded), + private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty()); - private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, Step.Status.succeeded), + private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty()); - private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, Step.Status.succeeded), + private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty()); @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ee65ffee62f..4abbf5be09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -394,7 +394,8 @@ public class DeploymentContext { setEndpoints(JobType.systemTest.zone(tester.controller().system())); setTesterEndpoints(JobType.systemTest.zone(tester.controller().system())); runner.run(); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.endTests)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); + assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent()); return id; } @@ -418,13 +419,13 @@ public class DeploymentContext { runner.advance(currentRun(job)); if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installInitialReal)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); Versions versions = currentRun(job).versions(); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform())); configServer().convergeServices(id.application(), zone); setEndpoints(zone); runner.advance(currentRun(job)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installInitialReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); } } @@ -434,7 +435,7 @@ public class DeploymentContext { ZoneId zone = zone(job); DeploymentId deployment = new DeploymentId(job.application(), zone); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), currentRun(job).versions().targetPlatform()); runner.advance(currentRun(job)); } @@ -482,16 +483,16 @@ public class DeploymentContext { RunId id = currentRun(job).id(); ZoneId zone = zone(job); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().convergeServices(id.application(), zone); setEndpoints(zone); runner.advance(currentRun(job)); if (job.type().environment().isManuallyDeployed()) { - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); assertTrue(jobs.run(id).get().hasEnded()); return; } - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); } /** Installs tester and starts tests. */ @@ -499,13 +500,13 @@ public class DeploymentContext { RunId id = currentRun(job).id(); ZoneId zone = zone(job); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installTester)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().nodeRepository().doUpgrade(new DeploymentId(TesterId.of(job.application()).id(), zone), Optional.empty(), currentRun(job).versions().targetPlatform()); runner.advance(currentRun(job)); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installTester)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); runner.advance(currentRun(job)); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installTester)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); setTesterEndpoints(zone); runner.advance(currentRun(job)); } @@ -517,11 +518,11 @@ public class DeploymentContext { // All installation is complete and endpoints are ready, so tests may begin. if ( ! job.type().isTest()) - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installTester)); - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.startTests)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startTests)); - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.endTests)); + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); cloud.set(TesterCloud.Status.SUCCESS); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 0340cb25d6f..70bc783a757 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -105,7 +105,7 @@ public class InternalStepRunnerTest { tester.setEndpoints(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installInitialReal)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installInitialReal)); tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); @@ -118,7 +118,7 @@ public class InternalStepRunnerTest { singletonList("Refeed it!"))))); tester.runner().run(); - assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.deployReal)); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); } @Test @@ -140,15 +140,15 @@ public class InternalStepRunnerTest { singletonList("Restart it!"))), Collections.emptyList())); tester.runner().run(); - assertEquals(succeeded, tester.jobs().run(id).get().steps().get(Step.deployReal)); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); tester.configServer().convergeServices(app.instanceId(), zone); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); tester.configServer().nodeRepository().doRestart(app.deploymentIdIn(zone), Optional.of(host)); tester.configServer().nodeRepository().requestReboot(app.deploymentIdIn(zone), Optional.of(host)); tester.runner().run(); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); @@ -174,8 +174,8 @@ public class InternalStepRunnerTest { tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().steps().get(Step.installTester)); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); } @Test @@ -185,11 +185,11 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installTester)); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); } @@ -204,7 +204,7 @@ public class InternalStepRunnerTest { tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.startTests)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); } @Test @@ -213,8 +213,8 @@ public class InternalStepRunnerTest { tester.runner().run();; tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installTester)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); tester.controller().curator().writeRoutingPolicies(app.instanceId(), Set.of(new RoutingPolicy(app.instanceId(), ClusterSpec.Id.from("default"), @@ -229,8 +229,8 @@ public class InternalStepRunnerTest { Optional.empty(), emptySet(), true))); tester.runner().run();; - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installReal)); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().steps().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); } @Test @@ -238,7 +238,7 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.NOT_STARTED); tester.runner().run(); - assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); } @Test @@ -252,7 +252,7 @@ public class InternalStepRunnerTest { assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(321), error, "Failure!"), new LogEntry(lastId + 2, tester.clock().instant(), info, "Tests failed.")); - assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); } @Test @@ -263,7 +263,7 @@ public class InternalStepRunnerTest { long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); - assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(123), error, "Error!"), new LogEntry(lastId + 2, tester.clock().instant(), info, "Tester failed running its tests!")); @@ -273,7 +273,7 @@ public class InternalStepRunnerTest { public void testsSucceedWhenTheyDoRemotely() { RunId id = app.startSystemTestTests(); tester.runner().run(); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(app.testerId().id(), JobType.systemTest.zone(system()))).get(0).endpoint()), tester.cloud().testerUrl()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); @@ -304,7 +304,7 @@ public class InternalStepRunnerTest { new LogEntry(lastId + 2, Instant.ofEpochMilli(1234), info, "Steady!"), new LogEntry(lastId + 3, Instant.ofEpochMilli(12345), info, "Success!"), new LogEntry(lastId + 4, tester.clock().instant(), info, "Tests completed successfully.")); - assertEquals(succeeded, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); } @Test @@ -313,7 +313,7 @@ public class InternalStepRunnerTest { tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); tester.runner().run(); RunId id = tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().id(); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); Version version = new Version("7.8.9"); Future<?> concurrentDeployment = Executors.newSingleThreadExecutor().submit(() -> { @@ -329,7 +329,7 @@ public class InternalStepRunnerTest { tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(app.instanceId(), zone); tester.setEndpoints(app.instanceId(), zone); - assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); @@ -366,7 +366,7 @@ public class InternalStepRunnerTest { tester.configServer().setLogStream(vespaLog); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); - assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 08b5380922b..74e387ef17e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -26,6 +25,7 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; @@ -97,14 +97,14 @@ public class JobRunnerTest { catch (IllegalStateException e) { } jobs.start(id, stagingTest, versions); - assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); + assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); assertFalse(jobs.last(id, systemTest).get().hasEnded()); - assertTrue(jobs.last(id, stagingTest).get().steps().values().stream().allMatch(unfinished::equals)); + assertTrue(jobs.last(id, stagingTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); assertFalse(jobs.last(id, stagingTest).get().hasEnded()); runner.maintain(); phaser.arriveAndAwaitAdvance(); - assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); + assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(succeeded::equals)); assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); } @@ -125,45 +125,54 @@ public class JobRunnerTest { jobs.start(id, systemTest, versions); RunId first = run.get().id(); - Map<Step, Status> steps = run.get().steps(); + Map<Step, Status> steps = run.get().stepStatuses(); runner.maintain(); - assertEquals(steps, run.get().steps()); + assertEquals(steps, run.get().stepStatuses()); assertEquals(List.of(deployTester), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester); outcomes.put(deployTester, running); runner.maintain(); assertEquals(List.of(deployReal), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal); outcomes.put(deployReal, running); runner.maintain(); assertEquals(List.of(installTester, installReal), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal); outcomes.put(installReal, running); runner.maintain(); assertEquals(List.of(installTester), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal); outcomes.put(installTester, running); runner.maintain(); assertEquals(List.of(startTests), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests); outcomes.put(startTests, running); runner.maintain(); assertEquals(List.of(endTests), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests); // Failure ending tests fails the run, but run-always steps continue. outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); assertEquals(List.of(copyVespaLogs, deactivateTester), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester); outcomes.put(copyVespaLogs, running); runner.maintain(); assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester, deactivateReal); // Abortion does nothing, as the run has already failed. jobs.abort(run.get().id()); runner.maintain(); assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester, deactivateReal); outcomes.put(deactivateReal, running); outcomes.put(deactivateTester, running); @@ -183,9 +192,11 @@ public class JobRunnerTest { assertTrue(run.get().hasEnded()); assertTrue(run.get().hasFailed()); assertFalse(run.get().status() == aborted); - assertEquals(failed, run.get().steps().get(deployTester)); - assertEquals(unfinished, run.get().steps().get(installTester)); - assertEquals(succeeded, run.get().steps().get(report)); + assertEquals(failed, run.get().stepStatuses().get(deployTester)); + assertEquals(unfinished, run.get().stepStatuses().get(installTester)); + assertEquals(succeeded, run.get().stepStatuses().get(report)); + // deployTester, plus all forced steps: + assertStepsWithStartTime(run.get(), deployTester, copyVespaLogs, deactivateTester, deactivateReal, report); assertEquals(2, jobs.runs(id, systemTest).size()); @@ -197,6 +208,14 @@ public class JobRunnerTest { assertTrue(jobs.runs(id, systemTest).isEmpty()); } + private void assertStepsWithStartTime(Run lastRun, Step... stepsWithStartTime) { + Set<Step> actualStepsWithStartTime = lastRun.steps().entrySet().stream() + .filter(entry -> entry.getValue().startTime().isPresent()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + assertEquals(Set.of(stepsWithStartTime), actualStepsWithStartTime); + } + @Test public void locksAndGarbage() throws InterruptedException, BrokenBarrierException { DeploymentTester tester = new DeploymentTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index f76880c3f7a..1cd361b4d74 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -6,13 +6,14 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import org.junit.Test; import java.io.IOException; @@ -21,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Collections; +import java.util.Optional; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -98,21 +100,21 @@ public class RunSerializerTest { "5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n" + "-----END CERTIFICATE-----"), run.testerCertificate().get()); - assertEquals(ImmutableMap.<Step, Step.Status>builder() - .put(deployInitialReal, unfinished) - .put(installInitialReal, failed) - .put(deployReal, succeeded) - .put(installReal, unfinished) - .put(deactivateReal, failed) - .put(deployTester, succeeded) - .put(installTester, unfinished) - .put(deactivateTester, failed) - .put(copyVespaLogs, succeeded) - .put(startTests, succeeded) - .put(endTests, unfinished) - .put(report, failed) - .build(), - run.steps()); + assertEquals(ImmutableMap.<Step, StepInfo>builder() + .put(deployInitialReal, new StepInfo(deployInitialReal, unfinished, Optional.empty())) + .put(installInitialReal, new StepInfo(installInitialReal, failed, Optional.of(Instant.ofEpochMilli(1196676940000L)))) + .put(deployReal, new StepInfo(deployReal, succeeded, Optional.empty())) + .put(installReal, new StepInfo(installReal, unfinished, Optional.empty())) + .put(deactivateReal, new StepInfo(deactivateReal, failed, Optional.empty())) + .put(deployTester, new StepInfo(deployTester, succeeded, Optional.empty())) + .put(installTester, new StepInfo(installTester, unfinished, Optional.of(Instant.ofEpochMilli(1196677940000L)))) + .put(deactivateTester, new StepInfo(deactivateTester, failed, Optional.empty())) + .put(copyVespaLogs, new StepInfo(copyVespaLogs, succeeded, Optional.empty())) + .put(startTests, new StepInfo(startTests, succeeded, Optional.empty())) + .put(endTests, new StepInfo(endTests, unfinished, Optional.empty())) + .put(report, new StepInfo(report, failed, Optional.empty())) + .build(), + run.steps()); run = run.with(1L << 50) .with(Instant.now().truncatedTo(MILLIS)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 535c87d9a7a..bea13e68d11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -22,6 +22,14 @@ "endTests": "unfinished", "report": "failed" }, + "stepDetails": { + "installInitialReal": { + "startTime": 1196676940000 + }, + "installTester": { + "startTime": 1196677940000 + } + }, "versions": { "platform": "1.2.3", "repository": "git@github.com:user/repo.git", |