diff options
6 files changed, 100 insertions, 35 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7136666962d..de3e29386c9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -107,6 +107,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -498,7 +499,7 @@ public class ApplicationController { } /** Deploys an application package for an existing application instance. */ - public DeploymentResult deploy(JobId job, boolean deploySourceVersions, Consumer<String> deployLogger) { + public DeploymentResult deploy(JobId job, boolean deploySourceVersions, Consumer<String> deployLogger, UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { if (job.application().instance().isTester()) throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); @@ -528,8 +529,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, preparedEndpoints, - run.isDryRun(), run.testerCertificate()); - + run.isDryRun(), run.testerCertificate(), cloudAccountOverride); // Record the quota usage for this application var quotaUsage = deploymentQuotaUsage(zone, job.application()); @@ -650,22 +650,23 @@ public class ApplicationController { ApplicationPackageStream applicationPackage = new ApplicationPackageStream( () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), applicationPackage, zone, version, null, false, Optional.empty()).result(); + return deploy(application.id(), applicationPackage, zone, version, null, false, Optional.empty(), UnaryOperator.identity()).result(); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } } /** Deploys the given tester application to the given zone. */ - public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), applicationPackage, zone, platform, null, false, Optional.empty()).result(); + public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { + return deploy(tester.id(), applicationPackage, zone, platform, null, false, Optional.empty(), cloudAccountOverride).result(); } private record DeploymentDataAndResult(DeploymentData data, DeploymentResult result) {} private DeploymentDataAndResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, Supplier<PreparedEndpoints> preparedEndpoints, - boolean dryRun, Optional<X509Certificate> testerCertificate) { + boolean dryRun, Optional<X509Certificate> testerCertificate, + UnaryOperator<Optional<CloudAccount>> cloudAccountOverride) { DeploymentId deployment = new DeploymentId(application, zone); // Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails. interface CleanCloseable extends AutoCloseable { void close(); } @@ -697,9 +698,7 @@ public class ApplicationController { if (testerCertificate.isPresent()) { operatorCertificates = Stream.concat(operatorCertificates.stream(), testerCertificate.stream()).toList(); } - Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, - zone.environment().isTest() ? requireApplication(TenantAndApplicationId.from(application)).deploymentSpec() - : applicationPackage.truncatedPackage().deploymentSpec()); + Supplier<Optional<CloudAccount>> cloudAccount = () -> cloudAccountOverride.apply(decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec())); List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant()); Supplier<DeploymentEndpoints> endpoints = () -> { if (preparedEndpoints == null) return DeploymentEndpoints.none; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 0ce3a3d75d4..919facee0c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -7,10 +7,12 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.EndpointsChecker; import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.config.provision.EndpointsChecker.Status; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -60,6 +62,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -83,6 +86,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; @@ -92,7 +96,10 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.yolean.Exceptions.uncheck; +import static com.yahoo.yolean.Exceptions.uncheckInterruptedAndRestoreFlag; +import static java.lang.Math.min; import static java.util.Objects.requireNonNull; +import static java.util.function.Predicate.not; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; @@ -148,9 +155,9 @@ public class InternalStepRunner implements StepRunner { } catch (UncheckedIOException e) { logger.logWithInternalException(INFO, "IO exception running " + id + ": " + Exceptions.toMessageString(e), e); return Optional.empty(); - } catch (RuntimeException|LinkageError e) { + } catch (RuntimeException | LinkageError e) { logger.log(WARNING, "Unexpected exception running " + id, e); - if (step.get().alwaysRun() && !(e instanceof LinkageError)) { + if (step.get().alwaysRun() && ! (e instanceof LinkageError)) { logger.log("Will keep trying, as this is a cleanup step."); return Optional.empty(); } @@ -176,7 +183,10 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { Optional<X509Certificate> testerCertificate = controller.jobController().run(id).testerCertificate(); - return deploy(() -> controller.applications().deploy(id.job(), setTheStage, logger::log), + return deploy(() -> controller.applications().deploy(id.job(), + setTheStage, + logger::log, + account -> getCloudAccountWithOverrideForStaging(id, account)), controller.jobController().run(id) .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), @@ -198,7 +208,8 @@ public class InternalStepRunner implements StepRunner { return deploy(() -> controller.applications().deployTester(id.tester(), testerPackage(id), id.type().zone(), - platform), + platform, + cloudAccount -> setCloudAccountForStaging(id, cloudAccount)), controller.jobController().run(id) .stepInfo(deployTester).get() .startTime().get(), @@ -206,6 +217,36 @@ public class InternalStepRunner implements StepRunner { logger); } + private Optional<CloudAccount> setCloudAccountForStaging(RunId id, Optional<CloudAccount> account) { + if (id.type().environment() == Environment.staging) { + controller.jobController().locked(id, run -> run.with(account.orElse(CloudAccount.empty))); + } + return account; + } + + private Optional<CloudAccount> getCloudAccountWithOverrideForStaging(RunId id, Optional<CloudAccount> account) { + if (id.type().environment() == Environment.staging) { + Instant doom = controller.clock().instant().plusSeconds(60); // Sleeping is bad, but we're already in a sleepy code path: deployment. + while (true) { + Run run = controller.jobController().run(id); + Optional<CloudAccount> stored = run.cloudAccount(); + if (stored.isPresent()) + return stored.filter(not(CloudAccount.empty::equals)); + + // TODO jonmv: remove with next release + if (run.stepStatus(deployTester).get() != unfinished) + return account; // Use original value for runs which started prior to this code change, and resumed after. Extremely unlikely :> + + long millisToDoom = Duration.between(controller.clock().instant(), doom).toMillis(); + if (millisToDoom > 0) + uncheckInterruptedAndRestoreFlag(() -> Thread.sleep(min(millisToDoom, 5000))); + else + throw new CloudAccountNotSetException("Cloud account not yet set; must deploy tests first"); + } + } + return account; + } + private Optional<RunStatus> deploy(Supplier<DeploymentResult> deployment, Instant startTime, RunId id, DualLogger logger) { try { DeploymentResult result = deployment.get(); @@ -276,6 +317,10 @@ public class InternalStepRunner implements StepRunner { throw e; } + catch (CloudAccountNotSetException e) { + logger.log(INFO, "Timed out waiting for cloud account to be set for " + id + ": " + e.getMessage()); + return Optional.empty(); + } catch (IllegalArgumentException e) { logger.log(WARNING, e.getMessage()); return Optional.of(deploymentFailed); @@ -1006,4 +1051,8 @@ public class InternalStepRunner implements StepRunner { } + private static class CloudAccountNotSetException extends RuntimeException { + CloudAccountNotSetException(String message) { super(message); } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 36df2aeda10..0c5fb3fb3cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.security.cert.X509Certificate; @@ -15,7 +16,6 @@ import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -43,13 +43,14 @@ public class Run { private final Optional<ConvergenceSummary> convergenceSummary; private final Optional<X509Certificate> testerCertificate; private final boolean dryRun; + private final Optional<CloudAccount> cloudAccount; private final Optional<String> reason; // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, - Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { + Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<CloudAccount> cloudAccount, Optional<String> reason) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -64,6 +65,7 @@ public class Run { this.convergenceSummary = convergenceSummary; this.testerCertificate = testerCertificate; this.dryRun = dryRun; + this.cloudAccount = cloudAccount; this.reason = reason; } @@ -72,7 +74,7 @@ public class Run { profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), - Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); + Optional.empty(), profile == JobProfile.developmentDryRun, Optional.empty(), triggeredBy); } /** Returns a new Run with the status of the given completed step set accordingly. */ @@ -87,7 +89,7 @@ public class Run { steps.put(step.get(), stepInfo.with(Step.Status.of(status))); RunStatus newStatus = hasFailed() || status == running ? this.status : status; return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord, - lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } /** Returns a new Run with a new start time*/ @@ -102,13 +104,13 @@ public class Run { steps.put(step.get(), stepInfo.with(startTime)); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } public Run finished(Instant now) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), sleepUntil, status == running ? success : status, - lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun, reason); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty(), dryRun, cloudAccount, reason); } public Run aborted(boolean cancelledByHumans) { @@ -116,7 +118,7 @@ public class Run { return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, cancelledByHumans ? cancelled : aborted, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, - convergenceSummary, testerCertificate, dryRun, reason); + convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } public Run reset() { @@ -124,43 +126,49 @@ public class Run { Map<Step, StepInfo> reset = new EnumMap<>(steps); reset.replaceAll((step, __) -> StepInfo.initial(step)); return new Run(id, reset, versions, isRedeployment, start, end, sleepUntil, running, -1, lastVespaLogTimestamp, - Optional.empty(), Optional.empty(), testerCertificate, dryRun, reason); + Optional.empty(), Optional.empty(), testerCertificate, dryRun, cloudAccount, reason); } public Run with(long lastTestRecord) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun, reason); + Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun, reason); + noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate, dryRun, cloudAccount, reason); } public Run with(X509Certificate testerCertificate) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun, reason); + noNodesDownSince, convergenceSummary, Optional.of(testerCertificate), dryRun, cloudAccount, reason); } public Run sleepingUntil(Instant instant) { requireActive(); return new Run(id, steps, versions, isRedeployment, start, end, Optional.of(instant), status, lastTestRecord, lastVespaLogTimestamp, - noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason); + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, cloudAccount, reason); + } + + public Run with(CloudAccount account) { + requireActive(); + return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate, dryRun, Optional.of(account), reason); } /** Returns the id of this run. */ @@ -266,6 +274,9 @@ public class Run { /** Whether this is a dry run deployment. */ public boolean isDryRun() { return dryRun; } + /** Cloud account override to use for this run, if set. This should only be used by staging tests. */ + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + /** The specific reason for triggering this run, if any. This should be empty for jobs triggered bvy deployment orchestration. */ public Optional<String> reason() { return reason; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 4547eed24c8..73d0bf6cad6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -98,6 +99,7 @@ class RunSerializer { private static final String convergenceSummaryField = "convergenceSummaryV2"; private static final String testerCertificateField = "testerCertificate"; private static final String isDryRunField = "isDryRun"; + private static final String cloudAccountField = "account"; private static final String reasonField = "reason"; Run runFromSlime(Slime slime) { @@ -142,10 +144,9 @@ class RunSerializer { Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), SlimeUtils.optionalInstant(runObject.field(noNodesDownSinceField)), convergenceSummaryFrom(runObject.field(convergenceSummaryField)), - Optional.of(runObject.field(testerCertificateField)) - .filter(Inspector::valid) - .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())), + SlimeUtils.optionalString(runObject.field(testerCertificateField)).map(X509CertificateUtils::fromPem), runObject.field(isDryRunField).valid() && runObject.field(isDryRunField).asBool(), + SlimeUtils.optionalString(runObject.field(cloudAccountField)).map(CloudAccount::from), SlimeUtils.optionalString(runObject.field(reasonField))); } @@ -239,6 +240,7 @@ class RunSerializer { versionsObject.setObject(sourceField)); }); runObject.setBool(isDryRunField, run.isDryRun()); + run.cloudAccount().ifPresent(account -> runObject.setString(cloudAccountField, account.value())); run.reason().ifPresent(reason -> runObject.setString(reasonField, reason)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index a03583c4a59..ed5226ebc8b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -400,6 +400,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer catch (IOException e) { throw new UncheckedIOException(e); } + deployment.cloudAccount(); // Supplier with side effects >_< lastPrepareVersion = deployment.platform(); if (prepareException != null) prepareException.accept(ApplicationId.from(deployment.instance().tenant(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 3aac3e2f757..cae5037ab6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -99,6 +100,7 @@ public class RunSerializerTest { "5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n" + "-----END CERTIFICATE-----"), run.testerCertificate().get()); + assertEquals(Optional.empty(), run.cloudAccount()); assertEquals(ImmutableMap.<Step, StepInfo>builder() .put(deployInitialReal, new StepInfo(deployInitialReal, unfinished, Optional.empty())) .put(installInitialReal, new StepInfo(installInitialReal, failed, Optional.of(Instant.ofEpochMilli(1196676940000L)))) @@ -118,10 +120,11 @@ public class RunSerializerTest { run.steps()); run = run.with(1L << 50) - .with(Instant.now().truncatedTo(MILLIS)) - .noNodesDownSince(Instant.now().truncatedTo(MILLIS)) - .aborted(false) - .finished(Instant.now().truncatedTo(MILLIS)); + .with(Instant.now().truncatedTo(MILLIS)) + .noNodesDownSince(Instant.now().truncatedTo(MILLIS)) + .aborted(false) + .with(CloudAccount.from("gcp:foobar")) + .finished(Instant.now().truncatedTo(MILLIS)); assertEquals(aborted, run.status()); assertTrue(run.hasEnded()); |