diff options
Diffstat (limited to 'controller-server')
11 files changed, 409 insertions, 216 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 61231404a94..02e0d94920e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -16,11 +15,9 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; /** * Information about which deployment jobs an application should run and their current status. @@ -123,12 +120,27 @@ public class DeploymentJobs { return true; } if (environment == Environment.staging) { - return isSuccessful(JobType.systemTest, change.get()); + return isSuccessful(change.get(), JobType.systemTest); } else if (environment == Environment.prod) { - return isSuccessful(JobType.stagingTest, change.get()); + return isSuccessful(change.get(), JobType.stagingTest); } return true; // other environments do not have any preconditions } + + /** Returns whether change has been deployed completely */ + public boolean isDeployed(Change change) { + return status.values().stream() + .filter(status -> status.type().isProduction()) + .allMatch(status -> isSuccessful(change, status.type())); + } + + /** Returns whether job has completed successfully */ + public boolean isSuccessful(Change change, JobType jobType) { + return Optional.ofNullable(jobStatus().get(jobType)) + .filter(JobStatus::isSuccess) + .filter(status -> status.lastCompletedFor(change)) + .isPresent(); + } /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */ public Instant failingSince() { @@ -150,13 +162,6 @@ public class DeploymentJobs { public Optional<String> jiraIssueId() { return jiraIssueId; } - private boolean isSuccessful(JobType jobType, Change change) { - return Optional.ofNullable(jobStatus().get(jobType)) - .filter(JobStatus::isSuccess) - .filter(status -> status.lastCompletedFor(change)) - .isPresent(); - } - /** Job types that exist in the build system */ public enum JobType { @@ -252,14 +257,6 @@ public class DeploymentJobs { return from(system, new com.yahoo.config.provision.Zone(environment, region)); } - /** Returns the trigger order to use according to deployment spec */ - public static List<JobType> triggerOrder(SystemName system, DeploymentSpec deploymentSpec) { - return deploymentSpec.zones().stream() - .map(declaredZone -> JobType.from(system, declaredZone.environment(), - declaredZone.region().orElse(null))) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - private static Zone zone(SystemName system, String environment, String region) { return new Zone(system, Environment.from(environment), RegionName.from(region)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java new file mode 100644 index 00000000000..448ab419853 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -0,0 +1,162 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.collectingAndThen; + +/** + * This class determines the order of deployments according to an application's deployment spec. + * + * @author mpolden + */ +public class DeploymentOrder { + + private static final Logger log = Logger.getLogger(DeploymentOrder.class.getName()); + + private final Controller controller; + private final Clock clock; + + public DeploymentOrder(Controller controller) { + Objects.requireNonNull(controller, "controller cannot be null"); + this.controller = controller; + this.clock = controller.clock(); + } + + /** Returns a list of jobs to trigger after the given job */ + public List<JobType> nextAfter(JobType job, Application application) { + // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a + // new application with no previous deployments) + if (job == JobType.component) { + return Collections.singletonList(JobType.systemTest); + } + + // At this point we have deployed to system test, so deployment spec is available + List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); + Optional<DeploymentSpec.Step> currentStep = fromJob(job, application); + if ( ! currentStep.isPresent()) { + return Collections.emptyList(); + } + + // If this is the last deployment step there's nothing more to trigger + int currentIndex = deploymentSteps.indexOf(currentStep.get()); + if (currentIndex == deploymentSteps.size() - 1) { + return Collections.emptyList(); + } + + // Postpone if step hasn't completed all it's jobs for this change + if (!completedSuccessfully(currentStep.get(), application)) { + return Collections.emptyList(); + } + + // Postpone next job if delay has not passed yet + Duration delay = delayAfter(currentStep.get(), application); + if (postponeDeployment(delay, job, application)) { + log.info(String.format("Delaying next job after %s of %s by %s", job, application, delay)); + return Collections.emptyList(); + } + + DeploymentSpec.Step nextStep = deploymentSteps.get(currentIndex + 1); + return nextStep.zones().stream() + .map(this::toJob) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns whether the given job is first in a deployment */ + public boolean isFirst(JobType job) { + return job == JobType.component; + } + + /** Returns whether the given job is last in a deployment */ + public boolean isLast(JobType job, Application application) { + List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); + if (deploymentSteps.isEmpty()) { // Deployment spec not yet available + return false; + } + DeploymentSpec.Step lastStep = deploymentSteps.get(deploymentSteps.size() - 1); + return fromJob(job, application).get().equals(lastStep); + } + + /** Returns jobs for given deployment spec, in the order they are declared */ + public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) { + return deploymentSpec.steps().stream() + .flatMap(step -> jobsFrom(step).stream()) + .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns jobs for the given step */ + private List<JobType> jobsFrom(DeploymentSpec.Step step) { + return step.zones().stream() + .map(this::toJob) + .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Returns whether all jobs have completed successfully for given step */ + private boolean completedSuccessfully(DeploymentSpec.Step step, Application application) { + return jobsFrom(step).stream() + .allMatch(job -> application.deploymentJobs().isSuccessful(application.deploying().get(), job)); + } + + /** Resolve deployment step from job */ + private Optional<DeploymentSpec.Step> fromJob(JobType job, Application application) { + for (DeploymentSpec.Step step : application.deploymentSpec().steps()) { + if (step.deploysTo(job.environment(), job.isProduction() ? job.region(controller.system()) : Optional.empty())) { + return Optional.of(step); + } + } + return Optional.empty(); + } + + /** Resolve job from deployment step */ + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)); + } + + /** Returns whether deployment should be postponed according to delay */ + private boolean postponeDeployment(Duration delay, JobType job, Application application) { + Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(job)) + .flatMap(JobStatus::lastSuccess) + .map(JobStatus.JobRun::at); + return lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant()); + } + + /** Find all steps that deploy to one or more zones */ + private static List<DeploymentSpec.Step> deploymentSteps(Application application) { + return application.deploymentSpec().steps().stream() + .filter(step -> step instanceof DeploymentSpec.DeclaredZone || + step instanceof DeploymentSpec.ParallelZones) + .collect(Collectors.toList()); + } + + /** Determines the delay that should pass after the given step */ + private static Duration delayAfter(DeploymentSpec.Step step, Application application) { + int stepIndex = application.deploymentSpec().steps().indexOf(step); + if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) { + return Duration.ZERO; + } + Duration totalDelay = Duration.ZERO; + List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps() + .subList(stepIndex + 1, application.deploymentSpec().steps().size()); + for (DeploymentSpec.Step s : remainingSteps) { + if (!(s instanceof DeploymentSpec.Delay)) { + break; + } + totalDelay = totalDelay.plus(((DeploymentSpec.Delay) s).duration()); + } + return totalDelay; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index ce1fee4dd14..5f529d21995 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -39,6 +39,7 @@ public class DeploymentTrigger { private final Controller controller; private final Clock clock; private final BuildSystem buildSystem; + private final DeploymentOrder order; public DeploymentTrigger(Controller controller, CuratorDb curator, Clock clock) { Objects.requireNonNull(controller,"controller cannot be null"); @@ -46,6 +47,7 @@ public class DeploymentTrigger { this.controller = controller; this.clock = clock; this.buildSystem = new PolledBuildSystem(controller, curator); + this.order = new DeploymentOrder(controller); } //--- Start of methods which triggers deployment jobs ------------------------- @@ -62,7 +64,7 @@ public class DeploymentTrigger { application = application.withJobCompletion(report, clock.instant(), controller); // Handle successful first and last job - if (isFirstJob(report.jobType()) && report.success()) { // the first job tells us that a change occurred + if (order.isFirst(report.jobType()) && report.success()) { // the first job tells us that a change occurred if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) { // postpone until the current deployment is done applications().store(application.withOutstandingChange(true), lock); return; @@ -70,17 +72,17 @@ public class DeploymentTrigger { else { // start a new change deployment application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown())); } - } else if (isLastJob(report.jobType(), application) && report.success()) { + } else if (order.isLast(report.jobType(), application) && report.success() && application.deploymentJobs().isDeployed(application.deploying().get())) { application = application.withDeploying(Optional.empty()); } // Trigger next if (report.success()) - application = trigger(nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock); + application = trigger(order.nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock); else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType())) application = trigger(report.jobType(), application, true, "Retrying due to out of capacity", lock); else if (shouldRetryNow(application)) - application = trigger(report.jobType(), application, "Retrying as job just started failing", lock); + application = trigger(report.jobType(), application, false, "Retrying as job just started failing", lock); applications().store(application, lock); } @@ -95,15 +97,15 @@ public class DeploymentTrigger { if (shouldRetryFromBeginning(application)) { // failed for a long time: Discard existing change and restart from the component job application = application.withDeploying(Optional.empty()); - application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock); + application = trigger(JobType.component, application, false, "Retrying failing deployment from beginning: " + cause, lock); applications().store(application, lock); } else { // retry the failed job (with backoff) - for (JobType jobType : JobType.triggerOrder(controller.system(), application.deploymentSpec())) { // retry the *first* failing job + for (JobType jobType : order.jobsFrom(application.deploymentSpec())) { // retry the *first* failing job JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); if (isFailing(jobStatus)) { if (shouldRetryNow(jobStatus)) { - application = trigger(jobType, application, "Retrying failing job: " + cause, lock); + application = trigger(jobType, application, false, "Retrying failing job: " + cause, lock); applications().store(application, lock); } break; @@ -133,7 +135,7 @@ public class DeploymentTrigger { // Trigger next try (Lock lock = applications().lock(application.id())) { application = applications().require(application.id()); - application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application, + application = trigger(order.nextAfter(lastSuccessfulJob.get().type(), application), application, "Resuming delayed deployment", lock); applications().store(application, lock); } @@ -154,7 +156,7 @@ public class DeploymentTrigger { application = application.withDeploying(Optional.of(change)); if (change instanceof Change.ApplicationChange) application = application.withOutstandingChange(false); - application = trigger(JobType.systemTest, application, "Deploying change", lock); + application = trigger(JobType.systemTest, application, false, "Deploying change", lock); applications().store(application, lock); } } @@ -177,77 +179,6 @@ public class DeploymentTrigger { private ApplicationController applications() { return controller.applications(); } - /** Returns the next job to trigger after this job, or null if none should be triggered */ - private JobType nextAfter(JobType jobType, Application application) { - // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a - // new application with no previous deployments) - if (jobType == JobType.component) { - return JobType.systemTest; - } - - // At this point we've at least deployed to system test, so deployment spec should be available - List<DeploymentSpec.DeclaredZone> zones = application.deploymentSpec().zones(); - Optional<DeploymentSpec.DeclaredZone> zoneForJob = zoneForJob(application, jobType); - if (!zoneForJob.isPresent()) { - return null; - } - int zoneIndex = application.deploymentSpec().zones().indexOf(zoneForJob.get()); - - // This is last zone - if (zoneIndex == zones.size() - 1) { - return null; - } - - // Skip next job if delay has not passed yet - Duration delay = delayAfter(application, zoneForJob.get()); - Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) - .flatMap(JobStatus::lastSuccess) - .map(JobStatus.JobRun::at); - if (lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant())) { - log.info(String.format("Delaying next job after %s of %s by %s", jobType, application, delay)); - return null; - } - - DeploymentSpec.DeclaredZone nextZone = application.deploymentSpec().zones().get(zoneIndex + 1); - return JobType.from(controller.system(), nextZone.environment(), nextZone.region().orElse(null)); - } - - private Duration delayAfter(Application application, DeploymentSpec.DeclaredZone zone) { - int stepIndex = application.deploymentSpec().steps().indexOf(zone); - if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) { - return Duration.ZERO; - } - Duration totalDelay = Duration.ZERO; - List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps() - .subList(stepIndex + 1, application.deploymentSpec().steps().size()); - for (DeploymentSpec.Step step : remainingSteps) { - if (!(step instanceof DeploymentSpec.Delay)) { - break; - } - totalDelay = totalDelay.plus(((DeploymentSpec.Delay) step).duration()); - } - return totalDelay; - } - - private Optional<DeploymentSpec.DeclaredZone> zoneForJob(Application application, JobType jobType) { - return application.deploymentSpec() - .zones() - .stream() - .filter(zone -> zone.deploysTo( - jobType.environment(), - jobType.isProduction() ? jobType.region(controller.system()) : Optional.empty())) - .findFirst(); - } - - private boolean isFirstJob(JobType jobType) { - return jobType == JobType.component; - } - - private boolean isLastJob(JobType jobType, Application application) { - List<JobType> triggerOrder = JobType.triggerOrder(controller.system(), application.deploymentSpec()); - return triggerOrder.isEmpty() || jobType.equals(triggerOrder.get(triggerOrder.size() - 1)); - } - private boolean isFailing(JobStatus jobStatusOrNull) { return jobStatusOrNull != null && !jobStatusOrNull.isSuccess(); } @@ -356,8 +287,11 @@ public class DeploymentTrigger { return application.withJobTriggering(jobType, clock.instant(), controller); } - private Application trigger(JobType jobType, Application application, String cause, Lock lock) { - return trigger(jobType, application, false, cause, lock); + private Application trigger(List<JobType> jobs, Application application, String cause, Lock lock) { + for (JobType job : jobs) { + application = trigger(job, application, false, cause, lock); + } + return application; } public BuildSystem buildSystem() { return buildSystem; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 3dddfaf58a1..29b34747573 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -104,8 +104,8 @@ public class ControllerTest { applications.notifyJobCompletion(mockReport(app1, component, true, false)); assertFalse("Revision is currently not known", ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); - tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); + tester.deployAndNotify(app1, applicationPackage, true, stagingTest); assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); @@ -120,7 +120,7 @@ public class ControllerTest { tester.clock().advance(Duration.ofSeconds(1)); // production job (failing) - tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, false); + tester.deployAndNotify(app1, applicationPackage, false, productionCorpUsEast1); assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) @@ -144,14 +144,14 @@ public class ControllerTest { // system and staging test job - succeeding applications.notifyJobCompletion(mockReport(app1, component, true, false)); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); assertStatus(JobStatus.initial(systemTest) .withTriggering(version1, revision, tester.clock().instant()) .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); - tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, stagingTest); // production job succeeding now - tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1); expectedJobStatus = expectedJobStatus .withTriggering(version1, revision, tester.clock().instant()) .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()); @@ -161,7 +161,7 @@ public class ControllerTest { assertStatus(JobStatus.initial(productionUsEast3) .withTriggering( version1, revision, tester.clock().instant()), app1.id(), tester.controller()); - tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size()); @@ -189,7 +189,7 @@ public class ControllerTest { .environment(Environment.prod) .region("us-east-3") .build(); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); assertNull("Zone was removed", applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); @@ -211,9 +211,9 @@ public class ControllerTest { // First deployment: An application change applications.notifyJobCompletion(mockReport(app1, component, true, false)); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); - tester.deployAndNotify(stagingTest, app1, applicationPackage, true); - tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); + tester.deployAndNotify(app1, applicationPackage, true, stagingTest); + tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); @@ -234,9 +234,9 @@ public class ControllerTest { .region("us-east-3") .build(); applications.notifyJobCompletion(mockReport(app1, component, true, false)); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); - tester.deployAndNotify(stagingTest, app1, applicationPackage, true); - tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); + tester.deployAndNotify(app1, applicationPackage, true, stagingTest); + tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); @@ -248,7 +248,7 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); @@ -256,10 +256,10 @@ public class ControllerTest { // Version upgrade changes system version Change.VersionChange change = new Change.VersionChange(newSystemVersion); applications.deploymentTrigger().triggerChange(app1.id(), change); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); - tester.deployAndNotify(stagingTest, app1, applicationPackage, true); - tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); - tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); + tester.deployAndNotify(app1, applicationPackage, true, stagingTest); + tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); + tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); @@ -326,37 +326,37 @@ public class ControllerTest { // Initial failure Instant initialFailure = tester.clock().instant(); tester.notifyJobCompletion(component, app, true); - tester.deployAndNotify(systemTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", initialFailure, firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); - tester.deployAndNotify(systemTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", initialFailure, firstFailing(app, tester).get().at()); // Success resets failingSince tester.clock().advance(Duration.ofMillis(1000)); - tester.deployAndNotify(systemTest, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, systemTest); assertFalse(firstFailing(app, tester).isPresent()); // Complete deployment - tester.deployAndNotify(stagingTest, app, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1); // Two repeated failures again. // Initial failure tester.clock().advance(Duration.ofMillis(1000)); initialFailure = tester.clock().instant(); tester.notifyJobCompletion(component, app, true); - tester.deployAndNotify(systemTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", initialFailure, firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); - tester.deployAndNotify(systemTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", initialFailure, firstFailing(app, tester).get().at()); } @@ -435,11 +435,11 @@ public class ControllerTest { // foo: passes system test tester.notifyJobCompletion(component, foo, true); - tester.deployAndNotify(systemTest, foo, applicationPackage, true); + tester.deployAndNotify(foo, applicationPackage, true, systemTest); // bar: passes system test tester.notifyJobCompletion(component, bar, true); - tester.deployAndNotify(systemTest, bar, applicationPackage, true); + tester.deployAndNotify(bar, applicationPackage, true, systemTest); // foo and bar: staging test jobs queued assertEquals(2, buildSystem.jobs().size()); @@ -455,14 +455,14 @@ public class ControllerTest { } // bar: Completes deployment - tester.deployAndNotify(stagingTest, bar, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, bar, applicationPackage, true); + tester.deployAndNotify(bar, applicationPackage, true, stagingTest); + tester.deployAndNotify(bar, applicationPackage, true, productionCorpUsEast1); // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by // out of capacity retry mechanism tester.clock().advance(Duration.ofMinutes(15)); tester.notifyJobCompletion(component, foo, true); - tester.deployAndNotify(systemTest, foo, applicationPackage, true); + tester.deployAndNotify(foo, applicationPackage, true, systemTest); tester.deploy(stagingTest, foo, applicationPackage); assertEquals(1, buildSystem.takeJobsToRun().size()); tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity)); @@ -470,7 +470,7 @@ public class ControllerTest { // bar: New change triggers another staging-test job tester.notifyJobCompletion(component, bar, true); - tester.deployAndNotify(systemTest, bar, applicationPackage, true); + tester.deployAndNotify(bar, applicationPackage, true, systemTest); assertEquals(1, buildSystem.jobs().size()); // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index aa115421f6a..23451c60f08 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -45,6 +46,13 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder parallel(String... regionName) { + environmentBody.append(" <parallel>\n"); + Arrays.stream(regionName).forEach(this::region); + environmentBody.append(" </parallel>\n"); + return this; + } + public ApplicationPackageBuilder delay(Duration delay) { environmentBody.append(" <delay seconds='"); environmentBody.append(delay.getSeconds()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 0e816be864d..9b05101b5eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -23,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; -import java.util.List; import java.util.Optional; import java.util.UUID; @@ -111,10 +109,10 @@ public class DeploymentTester { } private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) { - List<JobType> triggerOrder = JobType.triggerOrder(SystemName.main, applicationPackage.deploymentSpec()); - for (JobType job : triggerOrder) { + DeploymentOrder order = new DeploymentOrder(controller()); + for (JobType job : order.jobsFrom(applicationPackage.deploymentSpec())) { boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false); - deployAndNotify(job, application, applicationPackage, !failJob); + deployAndNotify(application, applicationPackage, !failJob, job); if (failJob) { break; } @@ -163,20 +161,24 @@ public class DeploymentTester { job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); } - public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) { - assertScheduledJob(application, job); - if (success) { - deploy(job, application, applicationPackage); + public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, JobType... jobs) { + assertScheduledJob(application, jobs); + for (JobType job : jobs) { + if (success) { + deploy(job, application, applicationPackage); + } + notifyJobCompletion(job, application, success); } - notifyJobCompletion(job, application, success); } - private void assertScheduledJob(Application application, JobType jobType) { - Optional<BuildService.BuildJob> job = findJob(application, jobType); - assertTrue(String.format("Job %s is scheduled for %s", jobType, application), job.isPresent()); + private void assertScheduledJob(Application application, JobType... jobs) { + for (JobType job : jobs) { + Optional<BuildService.BuildJob> buildJob = findJob(application, job); + assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent()); + assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId()); + assertEquals(job.id(), buildJob.get().jobName()); + } buildSystem().removeJobs(application.id()); - assertEquals((long) application.deploymentJobs().projectId().get(), job.get().projectId()); - assertEquals(jobType.id(), job.get().jobName()); } private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 0f48afc0ca4..7ed0ad843cc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -6,13 +6,14 @@ import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import org.junit.Test; import java.time.Duration; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -24,24 +25,42 @@ public class DeploymentTriggerTest { @Test public void testTriggerFailing() { DeploymentTester tester = new DeploymentTester(); - Application app1 = tester.createAndDeploy("app1", 1, "default"); + Application app = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + Version version = new Version(5, 1); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + + // Deploy completely once + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsWest1); - Version version = new Version(5, 2); - tester.deploymentTrigger().triggerChange(app1.id(), new Change.VersionChange(version)); - tester.completeUpgradeWithError(app1, version, "default", JobType.stagingTest); + // New version is released + version = new Version(5, 2); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + + tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size()); tester.buildSystem().takeJobsToRun(); - assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); + assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); - tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); + tester.failureRedeployer().maintain(); assertEquals("Retried job", 1, tester.buildSystem().jobs().size()); - assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); - tester.clock().advance(Duration.ofHours(7)); - tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); + tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); + tester.failureRedeployer().maintain(); assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName()); } @@ -63,11 +82,11 @@ public class DeploymentTriggerTest { tester.notifyJobCompletion(JobType.component, application, true); // Application is deployed to all test environments and declared zones - tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true); - tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true); - tester.deployAndNotify(JobType.productionCorpUsEast1, application, applicationPackage, true); - tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true); - tester.deployAndNotify(JobType.productionUsWest1, application, applicationPackage, true); + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionCorpUsEast1); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } @@ -91,9 +110,9 @@ public class DeploymentTriggerTest { tester.notifyJobCompletion(JobType.component, application, true); // Test jobs pass - tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true); + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job - tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); // 30 seconds pass, us-west-1 is triggered @@ -121,7 +140,7 @@ public class DeploymentTriggerTest { // 3 minutes pass, us-central-1 is triggered tester.clock().advance(Duration.ofMinutes(3)); tester.deploymentTrigger().triggerDelayed(); - tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); // Delayed trigger job runs again, with nothing to trigger @@ -130,6 +149,77 @@ public class DeploymentTriggerTest { assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } + @Test + public void deploymentSpecWithParallelDeployments() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .region("eu-west-1") + .build(); + + // Component job finishes + tester.notifyJobCompletion(JobType.component, application, true); + + // Test jobs pass + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); + + // Deploys in first region + assertEquals(1, tester.buildSystem().jobs().size()); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); + + // Deploys in two regions in parallel + assertEquals(2, tester.buildSystem().jobs().size()); + assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName()); + tester.buildSystem().takeJobsToRun(); + + tester.deploy(JobType.productionUsWest1, application, applicationPackage, false); + tester.notifyJobCompletion(JobType.productionUsWest1, application, true); + assertTrue("No more jobs triggered at this time", tester.buildSystem().jobs().isEmpty()); + + tester.deploy(JobType.productionUsEast3, application, applicationPackage, false); + tester.notifyJobCompletion(JobType.productionUsEast3, application, true); + + // Last region completes + assertEquals(1, tester.buildSystem().jobs().size()); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionEuWest1); + assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); + } + + @Test + public void parallelDeploymentCompletesOutOfOrder() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .parallel("us-east-3", "us-west-1") + .build(); + + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + + // Test environments pass + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + + // Parallel deployment + tester.deploy(DeploymentJobs.JobType.productionUsWest1, app, applicationPackage); + tester.deploy(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage); + + // Last declared job completes first + tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsWest1, app, true); + assertTrue("Change is present as not all jobs are complete", + tester.applications().require(app.id()).deploying().isPresent()); + + // All jobs complete + tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, true); + assertFalse("Change has been deployed", + tester.applications().require(app.id()).deploying().isPresent()); + } @Test public void testSuccessfulDeploymentApplicationPackageChanged() { @@ -155,13 +245,13 @@ public class DeploymentTriggerTest { tester.notifyJobCompletion(JobType.component, application, true); // Application is deployed to all test environments and declared zones - tester.deployAndNotify(JobType.systemTest, application, newApplicationPackage, true); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.systemTest); tester.deploy(JobType.stagingTest, application, previousApplicationPackage, true); - tester.deployAndNotify(JobType.stagingTest, application, newApplicationPackage, true); - tester.deployAndNotify(JobType.productionCorpUsEast1, application, newApplicationPackage, true); - tester.deployAndNotify(JobType.productionUsCentral1, application, newApplicationPackage, true); - tester.deployAndNotify(JobType.productionUsWest1, application, newApplicationPackage, true); - tester.deployAndNotify(JobType.productionApNortheast1, application, newApplicationPackage, true); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionCorpUsEast1); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsCentral1); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsWest1); + tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionApNortheast1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index f5a76f6446c..f8d09ac8b27 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -89,9 +89,9 @@ public class DeploymentIssueReporterTest { for (long i = 4; i <= 10; i++) { Application app = tester.createApplication("application" + i, "tenant" + i, 10 * i, i); tester.notifyJobCompletion(component, app, true); - tester.deployAndNotify(systemTest, app, applicationPackage, true); - tester.deployAndNotify(stagingTest, app, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, stagingTest); + tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1); } // Both the first tenants belong to the same JIRA queue. (Not sure if this is possible, but let's test it anyway. @@ -111,17 +111,17 @@ public class DeploymentIssueReporterTest { // app1 and app3 has one failure each. tester.notifyJobCompletion(component, app1, true); - tester.deployAndNotify(systemTest, app1, applicationPackage, true); - tester.deployAndNotify(stagingTest, app1, applicationPackage, false); + tester.deployAndNotify(app1, applicationPackage, true, systemTest); + tester.deployAndNotify(app1, applicationPackage, false, stagingTest); tester.notifyJobCompletion(component, app2, true); - tester.deployAndNotify(systemTest, app2, applicationPackage, true); - tester.deployAndNotify(stagingTest, app2, applicationPackage, true); + tester.deployAndNotify(app2, applicationPackage, true, systemTest); + tester.deployAndNotify(app2, applicationPackage, true, stagingTest); tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(systemTest, app3, applicationPackage, true); - tester.deployAndNotify(stagingTest, app3, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false); + tester.deployAndNotify(app3, applicationPackage, true, systemTest); + tester.deployAndNotify(app3, applicationPackage, true, stagingTest); + tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1); reporter.maintain(); reporter.maintain(); @@ -157,7 +157,7 @@ public class DeploymentIssueReporterTest { // Some time passes; tenant1 leaves her issue unattended, while tenant3 starts work and updates the issue. // app2 also has an intermittent failure; see that we detect this as a Vespa problem, and file an issue to ourselves. - tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, false); + tester.deployAndNotify(app2, applicationPackage, false, productionCorpUsEast1); tester.clock().advance(maxInactivityAge.plus(maxFailureAge)); issues.comment(openIssuesFor(app3).get(0).id(), "We are trying to fix it!"); @@ -177,8 +177,8 @@ public class DeploymentIssueReporterTest { // app3 fixes its problem, but the ticket is left open; see the resolved ticket is not escalated when another escalation period has passed. - tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, true); + tester.deployAndNotify(app2, applicationPackage, true, productionCorpUsEast1); + tester.deployAndNotify(app3, applicationPackage, true, productionCorpUsEast1); tester.clock().advance(maxInactivityAge.plus(Duration.ofDays(1))); reporter.maintain(); @@ -190,9 +190,9 @@ public class DeploymentIssueReporterTest { // app1 still does nothing with their issue; see the terminal user gets it in the end. // app3 now has a new failure past max failure age; see that a new issue is filed. tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(systemTest, app3, applicationPackage, true); - tester.deployAndNotify(stagingTest, app3, applicationPackage, true); - tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false); + tester.deployAndNotify(app3, applicationPackage, true, systemTest); + tester.deployAndNotify(app3, applicationPackage, true, stagingTest); + tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1); tester.clock().advance(maxInactivityAge.plus(maxFailureAge)); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 052ca87f791..b5ee0469e9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -41,9 +41,9 @@ public class FailureRedeployerTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); // New version is released version = Version.fromString("5.1"); @@ -52,12 +52,12 @@ public class FailureRedeployerTest { tester.upgrader().maintain(); // Test environments pass - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); // Production job fails and is retried tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3); assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size()); assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); @@ -75,11 +75,11 @@ public class FailureRedeployerTest { .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id()))); // Test environments pass - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); // Production job fails again and exhausts all immediate retries - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3); tester.buildSystem().takeJobsToRun(); tester.clock().advance(Duration.ofMinutes(10)); tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, false); @@ -92,7 +92,7 @@ public class FailureRedeployerTest { assertEquals("Job is retried", 1, tester.buildSystem().jobs().size()); // Production job finally succeeds - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures()); } @@ -108,7 +108,7 @@ public class FailureRedeployerTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); // staging-test starts, but does not complete assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); @@ -139,9 +139,9 @@ public class FailureRedeployerTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); // New version is released version = Version.fromString("5.1"); @@ -151,7 +151,7 @@ public class FailureRedeployerTest { assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); // system-test fails and exhausts all immediate retries - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest); tester.buildSystem().takeJobsToRun(); tester.clock().advance(Duration.ofMinutes(10)); tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, app, false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index a832a591217..3244307e91c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -84,7 +84,7 @@ public class MetricsReporterTest { // 1 app fails system-test tester.notifyJobCompletion(component, app4, true); - tester.deployAndNotify(systemTest, app4, applicationPackage, false); + tester.deployAndNotify(app4, applicationPackage, false, systemTest); metricsReporter.maintain(); assertEquals(25.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index e5afcec87ad..e047a288fb9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -224,9 +224,9 @@ public class UpgraderTest { Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); tester.upgrader().maintain(); assertEquals("Application is on expected version: Nothing to do", 0, @@ -239,10 +239,10 @@ public class UpgraderTest { tester.upgrader().maintain(); // system-test completes successfully - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); // staging-test fails multiple times, exhausts retries and failure is recorded - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, false); + tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.stagingTest); tester.buildSystem().takeJobsToRun(); tester.clock().advance(Duration.ofMinutes(10)); tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, app, false); @@ -282,17 +282,17 @@ public class UpgraderTest { // Application is on 5.0 Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1); // Canary in prod.corp-us-east-1 is upgraded to controller version tester.upgrader().maintain(); assertEquals("Upgrade started", 1, tester.buildSystem().jobs().size()); assertEquals(Vtag.currentVersion, ((Change.VersionChange) tester.application(app.id()).deploying().get()).version()); - tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); - tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1); // System is upgraded to newer version, no upgrade triggered for canary as version is lower than controller version = Version.fromString("5.1"); |