// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; import static ai.vespa.validation.Validation.require; import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast2; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApSoutheast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionAwsUsEast1a; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionEuWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast2; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testEuWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * Tests a wide variety of deployment scenarios and configurations * * @author bratseth * @author mpolden * @author jonmv */ public class DeploymentTriggerTest { private final DeploymentTester tester = new DeploymentTester(); @Test void testTriggerFailing() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .upgradePolicy("default") .region("us-west-1") .build(); // Deploy completely once var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); // New version is released Version version = Version.fromString("6.3"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // staging-test fails deployment and is retried app.failDeployment(stagingTest); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size(), "Retried dead job"); app.assertRunning(stagingTest); app.runJob(stagingTest); // system-test is now the only running job -- production jobs haven't started yet, since it is unfinished. app.assertRunning(systemTest); assertEquals(1, tester.jobs().active().size()); // system-test fails and is retried app.timeOutUpgrade(systemTest); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size(), "Job is retried on failure"); app.runJob(systemTest); tester.triggerJobs(); app.assertRunning(productionUsWest1); tester.jobs().abort(tester.jobs().last(app.instanceId(), productionUsWest1).get().id(), "cancelled", true); tester.runner().run(); assertEquals(RunStatus.cancelled, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); tester.triggerJobs(); app.assertNotRunning(productionUsWest1); tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsWest1, "retry"); app.assertRunning(productionUsWest1); // invalid application is not retried tester.configServer().throwOnNextPrepare(new ConfigServerException(ErrorCode.INVALID_APPLICATION_PACKAGE, "nope", "bah")); tester.runner().run(); assertEquals(RunStatus.invalidApplication, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); tester.triggerJobs(); app.assertNotRunning(productionUsWest1); // production-us-west-1 fails, but the app loses its projectId, and the job isn't retried. app.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).triggerJobs(); tester.applications().lockApplicationOrThrow(app.application().id(), locked -> tester.applications().store(locked.withProjectId(OptionalLong.empty()))); app.timeOutConvergence(productionUsWest1); tester.triggerJobs(); assertEquals(0, tester.jobs().active().size(), "Job is not triggered when no projectId is present"); } @Test void revisionChangeWhenFailingMakesApplicationChangeWaitForPreviousToComplete() { DeploymentContext app = tester.newDeploymentContext(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .revisionChange(null) // separate by default, but we override this in test builder .region("us-east-3") .test("us-east-3") .build(); app.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); Optional v0 = app.lastSubmission(); app.submit(applicationPackage); Optional v1 = app.lastSubmission(); assertEquals(v0, app.instance().change().revision()); // Eager tests still run before new revision rolls out. app.runJob(systemTest).runJob(stagingTest); // v0 rolls out completely. app.runJob(testUsEast3); assertEquals(Optional.empty(), app.instance().change().revision()); // v1 starts rolling when v0 is done. tester.outstandingChangeDeployer().run(); assertEquals(v1, app.instance().change().revision()); // v1 fails, so v2 starts immediately. app.runJob(productionUsEast3).failDeployment(testUsEast3); app.submit(applicationPackage); Optional v2 = app.lastSubmission(); assertEquals(v2, app.instance().change().revision()); } @Test void leadingUpgradeAllowsApplicationChangeWhileUpgrading() { var applicationPackage = new ApplicationPackageBuilder().region("us-east-3") .upgradeRollout("leading") .build(); var app = tester.newDeploymentContext(); app.submit(applicationPackage).deploy(); Change upgrade = Change.of(new Version("6.8.9")); tester.controllerTester().upgradeSystem(upgrade.platform().get()); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); app.assertRunning(productionUsEast3); assertEquals(upgrade, app.instance().change()); app.submit(applicationPackage); assertEquals(upgrade.with(app.lastSubmission().get()), app.instance().change()); } @Test void abortsJobsOnNewApplicationChange() { var app = tester.newDeploymentContext(); app.submit() .runJob(systemTest) .runJob(stagingTest); tester.triggerJobs(); RunId id = tester.jobs().last(app.instanceId(), productionUsCentral1).get().id(); assertTrue(tester.jobs().active(id).isPresent()); app.submit(); assertTrue(tester.jobs().active(id).isPresent()); tester.triggerJobs(); tester.runner().run(); assertTrue(tester.jobs().active(id).isPresent()); // old run app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest); // outdated run is aborted when otherwise blocking a new run tester.triggerJobs(); app.jobAborted(productionUsCentral1); Versions outdated = tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions(); // Flesh bag re-triggers job, and _that_ is not aborted tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsCentral1, "flesh bag"); tester.triggerJobs(); app.runJob(productionUsCentral1); Versions reTriggered = tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions(); assertEquals(outdated, reTriggered); app.runJob(productionUsCentral1).runJob(productionUsWest1).runJob(productionUsEast3); assertEquals(Change.empty(), app.instance().change()); tester.controllerTester().upgradeSystem(new Version("6.9")); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest); tester.clock().advance(Duration.ofMinutes(1)); tester.triggerJobs(); // Upgrade is allowed to proceed ahead of revision change, and is not aborted. app.submit(); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); tester.runner().run(); assertEquals(Set.of(productionUsCentral1), tester.jobs().active().stream() .map(run -> run.id().type()) .collect(Collectors.toCollection(HashSet::new))); } @Test void similarDeploymentSpecsAreNotRolledOut() { ApplicationPackage firstPackage = new ApplicationPackageBuilder() .region("us-east-3") .build(); DeploymentContext app = tester.newDeploymentContext().submit(firstPackage, 5417, 0); var version = app.lastSubmission(); assertEquals(version, app.instance().change().revision()); app.runJob(systemTest) .runJob(stagingTest) .runJob(productionUsEast3); assertEquals(Change.empty(), app.instance().change()); // A similar application package is submitted. Since a new job is added, the original revision is again a target. ApplicationPackage secondPackage = new ApplicationPackageBuilder() .systemTest() .stagingTest() .region("us-east-3") .delay(Duration.ofHours(1)) .test("us-east-3") .build(); app.submit(secondPackage, 5417, 0); app.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); assertEquals(version, app.instance().change().revision()); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3); assertEquals(List.of(), tester.jobs().active()); assertEquals(Change.empty(), app.instance().change()); // The original application package is submitted again. No new jobs are added, so no change needs to roll out now. app.submit(firstPackage, 5417, 0); app.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); assertEquals(Change.empty(), app.instance().change()); } @Test void testOutstandingChangeWithNextRevisionTarget() { ApplicationPackage appPackage = new ApplicationPackageBuilder().revisionTarget("next") .revisionChange("when-failing") .region("us-east-3") .build(); DeploymentContext app = tester.newDeploymentContext() .submit(appPackage); Optional revision1 = app.lastSubmission(); app.submit(appPackage); Optional revision2 = app.lastSubmission(); app.submit(appPackage); Optional revision3 = app.lastSubmission(); app.submit(appPackage); Optional revision4 = app.lastSubmission(); app.submit(appPackage); Optional revision5 = app.lastSubmission(); // 5 revisions submitted; the first is rolling out, and the others are queued. tester.outstandingChangeDeployer().run(); assertEquals(revision1, app.instance().change().revision()); assertEquals(revision2, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The second revision is set as the target by user interaction. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(revision2.get())); tester.outstandingChangeDeployer().run(); assertEquals(revision2, app.instance().change().revision()); assertEquals(revision3, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The second revision deploys completely, and the third starts rolling out. app.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); assertEquals(revision3, app.instance().change().revision()); assertEquals(revision4, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // The third revision fails, and the fourth is chosen to replace it. app.triggerJobs().timeOutConvergence(systemTest); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); assertEquals(revision4, app.instance().change().revision()); assertEquals(revision5, app.deploymentStatus().outstandingChange(InstanceName.defaultName()).revision()); // Tests for outstanding change are relevant when current revision completes. app.runJob(systemTest).runJob(systemTest) .jobAborted(stagingTest).runJob(stagingTest).runJob(stagingTest) .runJob(productionUsEast3); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); assertEquals(revision5, app.instance().change().revision()); assertEquals(Change.empty(), app.deploymentStatus().outstandingChange(InstanceName.defaultName())); app.runJob(productionUsEast3); } @Test void deploymentSpecWithDelays() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .systemTest() .delay(Duration.ofSeconds(30)) .region("us-west-1") .delay(Duration.ofMinutes(2)) .delay(Duration.ofMinutes(2)) // Multiple delays are summed up .region("us-central-1") .delay(Duration.ofMinutes(10)) // Delays after last region are valid, but have no effect .build(); var app = tester.newDeploymentContext().submit(applicationPackage); // Test jobs pass app.runJob(systemTest); tester.clock().advance(Duration.ofSeconds(15)); app.runJob(stagingTest); tester.triggerJobs(); // No jobs have started yet, as 30 seconds have not yet passed. assertEquals(0, tester.jobs().active().size()); tester.clock().advance(Duration.ofSeconds(15)); tester.triggerJobs(); // 30 seconds after the declared test, jobs may begin. The implicit test does not affect the delay. assertEquals(1, tester.jobs().active().size()); app.assertRunning(productionUsWest1); // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress tester.clock().advance(Duration.ofMinutes(3)); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); app.assertRunning(productionUsWest1); // us-west-1 completes app.runJob(productionUsWest1); // Delayed trigger does nothing as not enough time has passed after us-west-1 completion tester.triggerJobs(); assertTrue(tester.jobs().active().isEmpty(), "No more jobs triggered at this time"); // 3 minutes pass, us-central-1 is still not triggered tester.clock().advance(Duration.ofMinutes(3)); tester.triggerJobs(); assertTrue(tester.jobs().active().isEmpty(), "No more jobs triggered at this time"); // 4 minutes pass, us-central-1 is triggered tester.clock().advance(Duration.ofMinutes(1)); tester.triggerJobs(); app.runJob(productionUsCentral1); assertTrue(tester.jobs().active().isEmpty(), "All jobs consumed"); // Delayed trigger job runs again, with nothing to trigger tester.clock().advance(Duration.ofMinutes(10)); tester.triggerJobs(); assertTrue(tester.jobs().active().isEmpty(), "All jobs consumed"); } @Test void deploymentSpecWithParallelDeployments() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-central-1") .parallel("us-west-1", "us-east-3") .region("eu-west-1") .build(); var app = tester.newDeploymentContext().submit(applicationPackage); // Test jobs pass app.runJob(systemTest).runJob(stagingTest); // Deploys in first region tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); app.runJob(productionUsCentral1); // Deploys in two regions in parallel tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app.assertRunning(productionUsEast3); app.assertRunning(productionUsWest1); app.runJob(productionUsWest1); assertEquals(1, tester.jobs().active().size()); app.assertRunning(productionUsEast3); app.runJob(productionUsEast3); // Last region completes tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); app.runJob(productionEuWest1); assertTrue(tester.jobs().active().isEmpty(), "All jobs consumed"); } @Test void testNoOtherChangesDuringSuspension() { // Application is deployed in 3 regions: ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-central-1") .parallel("us-west-1", "us-east-3") .build(); var application = tester.newDeploymentContext().submit().deploy(); // The first production zone is suspended: tester.configServer().setSuspension(application.deploymentIdIn(ZoneId.from("prod", "us-central-1")), true); // A new change needs to be pushed out, but should not go beyond the suspended zone: application.submit() .runJob(systemTest) .runJob(stagingTest) .runJob(productionUsCentral1); tester.triggerJobs(); application.assertNotRunning(productionUsEast3); application.assertNotRunning(productionUsWest1); // The zone is unsuspended so jobs start: tester.configServer().setSuspension(application.deploymentIdIn(ZoneId.from("prod", "us-central-1")), false); tester.triggerJobs(); application.runJob(productionUsWest1).runJob(productionUsEast3); assertEquals(Change.empty(), application.instance().change()); } @Test void testBlockRevisionChange() { // Tuesday, 17:30 tester.at(Instant.parse("2017-09-26T17:30:00.00Z")); Version version = Version.fromString("6.2"); tester.controllerTester().upgradeSystem(version); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .upgradePolicy("canary") // Block application version changes on tuesday in hours 18 and 19 .blockChange(true, false, "tue", "18-19", "UTC") .region("us-west-1") .region("us-central-1") .region("us-east-3") .build(); var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); tester.clock().advance(Duration.ofHours(1)); // --------------- Enter block window: 18:30 tester.triggerJobs(); assertEquals(0, tester.jobs().active().size()); app.submit(applicationPackage); assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); app.runJob(systemTest).runJob(stagingTest); tester.outstandingChangeDeployer().run(); assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); tester.triggerJobs(); assertEquals(emptyList(), tester.jobs().active()); tester.clock().advance(Duration.ofHours(2)); // ---------------- Exit block window: 20:30 tester.outstandingChangeDeployer().run(); assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); tester.triggerJobs(); // Tests already run for the blocked production job. app.assertRunning(productionUsWest1); } @Test void testCompletionOfPartOfChangeDuringBlockWindow() { // Tuesday, 17:30 tester.at(Instant.parse("2017-09-26T17:30:00.00Z")); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .blockChange(true, true, "tue", "18", "UTC") .region("us-west-1") .region("us-east-3") .build(); var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); // Application on (6.1, 1.0.1) Version v1 = Version.fromString("6.1"); // Application is mid-upgrade when block window begins, and gets an outstanding change. Version v2 = Version.fromString("6.2"); tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); app.runJob(stagingTest).runJob(systemTest); // Entering block window will keep the outstanding change in place. tester.clock().advance(Duration.ofHours(1)); app.submit(applicationPackage); app.runJob(productionUsWest1); assertEquals(1, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetRevision().number()); assertEquals(2, app.deploymentStatus().outstandingChange(app.instance().name()).revision().get().number()); tester.triggerJobs(); // Platform upgrade keeps rolling, since it began before block window, and tests for the new revision have also started. assertEquals(3, tester.jobs().active().size()); app.runJob(productionUsEast3); assertEquals(2, tester.jobs().active().size()); // Upgrade is done, and outstanding change rolls out when block window ends. assertEquals(Change.empty(), app.instance().change()); assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); app.runJob(stagingTest).runJob(systemTest); tester.clock().advance(Duration.ofHours(1)); tester.outstandingChangeDeployer().run(); assertTrue(app.instance().change().hasTargets()); assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); app.runJob(productionUsWest1).runJob(productionUsEast3); assertFalse(app.instance().change().hasTargets()); } @Test void testJobPause() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") .region("us-east-3") .build(); var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); tester.controllerTester().upgradeSystem(new Version("6.8.7")); tester.upgrader().maintain(); tester.deploymentTrigger().pauseJob(app.instanceId(), productionUsWest1, tester.clock().instant().plus(Duration.ofSeconds(1))); tester.deploymentTrigger().pauseJob(app.instanceId(), productionUsEast3, tester.clock().instant().plus(Duration.ofSeconds(3))); // us-west-1 does not trigger when paused. app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); app.assertNotRunning(productionUsWest1); // us-west-1 triggers when no longer paused, but does not retry when paused again. tester.clock().advance(Duration.ofMillis(1500)); tester.triggerJobs(); app.assertRunning(productionUsWest1); tester.deploymentTrigger().pauseJob(app.instanceId(), productionUsWest1, tester.clock().instant().plus(Duration.ofSeconds(1))); app.failDeployment(productionUsWest1); tester.triggerJobs(); app.assertNotRunning(productionUsWest1); tester.clock().advance(Duration.ofMillis(1000)); tester.triggerJobs(); app.runJob(productionUsWest1); // us-east-3 does not automatically trigger when paused, but does when forced. tester.triggerJobs(); app.assertNotRunning(productionUsEast3); tester.deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "mrTrigger", true, true, false); app.assertRunning(productionUsEast3); assertFalse(app.instance().jobPause(productionUsEast3).isPresent()); assertEquals(app.deployment(productionUsEast3.zone()).version(), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); } @Test void applicationVersionIsNotDowngraded() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-central-1") .region("eu-west-1") .build(); var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); // productionUsCentral1 fails after deployment, causing a mismatch between deployed and successful state. app.submit(applicationPackage) .runJob(systemTest) .runJob(stagingTest) .timeOutUpgrade(productionUsCentral1); RevisionId appVersion1 = app.lastSubmission().get(); assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).revision()); // Verify the application change is not removed when platform change is cancelled. tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); assertEquals(Change.of(appVersion1), app.instance().change()); // Now cancel the change as is done through the web API. tester.deploymentTrigger().cancelChange(app.instanceId(), ALL); assertEquals(Change.empty(), app.instance().change()); // A new version is released, which should now deploy the currently deployed application version to avoid downgrades. Version version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).failDeployment(productionUsCentral1); // The last job has a different target, and the tests need to run again. // These may now start, since the first job has been triggered once, and thus is verified already. app.runJob(systemTest).runJob(stagingTest); // Finally, the two production jobs complete, in order. app.runJob(productionUsCentral1).runJob(productionEuWest1); assertEquals(appVersion1, app.deployment(ZoneId.from("prod.us-central-1")).revision()); } RevisionId latestDeployed(Instance instance) { return instance.productionDeployments().values().stream() .map(Deployment::revision) .reduce((o, n) -> require(o.equals(n), n, "all versions should be equal, but got " + o + " and " + n)) .orElseThrow(() -> new AssertionError("no versions deployed")); } @Test void downgradingApplicationVersionWorks() { var app = tester.newDeploymentContext().submit().deploy(); RevisionId appVersion0 = app.lastSubmission().get(); assertEquals(appVersion0, latestDeployed(app.instance())); app.submit().deploy(); RevisionId appVersion1 = app.lastSubmission().get(); assertEquals(appVersion1, latestDeployed(app.instance())); // Downgrading application version. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0).withRevisionPin()); assertEquals(Change.of(appVersion0).withRevisionPin(), app.instance().change()); app.runJob(stagingTest) .runJob(productionUsCentral1) .runJob(productionUsEast3) .runJob(productionUsWest1); assertEquals(Change.empty().withRevisionPin(), app.instance().change()); assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone()).revision()); assertEquals(appVersion0, latestDeployed(app.instance())); tester.outstandingChangeDeployer().run(); assertEquals(Change.empty().withRevisionPin(), app.instance().change()); tester.deploymentTrigger().cancelChange(app.instanceId(), ALL); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(appVersion1), app.instance().change()); } @Test void settingANoOpChangeIsANoOp() { var app = tester.newDeploymentContext().submit(); app.deploy(); RevisionId appVersion0 = app.lastSubmission().get(); assertEquals(appVersion0, latestDeployed(app.instance())); app.submit().deploy(); RevisionId appVersion1 = app.lastSubmission().get(); assertEquals(appVersion1, latestDeployed(app.instance())); // Triggering a roll-out of an already deployed application is a no-op. assertEquals(Change.empty(), app.instance().change()); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion1)); assertEquals(Change.empty(), app.instance().change()); assertEquals(appVersion1, latestDeployed(app.instance())); } @Test void stepIsCompletePreciselyWhenItShouldBe() { var app1 = tester.newDeploymentContext("tenant1", "app1", "default"); var app2 = tester.newDeploymentContext("tenant1", "app2", "default"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-central-1") .region("eu-west-1") .build(); // System upgrades to version0 and applications deploy on that version Version version0 = Version.fromString("7.0"); tester.controllerTester().upgradeSystem(version0); app1.submit(applicationPackage).deploy(); app2.submit(applicationPackage).deploy(); // version1 is released and application1 skips upgrading to that version Version version1 = Version.fromString("7.1"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); // Deploy application2 to keep this version present in the system app2.deployPlatform(version1); tester.deploymentTrigger().cancelChange(app1.instanceId(), ALL); // version2 is released and application1 starts upgrading Version version2 = Version.fromString("7.2"); tester.controllerTester().upgradeSystem(version2); tester.upgrader().maintain(); tester.triggerJobs(); app1.jobAborted(systemTest).jobAborted(stagingTest); app1.runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); Instant triggered = app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start(); tester.clock().advance(Duration.ofHours(1)); // version2 becomes broken and upgrade targets latest non-broken tester.upgrader().overrideConfidence(version2, VespaVersion.Confidence.broken); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); // Cancel upgrades to broken version assertEquals(Change.of(version1), app1.instance().change(), "Change becomes latest non-broken version"); // version1 proceeds 'til the last job, where it fails; us-central-1 is skipped, as current change is strictly dominated by what's deployed there. app1.runJob(systemTest).runJob(stagingTest) .failDeployment(productionEuWest1); assertEquals(triggered, app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start()); // Roll out a new application version, which gives a dual change -- this should trigger us-central-1, but only as long as it hasn't yet deployed there. RevisionId revision1 = app1.lastSubmission().get(); app1.submit(applicationPackage); RevisionId revision2 = app1.lastSubmission().get(); app1.runJob(systemTest) // Tests for new revision on version2 .runJob(stagingTest) .runJob(systemTest) // Tests for new revision on version1 .runJob(stagingTest); assertEquals(Change.of(version1).with(revision2), app1.instance().change()); tester.triggerJobs(); app1.assertRunning(productionUsCentral1); assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone()).version()); assertEquals(revision1, app1.deployment(productionUsCentral1.zone()).revision()); assertTrue(triggered.isBefore(app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start())); // Change has a higher application version than what is deployed -- deployment should trigger. app1.timeOutUpgrade(productionUsCentral1); assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); assertEquals(revision2, app1.deployment(productionUsCentral1.zone()).revision()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. tester.clock().advance(Duration.ofHours(3)); // Enough time for retry tester.triggerJobs(); // Failing job is not retried as change has been deployed app1.assertNotRunning(productionUsCentral1); // Last job has a different deployment target, so tests need to run again. app1.runJob(productionEuWest1) // Upgrade completes, and revision is the only change. .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. .runJob(productionEuWest1); // Finally, west changes revision. assertEquals(Change.empty(), app1.instance().change()); assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus()); } @Test void eachParallelDeployTargetIsTested() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .parallel("eu-west-1", "us-east-3") .build(); // Application version 1 and platform version 6.1. var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); // Success in first prod zone, change cancelled between triggering and completion of eu west job. // One of the parallel zones get a deployment, but both fail their jobs. Version v1 = new Version("6.1"); Version v2 = new Version("6.2"); tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest); app.timeOutConvergence(productionEuWest1); tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); assertEquals(v2, app.deployment(productionEuWest1.zone()).version()); assertEquals(v1, app.deployment(productionUsEast3.zone()).version()); // New application version should run system and staging tests against both 6.1 and 6.2, in no particular order. app.submit(applicationPackage); tester.triggerJobs(); Version firstTested = app.instanceJobs().get(systemTest).lastTriggered().get().versions().targetPlatform(); assertEquals(firstTested, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetPlatform()); app.runJob(systemTest).runJob(stagingTest); // Test jobs for next production zone can start and run immediately. tester.triggerJobs(); assertNotEquals(firstTested, app.instanceJobs().get(systemTest).lastTriggered().get().versions().targetPlatform()); assertNotEquals(firstTested, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetPlatform()); app.runJob(systemTest).runJob(stagingTest); // Finish old run of the aborted production job. app.triggerJobs().jobAborted(productionUsEast3); // New upgrade is already tested for both jobs. // Both jobs fail again, and must be re-triggered -- this is ok, as they are both already triggered on their current targets. app.failDeployment(productionEuWest1).failDeployment(productionUsEast3) .runJob(productionEuWest1).runJob(productionUsEast3); assertFalse(app.instance().change().hasTargets()); assertEquals(2, app.instanceJobs().get(productionEuWest1).lastSuccess().get().versions().targetRevision().number()); assertEquals(2, app.instanceJobs().get(productionUsEast3).lastSuccess().get().versions().targetRevision().number()); } @Test void retriesFailingJobs() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-central-1") .build(); // Deploy completely on default application and platform versions var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); // New application change is deployed and fails in system-test for a while app.submit(applicationPackage).runJob(stagingTest).failDeployment(systemTest); // Retries immediately once app.failDeployment(systemTest); tester.triggerJobs(); app.assertRunning(systemTest); // Stops immediate retry when next triggering is considered after first failure tester.clock().advance(Duration.ofSeconds(1)); app.failDeployment(systemTest); tester.triggerJobs(); app.assertNotRunning(systemTest); // Retries after 10 minutes since previous completion, plus half the time since the first failure tester.clock().advance(Duration.ofMinutes(10).plus(Duration.ofSeconds(1))); tester.triggerJobs(); app.assertRunning(systemTest); // Retries less frequently as more time passes app.failDeployment(systemTest); tester.clock().advance(Duration.ofMinutes(15)); tester.triggerJobs(); app.assertNotRunning(systemTest); // Retries again when sufficient time has passed tester.clock().advance(Duration.ofSeconds(2)); tester.triggerJobs(); app.assertRunning(systemTest); // Still fails and is not retried app.failDeployment(systemTest); tester.triggerJobs(); app.assertNotRunning(systemTest); // Another application change is deployed and fixes system-test. Change is triggered immediately as target changes app.submit(applicationPackage).deploy(); assertTrue(tester.jobs().active().isEmpty(), "Deployment completed"); } @Test void testPlatformVersionSelection() { // Setup system ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") .build(); Version version1 = tester.controller().readSystemVersion(); var app1 = tester.newDeploymentContext(); // First deployment: An application change app1.submit(applicationPackage).deploy(); assertEquals(version1, app1.application().oldestDeployedPlatform().get(), "First deployment gets system version"); assertEquals(version1, tester.configServer().lastPrepareVersion().get()); // Application change after a new system version, and a region added Version version2 = new Version(version1.getMajor(), version1.getMinor() + 1); tester.controllerTester().upgradeSystem(version2); applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") .region("us-east-3") .build(); app1.submit(applicationPackage).deploy(); assertEquals(version1, app1.application().oldestDeployedPlatform().get(), "Application change preserves version, and new region gets oldest version too"); assertEquals(version1, tester.configServer().lastPrepareVersion().get()); assertFalse(app1.instance().change().hasTargets(), "Change deployed"); tester.upgrader().maintain(); app1.deployPlatform(version2); assertEquals(version2, app1.application().oldestDeployedPlatform().get(), "Version upgrade changes version"); assertEquals(version2, tester.configServer().lastPrepareVersion().get()); } @Test void requeueNodeAllocationFailureStagingJob() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-east-3") .build(); var app1 = tester.newDeploymentContext("tenant1", "app1", "default").submit(applicationPackage); var app2 = tester.newDeploymentContext("tenant2", "app2", "default").submit(applicationPackage); var app3 = tester.newDeploymentContext("tenant3", "app3", "default").submit(applicationPackage); // all applications: system-test completes successfully with some time in between, to determine trigger order. app2.runJob(systemTest); tester.clock().advance(Duration.ofMinutes(1)); app1.runJob(systemTest); tester.clock().advance(Duration.ofMinutes(1)); app3.runJob(systemTest); // all applications: staging test jobs queued tester.triggerJobs(); assertEquals(3, tester.jobs().active().size()); // Abort all running jobs, so we have three candidate jobs, of which only one should be triggered at a time. tester.abortAll(); assertEquals(List.of(), tester.jobs().active()); tester.readyJobsTrigger().maintain(); assertEquals(1, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); assertEquals(2, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); assertEquals(3, tester.jobs().active().size()); // Remove the jobs for app1 and app2, and then let app3 fail node allocation. // All three jobs are now eligible, but the one for app3 should trigger first as a nodeAllocationFailure-retry. app3.nodeAllocationFailure(stagingTest); app1.abortJob(stagingTest); app2.abortJob(stagingTest); tester.readyJobsTrigger().maintain(); app3.assertRunning(stagingTest); assertEquals(1, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); assertEquals(2, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); assertEquals(3, tester.jobs().active().size()); // Finish deployment for apps 2 and 3, then release a new version, leaving only app1 with an application upgrade. app2.deploy(); app3.deploy(); app1.runJob(stagingTest); assertEquals(0, tester.jobs().active().size()); tester.controllerTester().upgradeSystem(new Version("6.2")); tester.upgrader().maintain(); app1.submit(applicationPackage); // Tests for app1 trigger before the others since it carries an application upgrade. tester.readyJobsTrigger().run(); app1.assertRunning(systemTest); app1.assertRunning(stagingTest); assertEquals(2, tester.jobs().active().size()); // Let the test jobs start, remove everything except system test for app3, which fails node allocation again. tester.triggerJobs(); app3.nodeAllocationFailure(systemTest); app1.abortJob(systemTest); app1.abortJob(stagingTest); app2.abortJob(systemTest); app2.abortJob(stagingTest); app3.abortJob(stagingTest); assertEquals(0, tester.jobs().active().size()); assertTrue(app1.instance().change().revision().isPresent()); assertFalse(app2.instance().change().revision().isPresent()); assertFalse(app3.instance().change().revision().isPresent()); tester.readyJobsTrigger().maintain(); app1.assertRunning(stagingTest); app3.assertRunning(systemTest); assertEquals(2, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); app1.assertRunning(systemTest); assertEquals(4, tester.jobs().active().size()); tester.readyJobsTrigger().maintain(); app3.assertRunning(stagingTest); app2.assertRunning(stagingTest); app2.assertRunning(systemTest); assertEquals(6, tester.jobs().active().size()); } @Test void testUserInstancesNotInDeploymentSpec() { var app = tester.newDeploymentContext(); tester.controller().applications().createInstance(app.application().id().instance("user")); app.submit().deploy(); } @Test void testMultipleInstancesWithDifferentChanges() { DeploymentContext i1 = tester.newDeploymentContext("t", "a", "i1"); DeploymentContext i2 = tester.newDeploymentContext("t", "a", "i2"); DeploymentContext i3 = tester.newDeploymentContext("t", "a", "i3"); DeploymentContext i4 = tester.newDeploymentContext("t", "a", "i4"); ApplicationPackage applicationPackage = ApplicationPackageBuilder .fromDeploymentXml(""" us-east-3 us-east-3 us-east-3 us-east-3 us-east-3 """); // Package is submitted, and change propagated to the two first instances. i1.submit(applicationPackage); Optional v0 = i1.lastSubmission(); tester.outstandingChangeDeployer().run(); assertEquals(v0, i1.instance().change().revision()); assertEquals(v0, i2.instance().change().revision()); assertEquals(Optional.empty(), i3.instance().change().revision()); assertEquals(Optional.empty(), i4.instance().change().revision()); // Tests run in i4, as they're declared there, and i1 and i2 get to work i4.runJob(systemTest).runJob(stagingTest); i1.runJob(productionUsEast3); i2.runJob(productionUsEast3); // Since the post-deployment delay of i1 is incomplete, i3 doesn't yet get the change. tester.outstandingChangeDeployer().run(); assertEquals(v0, Optional.of(latestDeployed(i1.instance()))); assertEquals(v0, Optional.of(latestDeployed(i2.instance()))); assertEquals(Optional.empty(), i1.instance().change().revision()); assertEquals(Optional.empty(), i2.instance().change().revision()); assertEquals(Optional.empty(), i3.instance().change().revision()); assertEquals(Optional.empty(), i4.instance().change().revision()); // When the delay is done, i3 gets the change. tester.clock().advance(Duration.ofHours(6)); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), i1.instance().change().revision()); assertEquals(Optional.empty(), i2.instance().change().revision()); assertEquals(v0, i3.instance().change().revision()); assertEquals(Optional.empty(), i4.instance().change().revision()); // v0 begins roll-out in i3, and v1 is submitted and rolls out in i1 and i2 some time later i3.runJob(productionUsEast3); // v0 tester.clock().advance(Duration.ofHours(12)); i1.submit(applicationPackage); Optional v1 = i1.lastSubmission(); i4.runJob(systemTest).runJob(stagingTest); i1.runJob(productionUsEast3); // v1 i2.runJob(productionUsEast3); // v1 assertEquals(v1, Optional.of(latestDeployed(i1.instance()))); assertEquals(v1, Optional.of(latestDeployed(i2.instance()))); assertEquals(Optional.empty(), i1.instance().change().revision()); assertEquals(Optional.empty(), i2.instance().change().revision()); assertEquals(v0, i3.instance().change().revision()); assertEquals(Optional.empty(), i4.instance().change().revision()); // After some time, v2 also starts rolling out to i1 and i2, but does not complete in i2 tester.clock().advance(Duration.ofHours(3)); i1.submit(applicationPackage); Optional v2 = i1.lastSubmission(); i4.runJob(systemTest).runJob(stagingTest); i1.runJob(productionUsEast3); // v2 tester.clock().advance(Duration.ofHours(3)); // v1 is all done in i1 and i2, but does not yet roll out in i3; v2 is not completely rolled out there yet. tester.outstandingChangeDeployer().run(); assertEquals(v0, i3.instance().change().revision()); // i3 completes v0, which rolls out to i4; v1 is ready for i3, but v2 is not. i3.runJob(testUsEast3); assertEquals(Optional.empty(), i3.instance().change().revision()); tester.outstandingChangeDeployer().run(); assertEquals(v2, Optional.of(latestDeployed(i1.instance()))); assertEquals(v1, Optional.of(latestDeployed(i2.instance()))); assertEquals(v0, Optional.of(latestDeployed(i3.instance()))); assertEquals(Optional.empty(), i1.instance().change().revision()); assertEquals(v2, i2.instance().change().revision()); assertEquals(v1, i3.instance().change().revision()); assertEquals(v0, i4.instance().change().revision()); } @Test void testMultipleInstancesWithRevisionCatchingUpToUpgrade() { String spec = """ us-east-3 us-east-3 """; ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); DeploymentContext alpha = tester.newDeploymentContext("t", "a", "alpha"); DeploymentContext beta = tester.newDeploymentContext("t", "a", "beta"); alpha.submit(applicationPackage).deploy(); Version version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().run(); alpha.runJob(systemTest).runJob(stagingTest); assertEquals(Change.empty(), alpha.instance().change()); assertEquals(Change.empty(), beta.instance().change()); tester.upgrader().run(); assertEquals(Change.empty(), alpha.instance().change()); assertEquals(Change.of(version1), beta.instance().change()); tester.outstandingChangeDeployer().run(); beta.triggerJobs(); tester.runner().run(); tester.outstandingChangeDeployer().run(); beta.triggerJobs(); tester.outstandingChangeDeployer().run(); beta.assertRunning(productionUsEast3); beta.assertNotRunning(testUsEast3); alpha.submit(applicationPackage); Optional revision2 = alpha.lastSubmission(); assertEquals(Change.of(revision2.get()), alpha.instance().change()); assertEquals(Change.of(version1), beta.instance().change()); alpha.runJob(systemTest).runJob(stagingTest); assertEquals(Change.empty(), alpha.instance().change()); assertEquals(Change.of(version1), beta.instance().change()); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(version1).with(revision2.get()), beta.instance().change()); beta.triggerJobs(); tester.runner().run(); beta.triggerJobs(); beta.assertRunning(productionUsEast3); beta.assertNotRunning(testUsEast3); beta.runJob(productionUsEast3) .runJob(testUsEast3); assertEquals(Change.empty(), beta.instance().change()); } @Test void testMultipleInstances() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .instances("instance1,instance2") .region("us-east-3") .build(); var app = tester.newDeploymentContext("tenant1", "application1", "instance1") .submit(applicationPackage) .completeRollout(); assertEquals(2, app.application().instances().size()); assertEquals(2, app.application().productionDeployments().values().stream() .mapToInt(Collection::size) .sum()); } @Test void testDeclaredProductionTests() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-east-3") .delay(Duration.ofMinutes(1)) .test("us-east-3") .region("us-west-1") .region("us-central-1") .test("us-central-1") .test("us-west-1") .region("eu-west-1") .build(); var app = tester.newDeploymentContext().submit(applicationPackage); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); app.assertNotRunning(productionUsWest1); tester.clock().advance(Duration.ofMinutes(1)); app.runJob(testUsEast3) .runJob(productionUsWest1).runJob(productionUsCentral1) .runJob(testUsCentral1).runJob(testUsWest1) .runJob(productionEuWest1); assertEquals(Change.empty(), app.instance().change()); // Application starts upgrade, but confidence is broken after first zone. Tests won't run. Version version0 = app.application().oldestDeployedPlatform().get(); Version version1 = Version.fromString("6.7"); Version version2 = Version.fromString("6.8"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); tester.newDeploymentContext("keep", "version1", "alive").submit().deploy(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.failDeployment(testUsEast3); tester.triggerJobs(); app.assertRunning(testUsEast3); tester.upgrader().overrideConfidence(version1, VespaVersion.Confidence.broken); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); app.failDeployment(testUsEast3); app.assertNotRunning(testUsEast3); assertEquals(Change.empty(), app.instance().change()); // Application is pinned to previous version, and downgrades to that. Tests are re-run. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPlatformPin()); app.runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.failDeployment(testUsEast3); tester.clock().advance(Duration.ofMinutes(11)); // Job is cooling down after consecutive failures. app.runJob(testUsEast3); assertEquals(Change.empty().withPlatformPin(), app.instance().change()); // A new upgrade is attempted, and production tests wait for redeployment. tester.controllerTester().upgradeSystem(version2); tester.deploymentTrigger().cancelChange(app.instanceId(), ALL); tester.upgrader().overrideConfidence(version1, VespaVersion.Confidence.high); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); // App should target version2. assertEquals(Change.of(version2), app.instance().change()); // App partially upgrades to version2. app.runJob(systemTest).runJob(stagingTest); app.triggerJobs(); app.assertRunning(productionUsEast3); app.assertNotRunning(testUsEast3); app.runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.runJob(testUsEast3).runJob(productionUsWest1).triggerJobs(); app.assertRunning(productionUsCentral1); tester.runner().run(); app.triggerJobs(); app.assertNotRunning(testUsCentral1); app.assertNotRunning(testUsWest1); // Version2 gets broken, but Version1 has high confidence now, and is the new target. // Since us-east-3 is already on Version2, both deployment and tests to it should be skipped. tester.upgrader().overrideConfidence(version2, VespaVersion.Confidence.broken); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); // App should target version2. assertEquals(Change.of(version1), app.instance().change()); app.triggerJobs(); // Deployment to 6.8 already happened, so a downgrade to 6.7 won't, but production tests will still run. app.timeOutConvergence(productionUsCentral1); app.runJob(testUsCentral1).runJob(testUsWest1).runJob(productionEuWest1); assertEquals(version1, app.instance().deployments().get(ZoneId.from("prod.eu-west-1")).version()); } @Test void testDeployComplicatedDeploymentSpec() { String complicatedDeploymentSpec = """ us-west-1 us-east-3 eu-west-1 us-central-1 ap-northeast-1 ap-northeast-2 us-central-1 ap-northeast-2 ap-northeast-1 us-east-3 ap-southeast-1 us-east-3 eu-west-1 eu-west-1 eu-west-1 """; tester.atMondayMorning(); ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(complicatedDeploymentSpec); var app1 = tester.newDeploymentContext("t", "a", "instance").submit(applicationPackage); var app2 = tester.newDeploymentContext("t", "a", "other"); var app3 = tester.newDeploymentContext("t", "a", "last"); // Verify that the first submission rolls out as per the spec. tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app1.runJob(stagingTest); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); app2.runJob(systemTest); app1.runJob(productionUsWest1); tester.triggerJobs(); assertEquals(3, tester.jobs().active().size()); app1.runJob(productionUsEast3); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app2.assertNotRunning(testEuWest1); app2.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app2.runJob(testEuWest1); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofHours(1)); app1.runJob(productionUsCentral1); tester.triggerJobs(); assertEquals(4, tester.jobs().active().size()); app1.runJob(testUsCentral1); tester.triggerJobs(); assertEquals(3, tester.jobs().active().size()); app1.runJob(productionApNortheast2); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app1.runJob(productionApNortheast1); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofMinutes(30)); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofMinutes(30)); app1.runJob(testApNortheast1); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app1.runJob(testApNortheast2); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app1.runJob(testUsEast3); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); app1.runJob(productionApSoutheast1); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); app3.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); tester.atMondayMorning().clock().advance(Duration.ofDays(5)); // Inside revision block window for second, conservative instance. Version version = Version.fromString("6.2"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); assertEquals(Change.of(version), app1.instance().change()); assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); // Upgrade instance 1; upgrade rolls out first, with revision following. // The new platform won't roll out to the conservative instance until the normal one is upgraded. app1.submit(applicationPackage); assertEquals(Change.of(version).with(app1.application().revisions().last().get().id()), app1.instance().change()); // Upgrade platform. app2.runJob(systemTest); app1.runJob(stagingTest) .runJob(productionUsWest1) .runJob(productionUsEast3); // Upgrade revision tester.clock().advance(Duration.ofSeconds(1)); // Ensure we see revision as rolling after upgrade. app2.runJob(systemTest); // R app1.runJob(stagingTest) // R .runJob(productionUsWest1); // R // productionUsEast3 won't change revision before its production test has completed for the upgrade, which is one of the last jobs! tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(productionUsCentral1); app1.runJob(testUsCentral1); tester.clock().advance(Duration.ofSeconds(1)); app1.runJob(productionUsCentral1); // R app1.runJob(testUsCentral1); // R app1.runJob(productionApNortheast2); app1.runJob(productionApNortheast1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(testApNortheast1); app1.runJob(testApNortheast2); app1.runJob(productionApNortheast2); // R app1.runJob(productionApNortheast1); // R app1.runJob(testUsEast3); app1.runJob(productionApSoutheast1); tester.clock().advance(Duration.ofSeconds(1)); app1.runJob(productionUsEast3); // R tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); // R tester.clock().advance(Duration.ofMinutes(330)); app1.runJob(testApNortheast1); // R app1.runJob(testApNortheast2); // R app1.runJob(testUsEast3); // R app1.runJob(productionApSoutheast1); // R app1.runJob(stagingTest); // Tests with only the outstanding application change. app2.runJob(systemTest); // Tests with only the outstanding application change. // Confidence rises to 'high', for the new version, and instance 2 starts to upgrade. tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size(), tester.jobs().active().toString()); assertEquals(Change.empty(), app1.instance().change()); assertEquals(Change.of(version), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); app2.runJob(productionEuWest1) .failDeployment(testEuWest1); // Instance 2 failed the last job, and now exits block window, letting application change roll out with the upgrade. tester.clock().advance(Duration.ofDays(1)); // Leave block window for revisions. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); assertEquals(0, tester.jobs().active().size()); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); assertEquals(Change.empty(), app1.instance().change()); assertEquals(Change.of(version).with(app1.application().revisions().last().get().id()), app2.instance().change()); app2.runJob(productionEuWest1) .runJob(testEuWest1); assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); // Two first instances upgraded and with new revision — last instance gets both changes as well. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(version).with(app1.lastSubmission().get()), app3.instance().change()); tester.deploymentTrigger().cancelChange(app3.instanceId(), ALL); tester.outstandingChangeDeployer().run(); tester.upgrader().maintain(); assertEquals(Change.of(app1.lastSubmission().get()), app3.instance().change()); app3.runJob(productionEuWest1); tester.upgrader().maintain(); app1.runJob(stagingTest); app3.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); assertEquals(Change.empty(), app3.instance().change()); } @Test void testRevisionJoinsUpgradeWithSeparateRollout() { var appPackage = new ApplicationPackageBuilder().region("us-central-1") .region("us-east-3") .region("us-west-1") .upgradeRollout("separate") .build(); var app = tester.newDeploymentContext().submit(appPackage).deploy(); // Platform rolls through first production zone. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); tester.clock().advance(Duration.ofMinutes(1)); // Revision starts rolling, but stays behind. var revision0 = app.lastSubmission(); app.submit(appPackage); var revision1 = app.lastSubmission(); assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); // Upgrade got here first, so attempts to proceed alone, but the upgrade fails. app.triggerJobs(); assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); app.timeOutConvergence(productionUsEast3); // Revision is allowed to join. app.triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); app.runJob(productionUsEast3); // Platform and revision now proceed together. app.runJob(stagingTest); app.triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); // New upgrade fails in staging-test, and revision to fix it is submitted. var version2 = new Version("6.3"); tester.controllerTester().upgradeSystem(version2); tester.upgrader().maintain(); app.runJob(systemTest).failDeployment(stagingTest); tester.clock().advance(Duration.ofMinutes(30)); app.failDeployment(stagingTest); app.submit(appPackage); app.runJob(systemTest).runJob(stagingTest) // Tests run with combined upgrade. .runJob(productionUsCentral1) // Combined upgrade stays together. .runJob(productionUsEast3).runJob(productionUsWest1); assertEquals(Map.of(), app.deploymentStatus().jobsToRun()); assertEquals(Change.empty(), app.instance().change()); } @Test void testProductionTestBlockingDeploymentWithSeparateRollout() { var appPackage = new ApplicationPackageBuilder().region("us-east-3") .region("us-west-1") .delay(Duration.ofHours(1)) .test("us-east-3") .upgradeRollout("separate") .build(); var app = tester.newDeploymentContext().submit(appPackage) .runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(productionUsWest1); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3); assertEquals(Change.empty(), app.instance().change()); // Platform rolls through first production zone. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); // Revision starts rolling, but waits for production test to verify the upgrade. var revision0 = app.lastSubmission(); app.submit(appPackage); var revision1 = app.lastSubmission(); assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).triggerJobs(); app.assertRunning(productionUsWest1); app.assertNotRunning(productionUsEast3); // Upgrade got here first, so attempts to proceed alone, but the upgrade fails. app.triggerJobs(); assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.timeOutConvergence(productionUsWest1).triggerJobs(); // Upgrade now fails between us-east-3 deployment and test, so test is abandoned, and revision unblocked. app.assertRunning(productionUsEast3); assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); app.runJob(productionUsEast3).triggerJobs() .jobAborted(productionUsWest1).runJob(productionUsWest1); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3); assertEquals(Change.empty(), app.instance().change()); } @Test void testProductionTestNotBlockingDeploymentWithSimultaneousRollout() { var appPackage = new ApplicationPackageBuilder().region("us-east-3") .region("us-central-1") .region("us-west-1") .delay(Duration.ofHours(1)) .test("us-east-3") .test("us-west-1") .upgradeRollout("simultaneous") .build(); var app = tester.newDeploymentContext().submit(appPackage) .runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(productionUsCentral1).runJob(productionUsWest1); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3).runJob(testUsWest1); assertEquals(Change.empty(), app.instance().change()); // Platform rolls through first production zone. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); // Revision starts rolling, and causes production test to abort when it reaches deployment. var revision0 = app.lastSubmission(); app.submit(appPackage); var revision1 = app.lastSubmission(); assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).triggerJobs(); app.assertRunning(productionUsCentral1); app.assertRunning(productionUsEast3); // Revision deploys to first prod zone. app.triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); tester.clock().advance(Duration.ofSeconds(1)); app.runJob(productionUsEast3); // Revision catches up in second prod zone. app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest).triggerJobs(); app.jobAborted(productionUsCentral1).triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsCentral1).get().versions()); app.runJob(productionUsCentral1).triggerJobs(); // Revision proceeds alone in third prod zone, making test targets different for the two prod tests. assertEquals(new Versions(version0, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.runJob(productionUsWest1); app.triggerJobs(); app.assertNotRunning(testUsEast3); tester.clock().advance(Duration.ofHours(1)); // Test lets revision proceed alone, and us-west-1 is blocked until tested. app.runJob(testUsEast3).triggerJobs(); app.assertNotRunning(productionUsWest1); app.runJob(testUsWest1).runJob(productionUsWest1).runJob(testUsWest1); // Test for us-east-3 is not re-run. assertEquals(Change.empty(), app.instance().change()); } @Test void testVeryLengthyPipelineRevisions() { String lengthyDeploymentSpec = """ us-east-3 us-east-3 us-east-3 us-east-3 us-east-3 us-east-3 """; var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); var alpha = tester.newDeploymentContext("t", "a", "alpha"); var beta = tester.newDeploymentContext("t", "a", "beta"); var gamma = tester.newDeploymentContext("t", "a", "gamma"); alpha.submit(appPackage, 0).deploy(); // revision2 is submitted, and rolls through alpha. var revision1 = alpha.lastSubmission(); alpha.submit(appPackage, 3); // Risk high enough that this may roll out alone to gamma. var revision2 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); assertEquals(Optional.empty(), alpha.instance().change().revision()); // revision3 is submitted when revision2 is half-way. tester.outstandingChangeDeployer().run(); beta.runJob(productionUsEast3); alpha.submit(appPackage, 2); // Will only roll out to gamma together with the next revision. var revision3 = alpha.lastSubmission(); beta.runJob(testUsEast3); assertEquals(Optional.empty(), beta.instance().change().revision()); // revision3 is the target for alpha, beta is done, revision2 is the target for gamma. tester.outstandingChangeDeployer().run(); assertEquals(revision3, alpha.instance().change().revision()); assertEquals(Optional.empty(), beta.instance().change().revision()); assertEquals(revision2, gamma.instance().change().revision()); // revision3 rolls to beta, then a couple of new revisions are submitted to alpha, and the latter is the new target. alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), alpha.instance().change().revision()); assertEquals(revision3, beta.instance().change().revision()); // revision5 supersedes revision4 alpha.submit(appPackage, 3); var revision4 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3); alpha.submit(appPackage, 2); var revision5 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), alpha.instance().change().revision()); assertEquals(revision3, beta.instance().change().revision()); // revision6 rolls through alpha, and becomes the next target for beta, which also completes revision3. alpha.submit(appPackage, 6); var revision6 = alpha.lastSubmission(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3) .runJob(testUsEast3); beta.runJob(productionUsEast3).runJob(testUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), alpha.instance().change().revision()); assertEquals(revision6, beta.instance().change().revision()); // revision 2 fails in gamma, but this does not bring on revision 3 gamma.failDeployment(productionUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(revision2, gamma.instance().change().revision()); // revision 2 completes in gamma gamma.runJob(productionUsEast3) .runJob(testUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), alpha.instance().change().revision()); assertEquals(Optional.empty(), gamma.instance().change().revision()); // no other revisions after 3 are ready, so gamma waits // revision6 rolls through beta, and revision3 is the next target for gamma with "when-clear" change-revision, now that 6 is blocking 4 and 5 alpha.jobAborted(stagingTest).runJob(stagingTest); beta.runJob(productionUsEast3).runJob(testUsEast3); assertEquals(Optional.empty(), beta.instance().change().revision()); tester.outstandingChangeDeployer().run(); assertEquals(Optional.empty(), alpha.instance().change().revision()); assertEquals(Optional.empty(), beta.instance().change().revision()); assertEquals(revision3, gamma.instance().change().revision()); // revision4 never became ready, but 5 did, so 4 is skipped, and 3 rolls out alone instead. // revision 6 is next, once 3 is done // revision 3 completes gamma.runJob(productionUsEast3) .runJob(testUsEast3); tester.outstandingChangeDeployer().run(); assertEquals(revision6, gamma.instance().change().revision()); // revision 7 becomes ready for gamma, but must wait for the idle time of 8 hours before being deployed alpha.submit(appPackage, 1); var revision7 = alpha.lastSubmission(); alpha.deploy(); tester.outstandingChangeDeployer(); assertEquals(Change.empty(), gamma.instance().change()); assertEquals(revision6.get(), gamma.deployment(ZoneId.from("prod.us-east-3")).revision()); tester.clock().advance(Duration.ofHours(8)); tester.outstandingChangeDeployer().run(); assertEquals(revision7, gamma.instance().change().revision()); // revision 8 is has too low risk to roll out on its own, but will start rolling immediately when revision 9 is submitted gamma.deploy(); alpha.submit(appPackage, 2); var revision8 = alpha.lastSubmission(); alpha.deploy(); tester.outstandingChangeDeployer(); assertEquals(Change.empty(), gamma.instance().change()); assertEquals(revision7.get(), gamma.deployment(ZoneId.from("prod.us-east-3")).revision()); alpha.submit(appPackage, 5); tester.outstandingChangeDeployer().run(); assertEquals(revision8, gamma.instance().change().revision()); } @Test void testVeryLengthyPipelineUpgrade() { String lengthyDeploymentSpec = """ us-east-3 us-east-3 us-east-3 us-east-3 us-east-3 us-east-3 """; var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); var alpha = tester.newDeploymentContext("t", "a", "alpha"); var beta = tester.newDeploymentContext("t", "a", "beta"); var gamma = tester.newDeploymentContext("t", "a", "gamma"); alpha.submit(appPackage).deploy(); // A version releases, but when the first upgrade has gotten through alpha, beta, and gamma, a newer version has high confidence. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); var version2 = new Version("6.3"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); alpha.runJob(systemTest).runJob(stagingTest) .runJob(productionUsEast3).runJob(testUsEast3); assertEquals(Change.empty(), alpha.instance().change()); tester.upgrader().maintain(); beta.runJob(productionUsEast3); tester.controllerTester().upgradeSystem(version2); beta.runJob(testUsEast3); assertEquals(Change.empty(), beta.instance().change()); tester.upgrader().maintain(); assertEquals(Change.of(version2), alpha.instance().change()); assertEquals(Change.empty(), beta.instance().change()); assertEquals(Change.of(version1), gamma.instance().change()); } @Test void testRevisionJoinsUpgradeWithLeadingRollout() { var appPackage = new ApplicationPackageBuilder().region("us-central-1") .region("us-east-3") .region("us-west-1") .upgradeRollout("leading") .build(); var app = tester.newDeploymentContext().submit(appPackage).deploy(); // Platform rolls through first production zone. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); tester.clock().advance(Duration.ofMinutes(1)); // Revision starts rolling, and catches up. var revision0 = app.lastSubmission(); app.submit(appPackage); var revision1 = app.lastSubmission(); assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); // Upgrade got here first, and has triggered, but is now obsolete. app.triggerJobs(); assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); // Once staging tests verify the joint upgrade, the job is replaced with that. app.runJob(stagingTest); app.triggerJobs(); app.jobAborted(productionUsEast3).runJob(productionUsEast3); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); // Platform and revision now proceed together. app.triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); } @Test void testRevisionPassesUpgradeWithSimultaneousRollout() { var appPackage = new ApplicationPackageBuilder().region("us-central-1") .region("us-east-3") .region("us-west-1") .upgradeRollout("simultaneous") .build(); var app = tester.newDeploymentContext().submit(appPackage).deploy(); // Platform rolls through first production zone. var version0 = tester.controller().readSystemVersion(); var version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); tester.clock().advance(Duration.ofMinutes(1)); // Revision starts rolling, and catches up. var revision0 = app.lastSubmission(); app.submit(appPackage); var revision1 = app.lastSubmission(); assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); // Upgrade got here first, and has triggered, but is now obsolete. app.triggerJobs(); app.assertRunning(productionUsEast3); assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); // Once staging tests verify the joint upgrade, the job is replaced with that. app.runJob(systemTest).runJob(stagingTest).runJob(stagingTest); app.triggerJobs(); app.jobAborted(productionUsEast3).runJob(productionUsEast3); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); // Revision now proceeds alone. app.triggerJobs(); assertEquals(new Versions(version0, revision1.get(), Optional.of(version0), revision0), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.runJob(productionUsWest1); // Upgrade follows. app.triggerJobs(); assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision1), tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); app.runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); } @Test void mixedDirectAndPipelineJobsInProduction() { ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("us-east-3") .region("aws-us-east-1a") .build(); ControllerTester wrapped = new ControllerTester(cd); wrapped.upgradeSystem(Version.fromString("6.1")); wrapped.computeVersionStatus(); DeploymentTester tester = new DeploymentTester(wrapped); var app = tester.newDeploymentContext(); app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); app.runJob(productionUsEast3) .abortJob(stagingTest) // Complete failing run. .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. .runJob(productionAwsUsEast1a); // Manually deploy to east again, then upgrade the system. app.runJob(productionUsEast3, cdPackage); var version = new Version("6.2"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // System and staging tests both require unknown versions, and are broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); app.runJob(productionUsEast3) .triggerJobs() .jobAborted(systemTest) .jobAborted(stagingTest) .runJob(systemTest) // Run test for aws zone again. .runJob(stagingTest) // Run test for aws zone again. .runJob(productionAwsUsEast1a); // Deploy manually again, then submit new package. app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.triggerJobs().runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); app.runJob(productionUsEast3) .jobAborted(stagingTest) .runJob(stagingTest) .runJob(productionAwsUsEast1a); } @Test void testsInSeparateInstance() { String deploymentSpec = """ eu-west-1 eu-west-1 """; ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentSpec); var canary = tester.newDeploymentContext("t", "a", "canary").submit(applicationPackage); var conservative = tester.newDeploymentContext("t", "a", "default"); canary.runJob(systemTest) .runJob(stagingTest); conservative.runJob(productionEuWest1) .runJob(testEuWest1); canary.submit(applicationPackage) .runJob(systemTest) .runJob(stagingTest); tester.outstandingChangeDeployer().run(); conservative.runJob(productionEuWest1) .runJob(testEuWest1); tester.controllerTester().upgradeSystem(new Version("6.7.7")); tester.upgrader().maintain(); canary.runJob(systemTest) .runJob(stagingTest); tester.upgrader().maintain(); conservative.runJob(productionEuWest1) .runJob(testEuWest1); } @Test void testEagerTests() { var app = tester.newDeploymentContext().submit().deploy(); // Start upgrade, then receive new submission. Version version1 = new Version("6.8.9"); RevisionId build1 = app.lastSubmission().get(); tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(stagingTest); app.submit(); RevisionId build2 = app.lastSubmission().get(); assertNotEquals(build1, build2); // App now free to start system tests eagerly, for new submission. These should run assuming upgrade succeeds. tester.triggerJobs(); app.assertRunning(stagingTest); assertEquals(version1, app.instanceJobs().get(stagingTest).lastCompleted().get().versions().targetPlatform()); assertEquals(build1, app.instanceJobs().get(stagingTest).lastCompleted().get().versions().targetRevision()); assertEquals(version1, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().sourcePlatform().get()); assertEquals(build1, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().sourceRevision().get()); assertEquals(version1, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetPlatform()); assertEquals(build2, app.instanceJobs().get(stagingTest).lastTriggered().get().versions().targetRevision()); // App completes upgrade, and outstanding change is triggered. This should let relevant, running jobs finish. app.runJob(systemTest) .runJob(productionUsCentral1) .runJob(productionUsEast3) .runJob(productionUsWest1); tester.outstandingChangeDeployer().run(); assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), stagingTest).get().status()); app.runJob(stagingTest); tester.triggerJobs(); app.assertNotRunning(stagingTest); } @Test void testTriggeringOfIdleTestJobsWhenFirstDeploymentIsOnNewerVersionThanChange() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder().systemTest() .stagingTest() .region("us-east-3") .region("us-west-1") .build(); var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); var appToAvoidVersionGC = tester.newDeploymentContext("g", "c", "default").submit().deploy(); Version version2 = new Version("6.8.9"); Version version3 = new Version("6.9.10"); tester.controllerTester().upgradeSystem(version2); tester.deploymentTrigger().forceChange(appToAvoidVersionGC.instanceId(), Change.of(version2)); appToAvoidVersionGC.deployPlatform(version2); // app upgrades first zone to version3, and then the other two to version2. tester.controllerTester().upgradeSystem(version3); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version3)); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); tester.upgrader().overrideConfidence(version3, VespaVersion.Confidence.broken); tester.controllerTester().computeVersionStatus(); tester.upgrader().run(); assertEquals(Optional.of(version2), app.instance().change().platform()); app.runJob(systemTest) .runJob(productionUsEast3) .runJob(stagingTest) .runJob(productionUsWest1); assertEquals(version3, app.instanceJobs().get(productionUsEast3).lastSuccess().get().versions().targetPlatform()); assertEquals(version2, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetPlatform()); assertEquals(Map.of(), app.deploymentStatus().jobsToRun()); assertEquals(Change.empty(), app.instance().change()); assertEquals(List.of(), tester.jobs().active()); } @Test void testRetriggerQueue() { var app = tester.newDeploymentContext().submit().deploy(); app.submit(); tester.triggerJobs(); tester.deploymentTrigger().reTrigger(app.instanceId(), productionUsEast3, null); tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3")), null); tester.deploymentTrigger().reTriggerOrAddToQueue(app.deploymentIdIn(ZoneId.from("prod", "us-east-3")), null); List retriggerEntries = tester.controller().curator().readRetriggerEntries(); assertEquals(1, retriggerEntries.size()); } @Test void testOrchestrationWithIncompatibleVersionPairs() { Version version1 = new Version("7"); Version version2 = new Version("8"); Version version3 = new Version("8.1"); tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class); // App deploys on version1. tester.controllerTester().upgradeSystem(version1); DeploymentContext app = tester.newDeploymentContext() .submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version1) .build()) .deploy(); // System upgrades to version2, and then version3, but the app is not upgraded. tester.controllerTester().upgradeSystem(version2); tester.upgrader().run(); assertEquals(Change.empty(), app.instance().change()); tester.newDeploymentContext("some", "other", "app") .submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version2) .build()) .deploy(); tester.controllerTester().upgradeSystem(version3); tester.upgrader().run(); assertEquals(Change.empty(), app.instance().change()); // App compiles against version2, but confidence is broken for the version on new major before app has time to upgrade. app.submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version2) .build()); tester.upgrader().overrideConfidence(version2, Confidence.normal); tester.upgrader().overrideConfidence(version3, Confidence.broken); tester.controllerTester().computeVersionStatus(); tester.upgrader().run(); tester.outstandingChangeDeployer().run(); // App instead deploys to version2. app.deploy(); assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); // App specifies version1 in deployment spec, compiles against version1, pins to version1, and then downgrades. app.submit(new ApplicationPackageBuilder().region("us-east-3") .majorVersion(7) .compileVersion(version1) .build()); tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPlatformPin()); app.deploy(); assertEquals(version1, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version1, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); // A new app, compiled against version1, is deployed on version1. DeploymentContext newApp = tester.newDeploymentContext("new", "app", "default") .submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version1) .build()) .deploy(); assertEquals(version1, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version1, newApp.application().revisions().get(tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); // The new app enters a platform block window, and is pinned to the old platform; // the new submission overrides both those settings, as the new revision should roll out regardless. tester.atMondayMorning(); tester.deploymentTrigger().forceChange(newApp.instanceId(), Change.empty().withPlatformPin()); newApp.submit(new ApplicationPackageBuilder().compileVersion(version2) .systemTest() .blockChange(false, true, "mon", "0-23", "UTC") .region("us-east-3") .build()); RevisionId newRevision = newApp.lastSubmission().get(); assertEquals(Change.of(newRevision).with(version2), newApp.instance().change()); newApp.deploy(); assertEquals(version2, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version2, newApp.application().revisions().get(tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); // New app compiles against old major, and downgrades when a pin is also applied. newApp.submit(new ApplicationPackageBuilder().compileVersion(version1) .systemTest() .region("us-east-3") .build()); newRevision = newApp.lastSubmission().get(); assertEquals(Change.of(newRevision).with(version1), newApp.instance().change()); tester.triggerJobs(); newApp.assertNotRunning(systemTest); // Without a pin, the platform won't downgrade, and 8 is incompatible with compiled 7. tester.outstandingChangeDeployer().run(); assertEquals(Change.of(newRevision).with(version1), newApp.instance().change()); tester.upgrader().run(); assertEquals(Change.of(newRevision).with(version1), newApp.instance().change()); tester.deploymentTrigger().forceChange(newApp.instanceId(), newApp.instance().change().withPlatformPin()); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); tester.upgrader().run(); assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); newApp.deploy(); assertEquals(version1, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version1, newApp.application().revisions().get(tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); } @Test void testOutdatedMajorIsIllegal() { Version version0 = new Version("6.2"); Version version1 = new Version("7.1"); tester.controllerTester().upgradeSystem(version0); DeploymentContext old = tester.newDeploymentContext("t", "a", "default").submit() .runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); old.runJob(JobType.dev("us-east-1"), applicationPackage()); tester.controllerTester().upgradeSystem(version1); tester.upgrader().overrideConfidence(version1, Confidence.high); tester.controllerTester().computeVersionStatus(); // New app can't deploy to 6.2 DeploymentContext app = tester.newDeploymentContext("t", "b", "default"); assertEquals("platform version 6.2 is not on a current major version in this system", assertThrows(IllegalArgumentException.class, () -> tester.jobs().deploy(app.instanceId(), JobType.dev("us-east-1"), Optional.of(version0), DeploymentContext.applicationPackage())) .getMessage()); // App which already deployed to 6.2 can still do so. tester.jobs().deploy(old.instanceId(), JobType.dev("us-east-1"), Optional.of(version0), DeploymentContext.applicationPackage()); app.submit(); assertEquals("platform version 6.2 is not on a current major version in this system", assertThrows(IllegalArgumentException.class, () -> tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0), false)) .getMessage()); tester.deploymentTrigger().forceChange(old.instanceId(), Change.of(version0), false); tester.deploymentTrigger().cancelChange(old.instanceId(), ALL); // Not even version incompatibility tricks the system. tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("7"), String.class); assertEquals("compile version 6.2 is incompatible with the current major version of this system", assertThrows(IllegalArgumentException.class, () -> app.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") .compileVersion(version0) .build())) .getMessage()); // Submit new revision on old major old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") .compileVersion(version0) .build()) .deploy(); // Upgrade. old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") .compileVersion(version1) .build()) .deploy(); // And downgrade again. old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") .compileVersion(version0) .build()); assertEquals(Change.of(version0).with(old.lastSubmission().get()), old.instance().change()); // An operator can still trigger roll-out of the otherwise illegal submission. tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(app.lastSubmission().get())); assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); } @Test void operatorMayForceUnknownVersion() { Version oldVersion = Version.fromString("6"); Version currentVersion = Version.fromString("7"); tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("7"), String.class); tester.controllerTester().upgradeSystem(currentVersion); assertEquals(List.of(currentVersion), tester.controller().readVersionStatus().versions().stream().map(VespaVersion::versionNumber).toList()); DeploymentContext app = tester.newDeploymentContext(); assertEquals("compile version 6 is incompatible with the current major version of this system", assertThrows(IllegalArgumentException.class, () -> app.submit(new ApplicationPackageBuilder().region("us-east-3") .majorVersion(6) .compileVersion(oldVersion) .build())) .getMessage()); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(oldVersion).with(app.application().revisions().last().get().id()).withPlatformPin()); app.deploy(); assertEquals(oldVersion, app.deployment(ZoneId.from("prod", "us-east-3")).version()); tester.controllerTester().computeVersionStatus(); assertEquals(List.of(oldVersion, currentVersion), tester.controller().readVersionStatus().versions().stream().map(VespaVersion::versionNumber).toList()); } @Test void testInitialDeploymentPlatform() { Version version0 = tester.controllerTester().controller().readSystemVersion(); Version version1 = new Version("6.2"); Version version2 = new Version("6.3"); assertEquals(version0, tester.newDeploymentContext("t", "a1", "default").submit().deploy().application().oldestDeployedPlatform().get()); // A new version, with normal confidence, is the default for a new app. tester.controllerTester().upgradeSystem(version1); tester.upgrader().overrideConfidence(version1, Confidence.normal); tester.controllerTester().computeVersionStatus(); assertEquals(version1, tester.newDeploymentContext("t", "a2", "default").submit().deploy().application().oldestDeployedPlatform().get()); // A newer version has broken confidence, leaving the previous version as the default. tester.controllerTester().upgradeSystem(version2); tester.upgrader().overrideConfidence(version2, Confidence.broken); tester.controllerTester().computeVersionStatus(); assertEquals(version1, tester.newDeploymentContext("t", "a3", "default").submit().deploy().application().oldestDeployedPlatform().get()); DeploymentContext dev1 = tester.newDeploymentContext("t", "d1", "default"); DeploymentContext dev2 = tester.newDeploymentContext("t", "d2", "default"); assertEquals(version1, dev1.runJob(JobType.dev("us-east-1"), DeploymentContext.applicationPackage()).deployment(ZoneId.from("dev", "us-east-1")).version()); DeploymentUpgrader devUpgrader = new DeploymentUpgrader(tester.controller(), Duration.ofHours(1)); for (int i = 0; i < 24; i++) { tester.clock().advance(Duration.ofHours(1)); devUpgrader.run(); } dev1.assertNotRunning(JobType.dev("us-east-1")); // Normal confidence lets the newest version be the default again. tester.upgrader().overrideConfidence(version2, Confidence.normal); tester.controllerTester().computeVersionStatus(); assertEquals(version2, tester.newDeploymentContext("t", "a4", "default").submit().deploy().application().oldestDeployedPlatform().get()); assertEquals(version1, dev1.runJob(JobType.dev("us-east-1"), DeploymentContext.applicationPackage()).deployment(ZoneId.from("dev", "us-east-1")).version()); assertEquals(version2, dev2.runJob(JobType.dev("us-east-1"), DeploymentContext.applicationPackage()).deployment(ZoneId.from("dev", "us-east-1")).version()); for (int i = 0; i < 24; i++) { tester.clock().advance(Duration.ofHours(1)); devUpgrader.run(); } dev1.assertRunning(JobType.dev("us-east-1")); dev1.runJob(JobType.dev("us-east-1")); assertEquals(version2, dev1.deployment(ZoneId.from("dev", "us-east-1")).version()); } @Test void testInstanceWithOnlySystemTestInTwoClouds() { String spec = """ us-east-3 alpha-centauri """; RegionName alphaCentauri = RegionName.from("alpha-centauri"); ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withCloud("centauri").withSystem(tester.controller().system()); ZoneApi testAlphaCentauri = builder.with(ZoneId.from(Environment.test, alphaCentauri)).build(); ZoneApi stagingAlphaCentauri = builder.with(ZoneId.from(Environment.staging, alphaCentauri)).build(); ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(prod, alphaCentauri)).build(); tester.controllerTester().zoneRegistry().addZones(testAlphaCentauri, stagingAlphaCentauri, prodAlphaCentauri); tester.controllerTester().setRoutingMethod(tester.controllerTester().zoneRegistry().zones().all().ids(), RoutingMethod.sharedLayer4); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); DeploymentContext tests = tester.newDeploymentContext("tenant", "application", "tests"); DeploymentContext main = tester.newDeploymentContext("tenant", "application", "main"); Version version1 = new Version("6.2"); tester.controllerTester().upgradeSystem(version1); tests.submit(appPackage); JobId systemTestJob = new JobId(tests.instanceId(), systemTest); JobId stagingTestJob = new JobId(tests.instanceId(), stagingTest); JobId mainJob = new JobId(main.instanceId(), productionUsEast3); JobId centauriJob = new JobId(main.instanceId(), JobType.deploymentTo(prodAlphaCentauri.getId())); JobType centuariTest = JobType.systemTest(tester.controllerTester().zoneRegistry(), CloudName.from("centauri")); JobType centuariStaging = JobType.stagingTest(tester.controllerTester().zoneRegistry(), CloudName.from("centauri")); assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(systemTest).runJob(stagingTest).triggerJobs(); assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); tests.triggerJobs(); assertEquals(3, tester.jobs().active().size()); tests.runJob(centuariTest); tester.outstandingChangeDeployer().run(); assertEquals(2, tester.jobs().active().size()); main.assertRunning(productionUsEast3); tests.runJob(centuariStaging); main.runJob(productionUsEast3).runJob(centauriJob.type()); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); // Versions 2 and 3 become available. // Tests instance fails on 2, then updates to 3. // Version 2 should not be a target for either instance. // Version 2 should also not be possible to set as a forced target for the tests instance. Version version2 = new Version("6.3"); tester.controllerTester().upgradeSystem(version2); tester.upgrader().run(); tester.triggerJobs(); assertEquals(Change.of(version2), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size()); Version version3 = new Version("6.4"); tester.controllerTester().upgradeSystem(version3); tests.runJob(systemTest) // Success in default cloud. .failDeployment(centuariTest); // Failure in centauri cloud. tester.upgrader().run(); assertEquals(Change.of(version3), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(systemTest).runJob(centuariTest); tester.upgrader().run(); tests.runJob(stagingTest).runJob(centuariStaging); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.of(version3), main.instance().change()); assertEquals(Set.of(mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); main.runJob(productionUsEast3); main.runJob(centauriJob.type()); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); tester.deploymentTrigger().forceChange(tests.instanceId(), Change.of(version2)); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); // Revisions 2 and 3 become available. // Tests instance fails on 2, then update to 3. // Revision 2 should not be a target for either instance. // Revision 2 should also not be possible to set as a forced target for the tests instance. tests.submit(appPackage); Optional revision2 = tests.lastSubmission(); assertEquals(Change.of(revision2.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size()); tests.submit(appPackage); Optional revision3 = tests.lastSubmission(); assertEquals(Change.of(revision2.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); tests.failDeployment(systemTest); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(revision3.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(systemTest); assertEquals(Change.of(revision3.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet()); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); tests.runJob(stagingTest); assertEquals(Change.of(revision3.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(centuariTest); tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.of(revision3.get()), main.instance().change()); assertEquals(Set.of(stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(centuariStaging); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.of(revision3.get()), main.instance().change()); assertEquals(Set.of(mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); main.runJob(productionUsEast3); main.runJob(centauriJob.type()); tester.outstandingChangeDeployer().run(); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); tester.deploymentTrigger().forceChange(tests.instanceId(), Change.of(revision2.get())); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); } @Test void testInstancesWithMultipleClouds() { String spec = """ alpha-centauri us-east-3 alpha-centauri alpha-centauri us-east-3 """; RegionName alphaCentauri = RegionName.from("alpha-centauri"); ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withCloud("centauri").withSystem(tester.controller().system()); ZoneApi testAlphaCentauri = builder.with(ZoneId.from(Environment.test, alphaCentauri)).build(); ZoneApi stagingAlphaCentauri = builder.with(ZoneId.from(Environment.staging, alphaCentauri)).build(); ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(prod, alphaCentauri)).build(); tester.controllerTester().zoneRegistry().addZones(testAlphaCentauri, stagingAlphaCentauri, prodAlphaCentauri); tester.controllerTester().setRoutingMethod(tester.controllerTester().zoneRegistry().zones().all().ids(), RoutingMethod.sharedLayer4); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); DeploymentContext alpha = tester.newDeploymentContext("tenant", "application", "alpha").submit(appPackage).deploy(); DeploymentContext beta = tester.newDeploymentContext("tenant", "application", "beta"); DeploymentContext gamma = tester.newDeploymentContext("tenant", "application", "gamma"); DeploymentContext nu = tester.newDeploymentContext("tenant", "application", "nu"); DeploymentContext omega = tester.newDeploymentContext("tenant", "application", "omega"); DeploymentContext separate = tester.newDeploymentContext("tenant", "application", "separate"); DeploymentContext independent = tester.newDeploymentContext("tenant", "application", "independent"); DeploymentContext dependent = tester.newDeploymentContext("tenant", "application", "dependent"); alpha.submit(appPackage); Map> jobs = alpha.deploymentStatus().jobsToRun(); JobType centauriTest = JobType.systemTest(tester.controller().zoneRegistry(), CloudName.from("centauri")); JobType centauriStaging = JobType.stagingTest(tester.controller().zoneRegistry(), CloudName.from("centauri")); JobType centauriProd = JobType.deploymentTo(ZoneId.from(prod, alphaCentauri)); assertQueued("separate", jobs, systemTest, centauriTest); assertQueued("separate", jobs, stagingTest, centauriStaging); assertQueued("independent", jobs, systemTest, centauriTest); assertQueued("alpha", jobs, systemTest); assertQueued("beta", jobs, centauriTest); assertQueued("gamma", jobs, centauriTest); // Once alpha runs its default system test, it also runs the centauri system test, as omega depends on it. alpha.runJob(systemTest); assertQueued("alpha", alpha.deploymentStatus().jobsToRun(), centauriTest); // Run tests, and see production jobs are triggered as they are verified. for (DeploymentContext app : List.of(alpha, beta, gamma, nu, omega, separate, independent, dependent)) tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(alpha.lastSubmission().get())); // Missing separate staging test. alpha.triggerJobs().assertNotRunning(productionUsEast3); beta.runJob(centauriTest); // Missing separate centauri staging. beta.triggerJobs().assertNotRunning(centauriProd); gamma.runJob(centauriTest); // Missing alpha centauri test, and nu centauri staging. omega.triggerJobs().assertNotRunning(centauriProd); alpha.runJob(centauriTest); omega.triggerJobs().assertNotRunning(centauriProd); nu.runJob(centauriStaging); omega.triggerJobs().assertRunning(centauriProd); separate.triggerJobs().assertNotRunning(centauriProd); separate.runJob(centauriStaging); separate.triggerJobs().assertNotRunning(centauriProd); beta.triggerJobs().assertRunning(centauriProd); separate.runJob(centauriTest); separate.triggerJobs().assertRunning(centauriProd); dependent.triggerJobs().assertNotRunning(productionUsEast3); separate.runJob(systemTest).runJob(stagingTest).triggerJobs(); dependent.triggerJobs().assertRunning(productionUsEast3); alpha.triggerJobs().assertRunning(productionUsEast3); separate.runJob(centauriProd); alpha.runJob(productionUsEast3); beta.runJob(centauriProd); omega.runJob(centauriProd); dependent.runJob(productionUsEast3); independent.runJob(centauriTest).runJob(systemTest); assertEquals(Map.of(), alpha.deploymentStatus().jobsToRun()); } private static void assertQueued(String instance, Map> jobs, JobType... expected) { List queued = jobs.get(new JobId(ApplicationId.from("tenant", "application", instance), expected[0])); Set remaining = new HashSet<>(); for (JobType ex : expected) remaining.add(ex.zone()); for (DeploymentStatus.Job q : queued) if ( ! remaining.remove(q.type().zone())) fail("unexpected queued job for " + instance + ": " + q.type()); if ( ! remaining.isEmpty()) fail("expected tests for " + instance + " were not queued in : " + remaining); } @Test void testNoTests() { DeploymentContext app = tester.newDeploymentContext(); app.submit(new ApplicationPackageBuilder().systemTest().region("us-east-3").build()); // Declared tests must have run actual tests to succeed. app.failTests(systemTest, true); assertFalse(tester.jobs().last(app.instanceId(), systemTest).get().hasSucceeded()); app.failTests(stagingTest, true); assertTrue(tester.jobs().last(app.instanceId(), stagingTest).get().hasSucceeded()); } @Test void testBrokenApplication() { DeploymentContext app = tester.newDeploymentContext(); app.submit().runJob(systemTest).failDeployment(stagingTest).failDeployment(stagingTest); tester.clock().advance(Duration.ofDays(31)); tester.outstandingChangeDeployer().run(); assertEquals(OptionalLong.empty(), app.application().projectId()); app.assertNotRunning(stagingTest); tester.triggerJobs(); app.assertNotRunning(stagingTest); assertEquals(4, app.deploymentStatus().jobsToRun().size()); app.submit().runJob(systemTest).failDeployment(stagingTest); tester.clock().advance(Duration.ofDays(20)); app.submit().runJob(systemTest).failDeployment(stagingTest); tester.clock().advance(Duration.ofDays(20)); tester.outstandingChangeDeployer().run(); assertEquals(OptionalLong.of(1000), app.application().projectId()); tester.clock().advance(Duration.ofDays(20)); tester.outstandingChangeDeployer().run(); assertEquals(OptionalLong.empty(), app.application().projectId()); app.assertNotRunning(stagingTest); tester.triggerJobs(); app.assertNotRunning(stagingTest); assertEquals(4, app.deploymentStatus().jobsToRun().size()); app.submit().runJob(systemTest).runJob(stagingTest).failDeployment(productionUsCentral1); tester.clock().advance(Duration.ofDays(31)); tester.outstandingChangeDeployer().run(); assertEquals(OptionalLong.empty(), app.application().projectId()); app.assertNotRunning(productionUsCentral1); tester.triggerJobs(); app.assertNotRunning(productionUsCentral1); assertEquals(3, app.deploymentStatus().jobsToRun().size()); app.submit().runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); tester.clock().advance(Duration.ofDays(31)); tester.outstandingChangeDeployer().run(); assertEquals(OptionalLong.of(1000), app.application().projectId()); app.assertNotRunning(productionUsCentral1); tester.triggerJobs(); app.assertRunning(productionUsCentral1); } @Test void testJobNames() { ZoneRegistryMock zones = new ZoneRegistryMock(SystemName.main); List existing = new ArrayList<>(zones.zones().all().zones()); existing.add(ZoneApiMock.newBuilder().withCloud("pink-clouds").withId("test.zone").build()); zones.setZones(existing); JobType defaultSystemTest = JobType.systemTest(zones, CloudName.DEFAULT); JobType pinkSystemTest = JobType.systemTest(zones, CloudName.from("pink-clouds")); // Job name is identity, used for looking up run history, etc.. assertEquals(defaultSystemTest, pinkSystemTest); assertEquals(defaultSystemTest, JobType.systemTest(zones, null)); assertEquals(defaultSystemTest, JobType.systemTest(zones, CloudName.from("dark-clouds"))); assertEquals(defaultSystemTest, JobType.fromJobName("system-test", zones)); assertEquals(ZoneId.from("test", "us-east-1"), defaultSystemTest.zone()); assertEquals(ZoneId.from("staging", "us-east-3"), JobType.stagingTest(zones, zones.systemZone().getCloudName()).zone()); assertEquals(ZoneId.from("test", "zone"), pinkSystemTest.zone()); assertEquals(ZoneId.from("staging", "us-east-3"), JobType.stagingTest(zones, CloudName.from("pink-clouds")).zone()); assertThrows(IllegalStateException.class, JobType.systemTest(zones, null)::zone); assertThrows(IllegalStateException.class, JobType.fromJobName("system-test", zones)::zone); assertThrows(IllegalStateException.class, JobType.fromJobName("staging-test", zones)::zone); } @Test void testOrderOfTests() { String deploymentXml = """ us-east-3 us-east-3 us-west-1 """; Version version1 = new Version("7.1"); tester.controllerTester().upgradeSystem(version1); ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentXml); tester.clock().setInstant(Instant.EPOCH.plusSeconds(8 * 60 * 60)); // Thursday morning. DeploymentContext app = tester.newDeploymentContext().submit(applicationPackage); RevisionId revision1 = app.lastSubmission().get(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3).runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); tester.clock().advance(Duration.ofDays(1)); // Enter block window. app.submit(applicationPackage); assertEquals(Change.empty(), app.instance().change()); Version version2 = new Version("7.2"); RevisionId revision2 = app.lastSubmission().get(); app.runJob(systemTest).runJob(stagingTest); app.triggerJobs().assertNotRunning(productionUsEast3); tester.controllerTester().upgradeSystem(version2); tester.clock().advance(Duration.ofDays(1)); // Leave block window. tester.upgrader().run(); tester.outstandingChangeDeployer().run(); assertEquals(Change.of(revision2).with(version2), app.instance().change()); app.runJob(systemTest).runJob(stagingTest); app.runJob(productionUsEast3); app.triggerJobs(); app.assertNotRunning(productionUsEast3); // Platform upgrade should not start before test is done with revision. tester.clock().advance(Duration.ofHours(1)); app.triggerJobs(); app.assertNotRunning(productionUsEast3); // Platform upgrade should not start before test is done with revision. app.runJob(testUsEast3); app.runJob(productionUsEast3) .runJob(productionUsWest1); tester.clock().advance(Duration.ofHours(1)); app.runJob(testUsEast3); app.runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); } @Test @Disabled // For benchmarking, not a test void miniBenchmark() { String spec = """ us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 us-east-3 us-east-3 us-west-1 us-west-1 eu-west-1 eu-west-1 us-central-1 us-central-1 """; tester.newDeploymentContext("t", "a", "prod1").submit(ApplicationPackageBuilder.fromDeploymentXml(spec)).deploy(); } }