Expand test for multiple instances with complex tests and two clouds

2 files changed, 72 insertions, 19 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index b38bdbb1eaf..f7e4feb4da1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -453,7 +453,9 @@ public class DeploymentContext {
     /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */
     private DeploymentContext runJob(JobType type, ApplicationId instance) {
         triggerJobs();
-        var job = currentRun(new JobId(instance, type)).id().job();
+        Run run = currentRun(new JobId(instance, type));
+        assertEquals(type.zone(), run.id().type().zone());
+        JobId job = run.id().job();
         doDeploy(job);
         if (job.type().isDeployment()) {
             doUpgrade(job);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 40743827217..d5f636b5294 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -45,6 +45,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import static ai.vespa.validation.Validation.require;
+import static com.yahoo.config.provision.Environment.prod;
 import static com.yahoo.config.provision.SystemName.cd;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast1;
@@ -2458,7 +2459,7 @@ public class DeploymentTriggerTest {
         ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withCloud("centauri").withSystem(tester.controller().system());
         ZoneApi testAlphaCentauri = builder.with(ZoneId.from(Environment.test, alphaCentauri)).build();
         ZoneApi stagingAlphaCentauri = builder.with(ZoneId.from(Environment.staging, alphaCentauri)).build();
-        ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(Environment.prod, alphaCentauri)).build();
+        ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(prod, alphaCentauri)).build();
 
         tester.controllerTester().zoneRegistry().addZones(testAlphaCentauri, stagingAlphaCentauri, prodAlphaCentauri);
         tester.controllerTester().setRoutingMethod(tester.controllerTester().zoneRegistry().zones().all().ids(), RoutingMethod.sharedLayer4);
@@ -2474,6 +2475,8 @@ public class DeploymentTriggerTest {
         JobId stagingTestJob = new JobId(tests.instanceId(), stagingTest);
         JobId mainJob = new JobId(main.instanceId(), productionUsEast3);
         JobId centauriJob = new JobId(main.instanceId(), JobType.deploymentTo(prodAlphaCentauri.getId()));
+        JobType centuariTest = JobType.systemTest(tester.controllerTester().zoneRegistry(), CloudName.from("centauri"));
+        JobType centuariStaging = JobType.stagingTest(tester.controllerTester().zoneRegistry(), CloudName.from("centauri"));
 
         assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet());
         tests.runJob(systemTest).runJob(stagingTest).triggerJobs();
@@ -2482,13 +2485,13 @@ public class DeploymentTriggerTest {
         tests.triggerJobs();
         assertEquals(3, tester.jobs().active().size());
 
-        tests.runJob(systemTest);
+        tests.runJob(centuariTest);
         tester.outstandingChangeDeployer().run();
 
         assertEquals(2, tester.jobs().active().size());
         main.assertRunning(productionUsEast3);
 
-        tests.runJob(stagingTest);
+        tests.runJob(centuariStaging);
         main.runJob(productionUsEast3).runJob(centauriJob.type());
 
         assertEquals(Change.empty(), tests.instance().change());
@@ -2512,16 +2515,16 @@ public class DeploymentTriggerTest {
         Version version3 = new Version("6.4");
         tester.controllerTester().upgradeSystem(version3);
         tests.runJob(systemTest)            // Success in default cloud.
-             .failDeployment(systemTest);   // Failure in centauri cloud.
+             .failDeployment(centuariTest);   // Failure in centauri cloud.
         tester.upgrader().run();
 
         assertEquals(Change.of(version3), tests.instance().change());
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet());
 
-        tests.runJob(systemTest).runJob(systemTest);
+        tests.runJob(systemTest).runJob(centuariTest);
         tester.upgrader().run();
-        tests.runJob(stagingTest).runJob(stagingTest);
+        tests.runJob(stagingTest).runJob(centuariStaging);
 
         assertEquals(Change.empty(), tests.instance().change());
         assertEquals(Change.of(version3), main.instance().change());
@@ -2579,7 +2582,7 @@ public class DeploymentTriggerTest {
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet());
 
-        tests.runJob(systemTest);
+        tests.runJob(centuariTest);
         tester.outstandingChangeDeployer().run();
         tester.outstandingChangeDeployer().run();
 
@@ -2587,7 +2590,7 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(revision3.get()), main.instance().change());
         assertEquals(Set.of(stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet());
 
-        tests.runJob(stagingTest);
+        tests.runJob(centuariStaging);
 
         assertEquals(Change.empty(), tests.instance().change());
         assertEquals(Change.of(revision3.get()), main.instance().change());
@@ -2663,19 +2666,27 @@ public class DeploymentTriggerTest {
         ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withCloud("centauri").withSystem(tester.controller().system());
         ZoneApi testAlphaCentauri = builder.with(ZoneId.from(Environment.test, alphaCentauri)).build();
         ZoneApi stagingAlphaCentauri = builder.with(ZoneId.from(Environment.staging, alphaCentauri)).build();
-        ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(Environment.prod, alphaCentauri)).build();
+        ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(prod, alphaCentauri)).build();
 
         tester.controllerTester().zoneRegistry().addZones(testAlphaCentauri, stagingAlphaCentauri, prodAlphaCentauri);
         tester.controllerTester().setRoutingMethod(tester.controllerTester().zoneRegistry().zones().all().ids(), RoutingMethod.sharedLayer4);
         tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController());
 
         ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec);
-        DeploymentContext app = tester.newDeploymentContext("tenant", "application", "alpha").submit(appPackage).deploy();
-        app.submit(appPackage);
-        Map<JobId, List<DeploymentStatus.Job>> jobs = app.deploymentStatus().jobsToRun();
+        DeploymentContext alpha = tester.newDeploymentContext("tenant", "application", "alpha").submit(appPackage).deploy();
+        DeploymentContext beta = tester.newDeploymentContext("tenant", "application", "beta");
+        DeploymentContext gamma = tester.newDeploymentContext("tenant", "application", "gamma");
+        DeploymentContext nu = tester.newDeploymentContext("tenant", "application", "nu");
+        DeploymentContext omega = tester.newDeploymentContext("tenant", "application", "omega");
+        DeploymentContext separate = tester.newDeploymentContext("tenant", "application", "separate");
+        DeploymentContext independent = tester.newDeploymentContext("tenant", "application", "independent");
+        DeploymentContext dependent = tester.newDeploymentContext("tenant", "application", "dependent");
+        alpha.submit(appPackage);
+        Map<JobId, List<DeploymentStatus.Job>> jobs = alpha.deploymentStatus().jobsToRun();
 
         JobType centauriTest = JobType.systemTest(tester.controller().zoneRegistry(), CloudName.from("centauri"));
         JobType centauriStaging = JobType.stagingTest(tester.controller().zoneRegistry(), CloudName.from("centauri"));
+        JobType centauriProd = JobType.deploymentTo(ZoneId.from(prod, alphaCentauri));
         assertQueued("separate", jobs, systemTest, centauriTest);
         assertQueued("separate", jobs, stagingTest, centauriStaging);
         assertQueued("independent", jobs, systemTest, centauriTest);
@@ -2684,8 +2695,51 @@ public class DeploymentTriggerTest {
         assertQueued("gamma", jobs, centauriTest);
 
         // Once alpha runs its default system test, it also runs the centauri system test, as omega depends on it.
-        app.runJob(systemTest);
-        assertQueued("alpha", app.deploymentStatus().jobsToRun(), centauriTest);
+        alpha.runJob(systemTest);
+        assertQueued("alpha", alpha.deploymentStatus().jobsToRun(), centauriTest);
+
+        // Run tests, and see production jobs are triggered as they are verified.
+        for (DeploymentContext app : List.of(alpha, beta, gamma, nu, omega, separate, independent, dependent))
+            tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(alpha.lastSubmission().get()));
+
+        // Missing separate staging test.
+        alpha.triggerJobs().assertNotRunning(productionUsEast3);
+
+        beta.runJob(centauriTest);
+        // Missing separate centauri staging.
+        beta.triggerJobs().assertNotRunning(centauriProd);
+
+        gamma.runJob(centauriTest);
+
+        // Missing alpha centauri test, and nu centauri staging.
+        omega.triggerJobs().assertNotRunning(centauriProd);
+        alpha.runJob(centauriTest);
+        omega.triggerJobs().assertNotRunning(centauriProd);
+        nu.runJob(centauriStaging);
+        omega.triggerJobs().assertRunning(centauriProd);
+
+        separate.triggerJobs().assertNotRunning(centauriProd);
+
+        separate.runJob(centauriStaging);
+        separate.triggerJobs().assertNotRunning(centauriProd);
+        beta.triggerJobs().assertRunning(centauriProd);
+
+        separate.runJob(centauriTest);
+        separate.triggerJobs().assertRunning(centauriProd);
+
+        dependent.triggerJobs().assertNotRunning(productionUsEast3);
+
+        separate.runJob(systemTest).runJob(stagingTest).triggerJobs();
+        dependent.triggerJobs().assertRunning(productionUsEast3);
+        alpha.triggerJobs().assertRunning(productionUsEast3);
+
+        separate.runJob(centauriProd);
+        alpha.runJob(productionUsEast3);
+        beta.runJob(centauriProd);
+        omega.runJob(centauriProd);
+        dependent.runJob(productionUsEast3);
+        independent.runJob(centauriTest).runJob(systemTest);
+        assertEquals(Map.of(), alpha.deploymentStatus().jobsToRun());
     }
 
     private static void assertQueued(String instance, Map<JobId, List<DeploymentStatus.Job>> jobs, JobType... expected) {
@@ -2788,7 +2842,7 @@ public class DeploymentTriggerTest {
     }
 
     @Test
-    void test() {
+    void testOrderOfTests() {
         String deploymentXml = """
                                <deployment version="1.0">
                                  <test/>
@@ -2802,9 +2856,6 @@ public class DeploymentTriggerTest {
                                  </prod>
                                </deployment>""";
 
-        // TODO jonmv: recreate problem where revision starts, then upgrade, while prod is blocked,
-        //             then both are tested as separate upgrades, but prod-test has them reversed.
-
         Version version1 = new Version("7.1");
         tester.controllerTester().upgradeSystem(version1);
         ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentXml);