Enforce upgrade order inside each zone, instead of across zones in a block

2 files changed, 23 insertions, 13 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index 9965d88c64e..d35458a5a2a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -51,10 +51,12 @@ public class SystemUpgrader extends Maintainer {
         for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) {
             boolean converged = true;
             for (SystemApplication application : applications) {
-                if (application.dependencies().stream().allMatch(dep -> convergedOn(target, dep, zones))) {
-                    deploy(zones, application, target);
+                for (ZoneId zone : zones) {
+                    if (dependenciesConvergedOn(target, application, zone)) {
+                        deploy(target, application, zone);
+                    }
+                    converged &= convergedOn(target, application, zone);
                 }
-                converged &= convergedOn(target, application, zones);
             }
             if (!converged) {
                 break;
@@ -63,17 +65,19 @@ public class SystemUpgrader extends Maintainer {
     }
 
     /** Deploy application on given version idempotently */
-    private void deploy(List<ZoneId> zones, SystemApplication application, Version target) {
-        for (ZoneId zone : zones) {
-            if (!wantedVersion(zone, application.id(), target).equals(target)) {
-                log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone));
-                controller().applications().deploy(application, zone, target);
-            }
+    private void deploy(Version target, SystemApplication application, ZoneId zone) {
+        if (!wantedVersion(zone, application.id(), target).equals(target)) {
+            log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone));
+            controller().applications().deploy(application, zone, target);
         }
     }
 
-    private boolean convergedOn(Version target, SystemApplication application, List<ZoneId> zones) {
-        return zones.stream().allMatch(zone -> currentVersion(zone, application.id(), target).equals(target));
+    private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) {
+        return currentVersion(zone, application.id(), target).equals(target);
+    }
+
+    private boolean dependenciesConvergedOn(Version target, SystemApplication application, ZoneId zone) {
+        return application.dependencies().stream().allMatch(dependency -> convergedOn(target, dependency, zone));
     }
 
     private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 3fbda16cb4a..1ec64f8d478 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -79,12 +79,18 @@ public class SystemUpgraderTest {
         assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4);
         assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4);
 
-        // zone 2 and 3: zone-config-server upgrades in parallel
+        // zone 2 and 3: zone-config-server upgrades, first in zone 2, then in zone 3
         tester.systemUpgrader().maintain();
         assertWantedVersion(SystemApplication.configServer, version2, zone2, zone3);
         assertWantedVersion(SystemApplication.configServer, version1, zone4);
         assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4);
-        completeUpgrade(SystemApplication.configServer, version2, zone2, zone3);
+        completeUpgrade(SystemApplication.configServer, version2, zone2);
+
+        // zone-application starts upgrading in zone 2, while zone-config-server completes upgrade in zone 3
+        tester.systemUpgrader().maintain();
+        assertWantedVersion(SystemApplication.zone, version2, zone2);
+        assertWantedVersion(SystemApplication.zone, version1, zone3);
+        completeUpgrade(SystemApplication.configServer, version2, zone3);
 
         // zone 2 and 3: zone-application upgrades in parallel
         tester.systemUpgrader().maintain();