Support per-cloud OS upgrades

1 files changed, 40 insertions, 14 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
index 2937c2a1fed..bf9fbeb26d3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
@@ -6,12 +6,15 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 
 import java.time.Duration;
 import java.util.Optional;
 import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Maintenance job that schedules upgrades of OS / kernel on nodes in the system.
@@ -20,14 +23,19 @@ import java.util.Set;
  */
 public class OsUpgrader extends InfrastructureUpgrader {
 
+    private static final Logger log = Logger.getLogger(OsUpgrader.class.getName());
+
     private static final Set<Node.State> upgradableNodeStates = ImmutableSet.of(
             Node.State.ready,
             Node.State.active,
             Node.State.reserved
     );
 
-    public OsUpgrader(Controller controller, Duration interval, JobControl jobControl) {
-        super(controller, interval, jobControl, controller.zoneRegistry().osUpgradePolicy());
+    private final CloudName cloud;
+
+    public OsUpgrader(Controller controller, Duration interval, JobControl jobControl, CloudName cloud) {
+        super(controller, interval, jobControl, controller.zoneRegistry().osUpgradePolicy(cloud), name(cloud));
+        this.cloud = cloud;
     }
 
     @Override
@@ -38,6 +46,8 @@ public class OsUpgrader extends InfrastructureUpgrader {
 
     @Override
     protected void upgrade(Version target, SystemApplication application, ZoneId zone) {
+        log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone));
+        // Node repository ensures the upgrade call is idempotent
         application.nodeTypesWithUpgradableOs().forEach(nodeType -> controller().configServer().nodeRepository()
                                                                                 .upgradeOs(zone, nodeType, target));
     }
@@ -48,26 +58,42 @@ public class OsUpgrader extends InfrastructureUpgrader {
     }
 
     @Override
-    protected boolean requireUpgradeOf(Node node, SystemApplication application) {
-        return eligibleForUpgrade(node, application);
-    }
-
-    private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) {
-        return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion);
+    protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) {
+        return cloud.equals(zone.cloud()) && eligibleForUpgrade(node, application);
     }
 
     @Override
     protected Optional<Version> targetVersion() {
-        // Only schedule upgrades if we have nodes in the system on a lower version
-        return controller().curator().readOsTargetVersion().filter(
-                target -> controller().osVersionStatus().versions().stream()
-                                      .anyMatch(osVersion -> osVersion.version().isBefore(target))
-        );
+        // Return target if we have nodes in this cloud on a lower version
+        return controller().osVersion(cloud)
+                           .filter(target -> controller().osVersionStatus().nodeVersionsIn(cloud).stream()
+                                                         .anyMatch(node -> node.version().isBefore(target.version())))
+                           .map(OsVersion::version);
+    }
+
+    private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) {
+        return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion);
     }
 
     /** Returns whether node in application should be upgraded by this */
     public static boolean eligibleForUpgrade(Node node, SystemApplication application) {
-        return upgradableNodeStates.contains(node.state()) && application.nodeTypesWithUpgradableOs().contains(node.type());
+        return upgradableNodeStates.contains(node.state()) &&
+               application.nodeTypesWithUpgradableOs().contains(node.type());
+    }
+
+    private static String name(CloudName cloud) {
+        return capitalize(cloud.value()) + OsUpgrader.class.getSimpleName(); // Prefix maintainer name with cloud name
+    }
+
+    private static String capitalize(String s) {
+        if (s.isEmpty()) {
+            return s;
+        }
+        char firstLetter = Character.toUpperCase(s.charAt(0));
+        if (s.length() > 1) {
+            return firstLetter + s.substring(1).toLowerCase();
+        }
+        return String.valueOf(firstLetter);
     }
 
 }