Schedule OS upgrades automatically in supported clouds

4 files changed, 159 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index b32b4ec73fb..bc0295abca3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -51,6 +51,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader));
         maintainers.add(new JobRunner(controller, intervals.jobRunner));
         maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater));
+        maintainers.add(new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler));
         maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer));
         maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher));
         maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer()));
@@ -99,6 +100,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration jobRunner;
         private final Duration osVersionStatusUpdater;
         private final Duration osUpgrader;
+        private final Duration osUpgradeScheduler;
         private final Duration contactInformationMaintainer;
         private final Duration nameServiceDispatcher;
         private final Duration costReportMaintainer;
@@ -124,6 +126,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.jobRunner = duration(90, SECONDS);
             this.osVersionStatusUpdater = duration(2, MINUTES);
             this.osUpgrader = duration(1, MINUTES);
+            this.osUpgradeScheduler = duration(3, HOURS);
             this.contactInformationMaintainer = duration(12, HOURS);
             this.nameServiceDispatcher = duration(10, SECONDS);
             this.costReportMaintainer = duration(2, HOURS);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
new file mode 100644
index 00000000000..b96dc47009b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -0,0 +1,87 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Automatically schedule OS upgrades.
+ *
+ * This is used in clouds where new OS versions regularly become available.
+ *
+ * @author mpolden
+ */
+public class OsUpgradeScheduler extends ControllerMaintainer {
+
+    private static final Duration MAX_VERSION_AGE = Duration.ofDays(30);
+    private static final DateTimeFormatter VERSION_DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd");
+
+    public OsUpgradeScheduler(Controller controller, Duration interval) {
+        super(controller, interval);
+    }
+
+    @Override
+    protected boolean maintain() {
+        for (var cloud : supportedClouds()) {
+            Optional<Version> newTarget = newTargetIn(cloud);
+            if (newTarget.isEmpty()) continue;
+            controller().upgradeOsIn(cloud, newTarget.get(), Optional.of(upgradeBudget()), false);
+        }
+        return true;
+    }
+
+    /** Returns the new target version for given cloud, if any */
+    private Optional<Version> newTargetIn(CloudName cloud) {
+        Optional<Version> currentTarget = controller().osVersionTarget(cloud)
+                                                      .map(OsVersionTarget::osVersion)
+                                                      .map(OsVersion::version);
+
+        if (currentTarget.isEmpty()) return Optional.empty();
+        if (!hasExpired(currentTarget.get())) return Optional.empty();
+        Instant now = controller().clock().instant();
+        String qualifier = LocalDate.ofInstant(now, ZoneOffset.UTC).format(VERSION_DATE_PATTERN);
+        return Optional.of(new Version(currentTarget.get().getMajor(),
+                                       currentTarget.get().getMinor(),
+                                       currentTarget.get().getMicro(),
+                                       qualifier));
+    }
+
+    /** Returns whether we should upgrade from given version */
+    private boolean hasExpired(Version version) {
+        String qualifier = version.getQualifier();
+        if (qualifier.matches("^\\d{8,}")) {
+            String dateString = qualifier.substring(0, 8);
+            Instant now = controller().clock().instant();
+            Instant versionDate = LocalDate.parse(dateString, VERSION_DATE_PATTERN)
+                                           .atStartOfDay(ZoneOffset.UTC)
+                                           .toInstant();
+            return versionDate.isBefore(now.minus(MAX_VERSION_AGE));
+        }
+        return false;
+    }
+
+    /** Returns the clouds where we can safely schedule OS upgrades */
+    private Set<CloudName> supportedClouds() {
+        return controller().zoneRegistry().zones().reprovisionToUpgradeOs().zones().stream()
+                           .map(ZoneApi::getCloudName)
+                           .collect(Collectors.toUnmodifiableSet());
+    }
+
+    private Duration upgradeBudget() {
+        return controller().system().isCd() ? Duration.ofHours(1) : Duration.ofDays(14);
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
new file mode 100644
index 00000000000..e8cdb6ec940
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
@@ -0,0 +1,66 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class OsUpgradeSchedulerTest {
+
+    @Test
+    public void maintain() {
+        ControllerTester tester = new ControllerTester();
+        OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
+        Instant initialTime = Instant.parse("2021-01-23T00:00:00.00Z");
+        tester.clock().setInstant(initialTime);
+
+        CloudName cloud = CloudName.from("cloud");
+        ZoneApi zone = zone("prod.us-west-1", cloud);
+        tester.zoneRegistry().setZones(zone).reprovisionToUpgradeOsIn(zone);
+
+        // Initial run does nothing as the cloud does not have a target
+        scheduler.maintain();
+        assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty());
+
+        // Target is set
+        Version version0 = Version.fromString("7.0.0.20210123190005");
+        tester.controller().upgradeOsIn(cloud, version0, Optional.of(Duration.ofDays(1)), false);
+
+        // Target remains unchanged as it hasn't expired yet
+        for (var interval : List.of(Duration.ZERO, Duration.ofDays(15))) {
+            tester.clock().advance(interval);
+            scheduler.maintain();
+            assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version());
+        }
+
+        // Just over 30 days pass, and a new target replaces the expired one
+        Version version1 = Version.fromString("7.0.0.20210222");
+        tester.clock().advance(Duration.ofDays(15).plus(Duration.ofSeconds(1)));
+        scheduler.maintain();
+        assertEquals("New target set", version1, tester.controller().osVersionTarget(cloud).get().osVersion().version());
+
+        // A few days pass and target remains unchanged
+        tester.clock().advance(Duration.ofDays(2));
+        scheduler.maintain();
+        assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version());
+    }
+
+    private static ZoneApi zone(String id, CloudName cloud) {
+        return ZoneApiMock.newBuilder().withId(id).with(cloud).build();
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 14fd7abd96c..f8ae1cfefa0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -46,6 +46,9 @@
       "name": "NameServiceDispatcher"
     },
     {
+      "name": "OsUpgradeScheduler"
+    },
+    {
       "name": "OsVersionStatusUpdater"
     },
     {