Support estimating next change

2 files changed, 79 insertions, 29 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
index 111931b638b..644a8c6c1ed 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -14,7 +14,9 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -32,23 +34,39 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
     @Override
     protected double maintain() {
         Instant now = controller().clock().instant();
-        if (!canTriggerAt(now)) return 1.0;
         for (var cloud : controller().clouds()) {
-            Release release = releaseIn(cloud);
-            upgradeTo(release, cloud, now);
+            Optional<Change> change = changeIn(cloud);
+            if (change.isEmpty()) continue;
+            if (!change.get().scheduleAt(now)) continue;
+            controller().upgradeOsIn(cloud, change.get().version(), change.get().upgradeBudget(), false);
         }
         return 1.0;
     }
 
-    /** Upgrade to given release in cloud */
-    private void upgradeTo(Release release, CloudName cloud, Instant now) {
+    /** Returns the wanted change for given cloud, if any */
+    public Optional<Change> changeIn(CloudName cloud) {
         Optional<OsVersionTarget> currentTarget = controller().osVersionTarget(cloud);
-        if (currentTarget.isEmpty()) return;
-        if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete
-
-        Version version = release.version(currentTarget.get(), now);
-        if (!version.isAfter(currentTarget.get().osVersion().version())) return;
-        controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false);
+        if (currentTarget.isEmpty()) return Optional.empty();
+        if (upgradingToNewMajor(cloud)) return Optional.empty(); // Skip further upgrades until major version upgrade is complete
+
+        Release release = releaseIn(cloud);
+        Instant instant = controller().clock().instant();
+        Version wantedVersion = release.version(currentTarget.get(), instant);
+        Version currentVersion = currentTarget.get().version();
+        if (release instanceof CalendarVersionedRelease) {
+            // Estimate the next change
+            while (!wantedVersion.isAfter(currentVersion)) {
+                instant = instant.plus(Duration.ofDays(1));
+                wantedVersion = release.version(currentTarget.get(), instant);
+            }
+        } else if (!wantedVersion.isAfter(currentVersion)) {
+            return Optional.empty(); // No change right now, and we cannot predict the next change for this kind of release
+        }
+        // Find trigger time
+        while (!canTriggerAt(instant)) {
+            instant = instant.truncatedTo(ChronoUnit.HOURS).plus(Duration.ofHours(1));
+        }
+        return Optional.of(new Change(wantedVersion, release.upgradeBudget(), instant));
     }
 
     private boolean upgradingToNewMajor(CloudName cloud) {
@@ -58,23 +76,24 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
                            .count() > 1;
     }
 
-    private boolean canTriggerAt(Instant instant) {
-        int hourOfDay = instant.atZone(ZoneOffset.UTC).getHour();
-        int dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek().getValue();
-        // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday
-        int startHour = controller().system().isCd() ? 2 : 7;
-        return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5;
-    }
-
     private Release releaseIn(CloudName cloud) {
         boolean useTaggedRelease = controller().zoneRegistry().zones().all().reprovisionToUpgradeOs().in(cloud)
-                                               .zones().isEmpty();
+                                             .zones().isEmpty();
         if (useTaggedRelease) {
             return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository());
         }
         return new CalendarVersionedRelease(controller().system());
     }
 
+    private boolean canTriggerAt(Instant instant) {
+        ZonedDateTime dateTime = instant.atZone(ZoneOffset.UTC);
+        int hourOfDay = dateTime.getHour();
+        int dayOfWeek = dateTime.getDayOfWeek().getValue();
+        // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday
+        int startHour = controller().system().isCd() ? 2 : 7;
+        return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5;
+    }
+
     private interface Release {
 
         /** The version number of this */
@@ -85,6 +104,22 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
 
     }
 
+    /** OS version change, its budget and the earliest time it can be scheduled */
+    public record Change(Version version, Duration upgradeBudget, Instant scheduleAt) {
+
+        public Change {
+            Objects.requireNonNull(version);
+            Objects.requireNonNull(upgradeBudget);
+            Objects.requireNonNull(scheduleAt);
+        }
+
+        /** Returns whether this can be scheduled at given instant */
+        public boolean scheduleAt(Instant instant) {
+            return !instant.isBefore(scheduleAt);
+        }
+
+    }
+
     /** OS release based on a tag */
     private record TaggedRelease(SystemName system, ArtifactRepository artifactRepository) implements Release {
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
index 9268ea5ca1c..fac15cd23c4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
@@ -14,9 +14,12 @@ import org.junit.jupiter.api.Test;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalDateTime;
 import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -57,9 +60,9 @@ public class OsUpgradeSchedulerTest {
         tester.clock().advance(Duration.ofDays(30));
         scheduler.maintain();
         assertEquals(version0,
-                tester.controller().osVersionTarget(cloud).get().osVersion().version(),
-                "Target is unchanged because we're outside trigger period");
-        tester.clock().advance(Duration.ofHours(7)); // Put us inside the trigger period
+                     tester.controller().osVersionTarget(cloud).get().osVersion().version(),
+                     "Target is unchanged because we're outside trigger period");
+        tester.clock().advance(Duration.ofHours(7).plusMinutes(5)); // Put us inside the trigger period
         scheduler.maintain();
         assertEquals(version1,
                 tester.controller().osVersionTarget(cloud).get().osVersion().version(),
@@ -69,11 +72,19 @@ public class OsUpgradeSchedulerTest {
         tester.clock().advance(Duration.ofDays(2));
         scheduler.maintain();
         assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version());
+
+        // Estimate next change
+        Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud);
+        assertTrue(nextChange.isPresent());
+        assertEquals("7.0.0.20220425", nextChange.get().version().toFullString());
+        assertEquals("2022-05-02T07:00:00", LocalDateTime.ofInstant(nextChange.get().scheduleAt(), ZoneOffset.UTC)
+                                                         .format(DateTimeFormatter.ISO_DATE_TIME));
     }
 
     @Test
     void schedule_stable_release() {
         ControllerTester tester = new ControllerTester();
+        OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
         Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period
         tester.clock().setInstant(t0);
 
@@ -86,19 +97,23 @@ public class OsUpgradeSchedulerTest {
         Version version1 = Version.fromString("8.1");
         tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable,
                 tester.clock().instant()));
-        scheduleUpgradeAfter(Duration.ZERO, version1, tester);
+        scheduleUpgradeAfter(Duration.ZERO, version1, scheduler, tester);
 
         // A newer version is triggered manually
         Version version3 = Version.fromString("8.3");
         tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false);
 
         // Nothing happens in next iteration as tagged release is older than manually triggered version
-        scheduleUpgradeAfter(Duration.ofDays(7), version3, tester);
+        scheduleUpgradeAfter(Duration.ofDays(7), version3, scheduler, tester);
+
+        // Next change cannot be estimated for tagged releases
+        assertTrue(scheduler.changeIn(cloud).isEmpty(), "Next change is unknown");
     }
 
     @Test
     void schedule_latest_release_in_cd() {
         ControllerTester tester = new ControllerTester(SystemName.cd);
+        OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
         Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period
         tester.clock().setInstant(t0);
 
@@ -111,10 +126,10 @@ public class OsUpgradeSchedulerTest {
         Version version1 = Version.fromString("8.1");
         tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest,
                 tester.clock().instant()));
-        scheduleUpgradeAfter(Duration.ZERO, version0, tester);
+        scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester);
 
         // Cooldown period passes and latest release is scheduled
-        scheduleUpgradeAfter(Duration.ofDays(1), version1, tester);
+        scheduleUpgradeAfter(Duration.ofDays(1), version1, scheduler, tester);
     }
 
     @Test
@@ -135,9 +150,9 @@ public class OsUpgradeSchedulerTest {
         });
     }
 
-    private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) {
+    private void scheduleUpgradeAfter(Duration duration, Version version, OsUpgradeScheduler scheduler, ControllerTester tester) {
         tester.clock().advance(duration);
-        new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain();
+        scheduler.maintain();
         CloudName cloud = tester.controller().clouds().iterator().next();
         OsVersionTarget target = tester.controller().osVersionTarget(cloud).get();
         assertEquals(version, target.osVersion().version());