Schedule upgrade to stable OS version

9 files changed, 294 insertions, 61 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
index 97f3acda67c..4a6cb10c5c2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
@@ -15,4 +15,7 @@ public interface ArtifactRepository {
     /** Returns the system application package of the given version. */
     byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version);
 
+    /** Returns the current stable OS version for the given major version */
+    StableOsVersion stableOsVersion(int major);
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java
new file mode 100644
index 00000000000..0eabb0b6b8a
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java
@@ -0,0 +1,52 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.deployment;
+
+import com.yahoo.component.Version;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * A stable OS version.
+ *
+ * @author mpolden
+ */
+public class StableOsVersion {
+
+    private final Version version;
+    private final Instant promotedAt;
+
+    public StableOsVersion(Version version, Instant promotedAt) {
+        this.version = Objects.requireNonNull(version);
+        this.promotedAt = Objects.requireNonNull(promotedAt);
+    }
+
+    /** The version number */
+    public Version version() {
+        return version;
+    }
+
+    /** Returns the time this was promoted to stable */
+    public Instant promotedAt() {
+        return promotedAt;
+    }
+
+    @Override
+    public String toString() {
+        return "os version " + version + ", promoted at " + promotedAt;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        StableOsVersion that = (StableOsVersion) o;
+        return version.equals(that.version) && promotedAt.equals(that.promotedAt);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(version, promotedAt);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 038c3ad65ab..6653d919db5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -38,6 +38,7 @@ import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 
 import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -227,6 +228,7 @@ public class Controller extends AbstractComponent {
         if (!clouds().contains(cloudName)) {
             throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system");
         }
+        Instant scheduledAt = clock.instant();
         try (Lock lock = curator.lockOsVersions()) {
             Set<OsVersionTarget> targets = new TreeSet<>(curator.readOsVersionTargets());
             if (!force && targets.stream().anyMatch(target -> target.osVersion().cloud().equals(cloudName) &&
@@ -235,7 +237,7 @@ public class Controller extends AbstractComponent {
                                                    version.toFullString());
             }
             targets.removeIf(target -> target.osVersion().cloud().equals(cloudName)); // Only allow a single target per cloud
-            targets.add(new OsVersionTarget(new OsVersion(version, cloudName), upgradeBudget));
+            targets.add(new OsVersionTarget(new OsVersion(version, cloudName), upgradeBudget, scheduledAt));
             curator.writeOsVersionTargets(targets);
             log.info("Triggered OS upgrade to " + version.toFullString() + " in cloud " +
                      cloudName.value() + ", with upgrade budget " + upgradeBudget);
@@ -287,7 +289,8 @@ public class Controller extends AbstractComponent {
         return secretStore;
     }
 
-    private Set<CloudName> clouds() {
+    /** Clouds present in this system */
+    public Set<CloudName> clouds() {
         return zoneRegistry.zones().all().zones().stream()
                            .map(ZoneApi::getCloudName)
                            .collect(Collectors.toUnmodifiableSet());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
index a02937a03e3..c8a9ef89782 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget;
 
 import java.time.Duration;
@@ -13,30 +14,18 @@ import java.time.Instant;
 import java.time.LocalDate;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
- * Automatically set the OS version target on a schedule.
- *
- * This is used in clouds where new OS versions regularly become available.
+ * Automatically schedule upgrades to the next OS version.
  *
  * @author mpolden
  */
 public class OsUpgradeScheduler extends ControllerMaintainer {
 
-    /** Trigger a new upgrade when the current target version reaches this age */
-    private static final Duration MAX_VERSION_AGE = Duration.ofDays(45);
-
-    /**
-     * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version that
-     * may not be available yet
-     */
-    private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7);
-
-    private static final DateTimeFormatter VERSION_DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd");
-
     public OsUpgradeScheduler(Controller controller, Duration interval) {
         super(controller, interval);
     }
@@ -45,47 +34,30 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
     protected double maintain() {
         Instant now = controller().clock().instant();
         if (!canTriggerAt(now)) return 1.0;
-        for (var cloud : supportedClouds()) {
-            Optional<Version> newTarget = newTargetIn(cloud, now);
-            if (newTarget.isEmpty()) continue;
-            controller().upgradeOsIn(cloud, newTarget.get(), upgradeBudget(), false);
+        for (var cloud : controller().clouds()) {
+            Release release = releaseIn(cloud);
+            upgradeTo(release, cloud, now);
         }
         return 1.0;
     }
 
-    /** Returns the new target version for given cloud, if any */
-    private Optional<Version> newTargetIn(CloudName cloud, Instant now) {
-        Optional<Version> currentTarget = controller().osVersionTarget(cloud)
-                                                      .map(OsVersionTarget::osVersion)
-                                                      .map(OsVersion::version);
-        if (currentTarget.isEmpty()) return Optional.empty();
-        if (!hasExpired(currentTarget.get())) return Optional.empty();
-        String qualifier = LocalDate.ofInstant(now.minus(AVAILABILITY_INTERVAL), ZoneOffset.UTC)
-                                    .format(VERSION_DATE_PATTERN);
-        return Optional.of(new Version(currentTarget.get().getMajor(),
-                                       currentTarget.get().getMinor(),
-                                       currentTarget.get().getMicro(),
-                                       qualifier));
-    }
+    /** Upgrade to given release in cloud */
+    private void upgradeTo(Release release, CloudName cloud, Instant now) {
+        Optional<OsVersionTarget> currentTarget = controller().osVersionTarget(cloud);
+        if (currentTarget.isEmpty()) return;
+        if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete
 
-    /** Returns whether we should upgrade from given version */
-    private boolean hasExpired(Version version) {
-        String qualifier = version.getQualifier();
-        if (!qualifier.matches("^\\d{8,}")) return false;
+        Version version = release.version(currentTarget.get(), now);
+        if (version.equals(currentTarget.get().osVersion().version())) return;
 
-        String dateString = qualifier.substring(0, 8);
-        Instant now = controller().clock().instant();
-        Instant versionDate = LocalDate.parse(dateString, VERSION_DATE_PATTERN)
-                                       .atStartOfDay(ZoneOffset.UTC)
-                                       .toInstant();
-        return versionDate.isBefore(now.minus(MAX_VERSION_AGE));
+        controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false);
     }
 
-    /** Returns the clouds where we can safely schedule OS upgrades */
-    private Set<CloudName> supportedClouds() {
-        return controller().zoneRegistry().zones().reprovisionToUpgradeOs().zones().stream()
-                           .map(ZoneApi::getCloudName)
-                           .collect(Collectors.toUnmodifiableSet());
+    private boolean upgradingToNewMajor(CloudName cloud) {
+        Set<Integer> majorVersions = controller().osVersionStatus().versionsIn(cloud).stream()
+                                                 .map(Version::getMajor)
+                                                 .collect(Collectors.toSet());
+        return majorVersions.size() > 1;
     }
 
     private boolean canTriggerAt(Instant instant) {
@@ -96,8 +68,110 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
                dayOfWeek < 5;
     }
 
-    private Duration upgradeBudget() {
-        return controller().system().isCd() ? Duration.ofHours(1) : Duration.ofDays(14);
+    private Release releaseIn(CloudName cloud) {
+        boolean useStableRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud)
+                                               .zones().isEmpty();
+        if (useStableRelease) {
+            return new StableRelease(controller().system(), controller().serviceRegistry().artifactRepository());
+        }
+        return new CalendarVersionedRelease(controller().system());
+    }
+
+    private interface Release {
+
+        /** The version number of this */
+        Version version(OsVersionTarget currentTarget, Instant now);
+
+        /** The budget to use when upgrading to this */
+        Duration upgradeBudget();
+
+    }
+
+    /** OS release based on a stable tag */
+    private static class StableRelease implements Release {
+
+        private final SystemName system;
+        private final ArtifactRepository artifactRepository;
+
+        private StableRelease(SystemName system, ArtifactRepository artifactRepository) {
+            this.system = Objects.requireNonNull(system);
+            this.artifactRepository = Objects.requireNonNull(artifactRepository);
+        }
+
+        @Override
+        public Version version(OsVersionTarget currentTarget, Instant now) {
+            StableOsVersion stableVersion = artifactRepository.stableOsVersion(currentTarget.osVersion().version().getMajor());
+            boolean cooldownPassed = stableVersion.promotedAt().isBefore(now.minus(cooldown()));
+            return cooldownPassed ? stableVersion.version() : currentTarget.osVersion().version();
+        }
+
+        @Override
+        public Duration upgradeBudget() {
+            return Duration.ZERO; // Stable releases happen in-place so no budget is required
+        }
+
+        /** The cool-down period that must pass before a stable version can be used */
+        private Duration cooldown() {
+            return system.isCd() ? Duration.ZERO : Duration.ofDays(14);
+        }
+
+    }
+
+    /** OS release based on calendar-versioning */
+    private static class CalendarVersionedRelease implements Release {
+
+        /** The time to wait before scheduling upgrade to next version */
+        private static final Duration SCHEDULING_INTERVAL = Duration.ofDays(45);
+
+        /**
+         * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version
+         * that has not been released yet. Example: Version N is the latest one and target is set to N+1. If N+1 does
+         * not exist the zone will not converge until N+1 has been released and we may end up triggering multiple
+         * rounds of upgrades.
+         */
+        private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7);
+
+        private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd");
+
+        private final SystemName system;
+
+        public CalendarVersionedRelease(SystemName system) {
+            this.system = Objects.requireNonNull(system);
+        }
+
+        @Override
+        public Version version(OsVersionTarget currentTarget, Instant now) {
+            Instant scheduledAt = currentTarget.scheduledAt();
+            if (currentTarget.scheduledAt().equals(Instant.EPOCH)) {
+                // TODO(mpolden): Remove this block after 2021-09-01. If we haven't written scheduledAt at least once,
+                //                we need to deduce the scheduled instant from the version.
+                Version version = currentTarget.osVersion().version();
+                String qualifier = version.getQualifier();
+                if (!qualifier.matches("^\\d{8,}")) throw new IllegalArgumentException("Could not parse instant from version " + version);
+
+                String dateString = qualifier.substring(0, 8);
+                scheduledAt = LocalDate.parse(dateString, CALENDAR_VERSION_PATTERN)
+                                       .atStartOfDay(ZoneOffset.UTC)
+                                       .toInstant();
+            }
+            Version currentVersion = currentTarget.osVersion().version();
+            if (scheduledAt.isBefore(now.minus(SCHEDULING_INTERVAL))) {
+                String calendarVersion = now.minus(AVAILABILITY_INTERVAL)
+                                            .atZone(ZoneOffset.UTC)
+                                            .format(CALENDAR_VERSION_PATTERN);
+                return new Version(currentVersion.getMajor(),
+                                   currentVersion.getMinor(),
+                                   currentVersion.getMicro(),
+                                   calendarVersion);
+            }
+            return currentVersion; // New version should not be scheduled yet
+        }
+
+        @Override
+        public Duration upgradeBudget() {
+            return system.isCd() ? Duration.ofHours(1) : Duration.ofDays(14);
+        }
+
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java
index 7c27533c144..4eac5a64b0c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java
@@ -5,10 +5,12 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Set;
 import java.util.TreeSet;
@@ -24,6 +26,7 @@ public class OsVersionTargetSerializer {
 
     private static final String versionsField = "versions";
     private static final String upgradeBudgetField = "upgradeBudget";
+    private static final String scheduledAtField = "scheduledAt";
 
     public OsVersionTargetSerializer(OsVersionSerializer osVersionSerializer) {
         this.osVersionSerializer = osVersionSerializer;
@@ -43,7 +46,10 @@ public class OsVersionTargetSerializer {
         array.traverse((ArrayTraverser) (i, inspector) -> {
             OsVersion osVersion = osVersionSerializer.fromSlime(inspector);
             Duration upgradeBudget = Duration.ofMillis(inspector.field(upgradeBudgetField).asLong());
-            osVersionTargets.add(new OsVersionTarget(osVersion, upgradeBudget));
+            // TODO(mpolden): Require after 2021-09-01
+            Instant scheduledAt = SlimeUtils.optionalInstant(inspector.field(scheduledAtField))
+                                            .orElse(Instant.EPOCH);
+            osVersionTargets.add(new OsVersionTarget(osVersion, upgradeBudget, scheduledAt));
         });
         return Collections.unmodifiableSet(osVersionTargets);
     }
@@ -51,6 +57,7 @@ public class OsVersionTargetSerializer {
     private void toSlime(OsVersionTarget target, Cursor object) {
         osVersionSerializer.toSlime(target.osVersion(), object);
         object.setLong(upgradeBudgetField, target.upgradeBudget().toMillis());
+        object.setLong(scheduledAtField, target.scheduledAt().toEpochMilli());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
index 35b6ac7a1a9..5dd9f4d0685 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions;
 import org.jetbrains.annotations.NotNull;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Objects;
 
 /**
@@ -22,10 +23,12 @@ public class OsVersionTarget implements Comparable<OsVersionTarget> {
 
     private final OsVersion osVersion;
     private final Duration upgradeBudget;
+    private final Instant scheduledAt;
 
-    public OsVersionTarget(OsVersion osVersion, Duration upgradeBudget) {
+    public OsVersionTarget(OsVersion osVersion, Duration upgradeBudget, Instant scheduledAt) {
         this.osVersion = Objects.requireNonNull(osVersion);
         this.upgradeBudget = Objects.requireNonNull(upgradeBudget);
+        this.scheduledAt = Objects.requireNonNull(scheduledAt);
         if (upgradeBudget.isNegative()) throw new IllegalArgumentException("upgradeBudget cannot be negative");
     }
 
@@ -39,18 +42,22 @@ public class OsVersionTarget implements Comparable<OsVersionTarget> {
         return upgradeBudget;
     }
 
+    /** Returns when this target was scheduled */
+    public Instant scheduledAt() {
+        return scheduledAt;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         OsVersionTarget that = (OsVersionTarget) o;
-        return osVersion.equals(that.osVersion) &&
-               upgradeBudget.equals(that.upgradeBudget);
+        return osVersion.equals(that.osVersion) && upgradeBudget.equals(that.upgradeBudget) && scheduledAt.equals(that.scheduledAt);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(osVersion, upgradeBudget);
+        return Objects.hash(osVersion, upgradeBudget, scheduledAt);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
index e5e4c9e46fc..1cbda9c165f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
@@ -6,15 +6,32 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion;
+
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * @author mpolden
  */
 public class ArtifactRepositoryMock extends AbstractComponent implements ArtifactRepository {
 
+    private final Map<Integer, StableOsVersion> stableOsVersions = new HashMap<>();
+
     @Override
     public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) {
         return new byte[0];
     }
 
+    @Override
+    public StableOsVersion stableOsVersion(int major) {
+        StableOsVersion version = stableOsVersions.get(major);
+        if (version == null) throw new IllegalArgumentException("No version set for major " + major);
+        return version;
+    }
+
+    public void promoteOsVersion(StableOsVersion stableOsVersion) {
+        stableOsVersions.put(stableOsVersion.version().getMajor(), stableOsVersion);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
index 7a0175845ca..18f03303eb0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
@@ -5,12 +5,16 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -21,7 +25,7 @@ import static org.junit.Assert.assertTrue;
 public class OsUpgradeSchedulerTest {
 
     @Test
-    public void maintain() {
+    public void schedule_calendar_versioned_release() {
         ControllerTester tester = new ControllerTester();
         OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
         Instant t0 = Instant.parse("2021-01-23T00:00:00.00Z"); // Outside trigger period
@@ -63,6 +67,71 @@ public class OsUpgradeSchedulerTest {
         assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version());
     }
 
+    @Test // TODO(mpolden): Remove this after 2021-09-01
+    public void schedule_calendar_versioned_without_scheduled_time() {
+        ControllerTester tester = new ControllerTester();
+        OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
+        Instant t0 = Instant.parse("2021-01-23T07:00:00.00Z"); // Inside trigger period
+        tester.clock().setInstant(t0);
+
+        CloudName cloud = CloudName.from("cloud");
+        ZoneApi zone = zone("prod.us-west-1", cloud);
+        tester.zoneRegistry().setZones(zone).reprovisionToUpgradeOsIn(zone);
+
+        // Initial run does nothing as the cloud does not have a target
+        scheduler.maintain();
+        assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty());
+
+        // Target is set
+        Version version0 = Version.fromString("7.0.0.20210123190005");
+        // Simulate setting target without scheduledAt, to force parsing scheduled time from version number
+        tester.curator().writeOsVersionTargets(Set.of(new OsVersionTarget(new OsVersion(version0, cloud),
+                                                                          Duration.ofDays(1), Instant.EPOCH)));
+
+        // Just over 45 days pass, and a new target replaces the expired one
+        Version version1 = Version.fromString("7.0.0.20210302");
+        tester.clock().advance(Duration.ofDays(45).plus(Duration.ofSeconds(1)));
+        scheduler.maintain();
+        assertEquals("New target set", version1,
+                     tester.controller().osVersionTarget(cloud).get().osVersion().version());
+    }
+
+    @Test
+    public void schedule_stable_release() {
+        ControllerTester tester = new ControllerTester();
+        OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1));
+        Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period
+        tester.clock().setInstant(t0);
+
+        // Set initial target
+        CloudName cloud = tester.controller().clouds().iterator().next();
+        Version version0 = Version.fromString("8.0");
+        tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false);
+
+        // New version is promoted to stable
+        Version version1 = Version.fromString("8.1");
+        tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version1, tester.clock().instant()));
+        scheduler.maintain();
+        assertEquals("Target is unchanged as not enough time has passed", version0,
+                     tester.controller().osVersionTarget(cloud).get().osVersion().version());
+
+        // Enough time passes since promotion of stable release
+        tester.clock().advance(Duration.ofDays(14).plus(Duration.ofSeconds(1)));
+        scheduler.maintain();
+        OsVersionTarget target = tester.controller().osVersionTarget(cloud).get();
+        assertEquals(version1, target.osVersion().version());
+        assertEquals("No budget when upgrading to stable release",
+                     Duration.ZERO, target.upgradeBudget());
+
+        // Another version is promoted, but target remains unchanged as the release hasn't aged enough
+        tester.clock().advance(Duration.ofDays(1));
+        Version version2 = Version.fromString("8.2");
+        tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version2, tester.clock().instant()));
+        scheduler.maintain();
+        assertEquals("Target is unchanged as not enough time has passed", version1,
+                     tester.controller().osVersionTarget(cloud).get().osVersion().version());
+    }
+
     private static ZoneApi zone(String id, CloudName cloud) {
         return ZoneApiMock.newBuilder().withId(id).with(cloud).build();
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java
index 6a3632ec2c9..8feaf32d9aa 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -22,8 +23,8 @@ public class OsVersionTargetSerializerTest {
     public void serialization() {
         OsVersionTargetSerializer serializer = new OsVersionTargetSerializer(new OsVersionSerializer());
         Set<OsVersionTarget> targets = ImmutableSet.of(
-                new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.defaultName()), Duration.ZERO),
-                new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.from("foo")), Duration.ofDays(1))
+                new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.defaultName()), Duration.ZERO, Instant.ofEpochMilli(123)),
+                new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.from("foo")), Duration.ofDays(1), Instant.ofEpochMilli(456))
         );
         Set<OsVersionTarget> serialized = serializer.fromSlime(serializer.toSlime(targets));
         assertEquals(targets, serialized);