From 182091b999279c466334eafa2e010ef69f8235be Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 11 Jul 2022 14:29:16 +0200 Subject: Support estimating next change --- .../controller/maintenance/OsUpgradeScheduler.java | 75 ++++++++++++++++------ .../maintenance/OsUpgradeSchedulerTest.java | 33 +++++++--- 2 files changed, 79 insertions(+), 29 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 111931b638b..644a8c6c1ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -14,7 +14,9 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Objects; import java.util.Optional; @@ -32,23 +34,39 @@ public class OsUpgradeScheduler extends ControllerMaintainer { @Override protected double maintain() { Instant now = controller().clock().instant(); - if (!canTriggerAt(now)) return 1.0; for (var cloud : controller().clouds()) { - Release release = releaseIn(cloud); - upgradeTo(release, cloud, now); + Optional change = changeIn(cloud); + if (change.isEmpty()) continue; + if (!change.get().scheduleAt(now)) continue; + controller().upgradeOsIn(cloud, change.get().version(), change.get().upgradeBudget(), false); } return 1.0; } - /** Upgrade to given release in cloud */ - private void upgradeTo(Release release, CloudName cloud, Instant now) { + /** Returns the wanted change for given cloud, if any */ + public Optional changeIn(CloudName cloud) { Optional currentTarget = controller().osVersionTarget(cloud); - if (currentTarget.isEmpty()) return; - if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete - - Version version = release.version(currentTarget.get(), now); - if (!version.isAfter(currentTarget.get().osVersion().version())) return; - controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false); + if (currentTarget.isEmpty()) return Optional.empty(); + if (upgradingToNewMajor(cloud)) return Optional.empty(); // Skip further upgrades until major version upgrade is complete + + Release release = releaseIn(cloud); + Instant instant = controller().clock().instant(); + Version wantedVersion = release.version(currentTarget.get(), instant); + Version currentVersion = currentTarget.get().version(); + if (release instanceof CalendarVersionedRelease) { + // Estimate the next change + while (!wantedVersion.isAfter(currentVersion)) { + instant = instant.plus(Duration.ofDays(1)); + wantedVersion = release.version(currentTarget.get(), instant); + } + } else if (!wantedVersion.isAfter(currentVersion)) { + return Optional.empty(); // No change right now, and we cannot predict the next change for this kind of release + } + // Find trigger time + while (!canTriggerAt(instant)) { + instant = instant.truncatedTo(ChronoUnit.HOURS).plus(Duration.ofHours(1)); + } + return Optional.of(new Change(wantedVersion, release.upgradeBudget(), instant)); } private boolean upgradingToNewMajor(CloudName cloud) { @@ -58,23 +76,24 @@ public class OsUpgradeScheduler extends ControllerMaintainer { .count() > 1; } - private boolean canTriggerAt(Instant instant) { - int hourOfDay = instant.atZone(ZoneOffset.UTC).getHour(); - int dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek().getValue(); - // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday - int startHour = controller().system().isCd() ? 2 : 7; - return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; - } - private Release releaseIn(CloudName cloud) { boolean useTaggedRelease = controller().zoneRegistry().zones().all().reprovisionToUpgradeOs().in(cloud) - .zones().isEmpty(); + .zones().isEmpty(); if (useTaggedRelease) { return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository()); } return new CalendarVersionedRelease(controller().system()); } + private boolean canTriggerAt(Instant instant) { + ZonedDateTime dateTime = instant.atZone(ZoneOffset.UTC); + int hourOfDay = dateTime.getHour(); + int dayOfWeek = dateTime.getDayOfWeek().getValue(); + // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday + int startHour = controller().system().isCd() ? 2 : 7; + return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; + } + private interface Release { /** The version number of this */ @@ -85,6 +104,22 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } + /** OS version change, its budget and the earliest time it can be scheduled */ + public record Change(Version version, Duration upgradeBudget, Instant scheduleAt) { + + public Change { + Objects.requireNonNull(version); + Objects.requireNonNull(upgradeBudget); + Objects.requireNonNull(scheduleAt); + } + + /** Returns whether this can be scheduled at given instant */ + public boolean scheduleAt(Instant instant) { + return !instant.isBefore(scheduleAt); + } + + } + /** OS release based on a tag */ private record TaggedRelease(SystemName system, ArtifactRepository artifactRepository) implements Release { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 9268ea5ca1c..fac15cd23c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -14,9 +14,12 @@ import org.junit.jupiter.api.Test; import java.time.Duration; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -57,9 +60,9 @@ public class OsUpgradeSchedulerTest { tester.clock().advance(Duration.ofDays(30)); scheduler.maintain(); assertEquals(version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version(), - "Target is unchanged because we're outside trigger period"); - tester.clock().advance(Duration.ofHours(7)); // Put us inside the trigger period + tester.controller().osVersionTarget(cloud).get().osVersion().version(), + "Target is unchanged because we're outside trigger period"); + tester.clock().advance(Duration.ofHours(7).plusMinutes(5)); // Put us inside the trigger period scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version(), @@ -69,11 +72,19 @@ public class OsUpgradeSchedulerTest { tester.clock().advance(Duration.ofDays(2)); scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + + // Estimate next change + Optional nextChange = scheduler.changeIn(cloud); + assertTrue(nextChange.isPresent()); + assertEquals("7.0.0.20220425", nextChange.get().version().toFullString()); + assertEquals("2022-05-02T07:00:00", LocalDateTime.ofInstant(nextChange.get().scheduleAt(), ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_DATE_TIME)); } @Test void schedule_stable_release() { ControllerTester tester = new ControllerTester(); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -86,19 +97,23 @@ public class OsUpgradeSchedulerTest { Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable, tester.clock().instant())); - scheduleUpgradeAfter(Duration.ZERO, version1, tester); + scheduleUpgradeAfter(Duration.ZERO, version1, scheduler, tester); // A newer version is triggered manually Version version3 = Version.fromString("8.3"); tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); // Nothing happens in next iteration as tagged release is older than manually triggered version - scheduleUpgradeAfter(Duration.ofDays(7), version3, tester); + scheduleUpgradeAfter(Duration.ofDays(7), version3, scheduler, tester); + + // Next change cannot be estimated for tagged releases + assertTrue(scheduler.changeIn(cloud).isEmpty(), "Next change is unknown"); } @Test void schedule_latest_release_in_cd() { ControllerTester tester = new ControllerTester(SystemName.cd); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -111,10 +126,10 @@ public class OsUpgradeSchedulerTest { Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest, tester.clock().instant())); - scheduleUpgradeAfter(Duration.ZERO, version0, tester); + scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester); // Cooldown period passes and latest release is scheduled - scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); + scheduleUpgradeAfter(Duration.ofDays(1), version1, scheduler, tester); } @Test @@ -135,9 +150,9 @@ public class OsUpgradeSchedulerTest { }); } - private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { + private void scheduleUpgradeAfter(Duration duration, Version version, OsUpgradeScheduler scheduler, ControllerTester tester) { tester.clock().advance(duration); - new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); + scheduler.maintain(); CloudName cloud = tester.controller().clouds().iterator().next(); OsVersionTarget target = tester.controller().osVersionTarget(cloud).get(); assertEquals(version, target.osVersion().version()); -- cgit v1.2.3