diff options
Diffstat (limited to 'controller-server/src')
5 files changed, 169 insertions, 0 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index e12c1903426..abc0784396c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -235,6 +235,8 @@ public class Controller extends AbstractComponent { targets.removeIf(target -> target.osVersion().cloud().equals(cloudName)); // Only allow a single target per cloud targets.add(new OsVersionTarget(new OsVersion(version, cloudName), upgradeBudget)); curator.writeOsVersionTargets(targets); + log.info("Triggered OS upgrade to " + version.toFullString() + " in cloud " + + cloudName.value() + upgradeBudget.map(b -> ", with upgrade budget " + b).orElse("")); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b32b4ec73fb..bc0295abca3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -51,6 +51,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); + maintainers.add(new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler)); maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); @@ -99,6 +100,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration jobRunner; private final Duration osVersionStatusUpdater; private final Duration osUpgrader; + private final Duration osUpgradeScheduler; private final Duration contactInformationMaintainer; private final Duration nameServiceDispatcher; private final Duration costReportMaintainer; @@ -124,6 +126,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobRunner = duration(90, SECONDS); this.osVersionStatusUpdater = duration(2, MINUTES); this.osUpgrader = duration(1, MINUTES); + this.osUpgradeScheduler = duration(3, HOURS); this.contactInformationMaintainer = duration(12, HOURS); this.nameServiceDispatcher = duration(10, SECONDS); this.costReportMaintainer = duration(2, HOURS); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java new file mode 100644 index 00000000000..cb6d95a3300 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -0,0 +1,95 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Automatically schedule OS upgrades. + * + * This is used in clouds where new OS versions regularly become available. + * + * @author mpolden + */ +public class OsUpgradeScheduler extends ControllerMaintainer { + + /** Trigger a new upgrade when the current target version reaches this age */ + private static final Duration MAX_VERSION_AGE = Duration.ofDays(30); + + /** + * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version that + * may not be available yet + */ + private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7); + + private static final DateTimeFormatter VERSION_DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public OsUpgradeScheduler(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected boolean maintain() { + for (var cloud : supportedClouds()) { + Optional<Version> newTarget = newTargetIn(cloud); + if (newTarget.isEmpty()) continue; + controller().upgradeOsIn(cloud, newTarget.get(), Optional.of(upgradeBudget()), false); + } + return true; + } + + /** Returns the new target version for given cloud, if any */ + private Optional<Version> newTargetIn(CloudName cloud) { + Optional<Version> currentTarget = controller().osVersionTarget(cloud) + .map(OsVersionTarget::osVersion) + .map(OsVersion::version); + if (currentTarget.isEmpty()) return Optional.empty(); + if (!hasExpired(currentTarget.get())) return Optional.empty(); + + Instant now = controller().clock().instant(); + String qualifier = LocalDate.ofInstant(now.minus(AVAILABILITY_INTERVAL), ZoneOffset.UTC) + .format(VERSION_DATE_PATTERN); + return Optional.of(new Version(currentTarget.get().getMajor(), + currentTarget.get().getMinor(), + currentTarget.get().getMicro(), + qualifier)); + } + + /** Returns whether we should upgrade from given version */ + private boolean hasExpired(Version version) { + String qualifier = version.getQualifier(); + if (!qualifier.matches("^\\d{8,}")) return false; + + String dateString = qualifier.substring(0, 8); + Instant now = controller().clock().instant(); + Instant versionDate = LocalDate.parse(dateString, VERSION_DATE_PATTERN) + .atStartOfDay(ZoneOffset.UTC) + .toInstant(); + return versionDate.isBefore(now.minus(MAX_VERSION_AGE)); + } + + /** Returns the clouds where we can safely schedule OS upgrades */ + private Set<CloudName> supportedClouds() { + return controller().zoneRegistry().zones().reprovisionToUpgradeOs().zones().stream() + .map(ZoneApi::getCloudName) + .collect(Collectors.toUnmodifiableSet()); + } + + private Duration upgradeBudget() { + return controller().system().isCd() ? Duration.ofHours(1) : Duration.ofDays(14); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java new file mode 100644 index 00000000000..4aed9b0bffe --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class OsUpgradeSchedulerTest { + + @Test + public void maintain() { + ControllerTester tester = new ControllerTester(); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); + Instant initialTime = Instant.parse("2021-01-23T00:00:00.00Z"); + tester.clock().setInstant(initialTime); + + CloudName cloud = CloudName.from("cloud"); + ZoneApi zone = zone("prod.us-west-1", cloud); + tester.zoneRegistry().setZones(zone).reprovisionToUpgradeOsIn(zone); + + // Initial run does nothing as the cloud does not have a target + scheduler.maintain(); + assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty()); + + // Target is set + Version version0 = Version.fromString("7.0.0.20210123190005"); + tester.controller().upgradeOsIn(cloud, version0, Optional.of(Duration.ofDays(1)), false); + + // Target remains unchanged as it hasn't expired yet + for (var interval : List.of(Duration.ZERO, Duration.ofDays(15))) { + tester.clock().advance(interval); + scheduler.maintain(); + assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + } + + // Just over 30 days pass, and a new target replaces the expired one + Version version1 = Version.fromString("7.0.0.20210215"); + tester.clock().advance(Duration.ofDays(15).plus(Duration.ofSeconds(1))); + scheduler.maintain(); + assertEquals("New target set", version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + + // A few days pass and target remains unchanged + tester.clock().advance(Duration.ofDays(2)); + scheduler.maintain(); + assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + } + + private static ZoneApi zone(String id, CloudName cloud) { + return ZoneApiMock.newBuilder().withId(id).with(cloud).build(); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 14fd7abd96c..f8ae1cfefa0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -46,6 +46,9 @@ "name": "NameServiceDispatcher" }, { + "name": "OsUpgradeScheduler" + }, + { "name": "OsVersionStatusUpdater" }, { |