diff options
author | Martin Polden <mpolden@mpolden.no> | 2021-02-09 13:58:24 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2021-02-09 15:13:19 +0100 |
commit | 9b80111064a0b44c5dc6ca24d26748e430609646 (patch) | |
tree | 415fdb80ec40a398b52980ce456464df64af25af /controller-server | |
parent | e11470603647ac5d41a19ad4b8a753a53e640df9 (diff) |
Schedule OS upgrades automatically in supported clouds
Diffstat (limited to 'controller-server')
4 files changed, 159 insertions, 0 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b32b4ec73fb..bc0295abca3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -51,6 +51,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); + maintainers.add(new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler)); maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); @@ -99,6 +100,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration jobRunner; private final Duration osVersionStatusUpdater; private final Duration osUpgrader; + private final Duration osUpgradeScheduler; private final Duration contactInformationMaintainer; private final Duration nameServiceDispatcher; private final Duration costReportMaintainer; @@ -124,6 +126,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobRunner = duration(90, SECONDS); this.osVersionStatusUpdater = duration(2, MINUTES); this.osUpgrader = duration(1, MINUTES); + this.osUpgradeScheduler = duration(3, HOURS); this.contactInformationMaintainer = duration(12, HOURS); this.nameServiceDispatcher = duration(10, SECONDS); this.costReportMaintainer = duration(2, HOURS); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java new file mode 100644 index 00000000000..b96dc47009b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -0,0 +1,87 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Automatically schedule OS upgrades. + * + * This is used in clouds where new OS versions regularly become available. + * + * @author mpolden + */ +public class OsUpgradeScheduler extends ControllerMaintainer { + + private static final Duration MAX_VERSION_AGE = Duration.ofDays(30); + private static final DateTimeFormatter VERSION_DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public OsUpgradeScheduler(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected boolean maintain() { + for (var cloud : supportedClouds()) { + Optional<Version> newTarget = newTargetIn(cloud); + if (newTarget.isEmpty()) continue; + controller().upgradeOsIn(cloud, newTarget.get(), Optional.of(upgradeBudget()), false); + } + return true; + } + + /** Returns the new target version for given cloud, if any */ + private Optional<Version> newTargetIn(CloudName cloud) { + Optional<Version> currentTarget = controller().osVersionTarget(cloud) + .map(OsVersionTarget::osVersion) + .map(OsVersion::version); + + if (currentTarget.isEmpty()) return Optional.empty(); + if (!hasExpired(currentTarget.get())) return Optional.empty(); + Instant now = controller().clock().instant(); + String qualifier = LocalDate.ofInstant(now, ZoneOffset.UTC).format(VERSION_DATE_PATTERN); + return Optional.of(new Version(currentTarget.get().getMajor(), + currentTarget.get().getMinor(), + currentTarget.get().getMicro(), + qualifier)); + } + + /** Returns whether we should upgrade from given version */ + private boolean hasExpired(Version version) { + String qualifier = version.getQualifier(); + if (qualifier.matches("^\\d{8,}")) { + String dateString = qualifier.substring(0, 8); + Instant now = controller().clock().instant(); + Instant versionDate = LocalDate.parse(dateString, VERSION_DATE_PATTERN) + .atStartOfDay(ZoneOffset.UTC) + .toInstant(); + return versionDate.isBefore(now.minus(MAX_VERSION_AGE)); + } + return false; + } + + /** Returns the clouds where we can safely schedule OS upgrades */ + private Set<CloudName> supportedClouds() { + return controller().zoneRegistry().zones().reprovisionToUpgradeOs().zones().stream() + .map(ZoneApi::getCloudName) + .collect(Collectors.toUnmodifiableSet()); + } + + private Duration upgradeBudget() { + return controller().system().isCd() ? Duration.ofHours(1) : Duration.ofDays(14); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java new file mode 100644 index 00000000000..e8cdb6ec940 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class OsUpgradeSchedulerTest { + + @Test + public void maintain() { + ControllerTester tester = new ControllerTester(); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); + Instant initialTime = Instant.parse("2021-01-23T00:00:00.00Z"); + tester.clock().setInstant(initialTime); + + CloudName cloud = CloudName.from("cloud"); + ZoneApi zone = zone("prod.us-west-1", cloud); + tester.zoneRegistry().setZones(zone).reprovisionToUpgradeOsIn(zone); + + // Initial run does nothing as the cloud does not have a target + scheduler.maintain(); + assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty()); + + // Target is set + Version version0 = Version.fromString("7.0.0.20210123190005"); + tester.controller().upgradeOsIn(cloud, version0, Optional.of(Duration.ofDays(1)), false); + + // Target remains unchanged as it hasn't expired yet + for (var interval : List.of(Duration.ZERO, Duration.ofDays(15))) { + tester.clock().advance(interval); + scheduler.maintain(); + assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + } + + // Just over 30 days pass, and a new target replaces the expired one + Version version1 = Version.fromString("7.0.0.20210222"); + tester.clock().advance(Duration.ofDays(15).plus(Duration.ofSeconds(1))); + scheduler.maintain(); + assertEquals("New target set", version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + + // A few days pass and target remains unchanged + tester.clock().advance(Duration.ofDays(2)); + scheduler.maintain(); + assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + } + + private static ZoneApi zone(String id, CloudName cloud) { + return ZoneApiMock.newBuilder().withId(id).with(cloud).build(); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 14fd7abd96c..f8ae1cfefa0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -46,6 +46,9 @@ "name": "NameServiceDispatcher" }, { + "name": "OsUpgradeScheduler" + }, + { "name": "OsVersionStatusUpdater" }, { |