diff options
Diffstat (limited to 'controller-server/src/main')
3 files changed, 100 insertions, 0 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index e12c1903426..abc0784396c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -235,6 +235,8 @@ public class Controller extends AbstractComponent { targets.removeIf(target -> target.osVersion().cloud().equals(cloudName)); // Only allow a single target per cloud targets.add(new OsVersionTarget(new OsVersion(version, cloudName), upgradeBudget)); curator.writeOsVersionTargets(targets); + log.info("Triggered OS upgrade to " + version.toFullString() + " in cloud " + + cloudName.value() + upgradeBudget.map(b -> ", with upgrade budget " + b).orElse("")); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b32b4ec73fb..bc0295abca3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -51,6 +51,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); + maintainers.add(new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler)); maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); @@ -99,6 +100,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration jobRunner; private final Duration osVersionStatusUpdater; private final Duration osUpgrader; + private final Duration osUpgradeScheduler; private final Duration contactInformationMaintainer; private final Duration nameServiceDispatcher; private final Duration costReportMaintainer; @@ -124,6 +126,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobRunner = duration(90, SECONDS); this.osVersionStatusUpdater = duration(2, MINUTES); this.osUpgrader = duration(1, MINUTES); + this.osUpgradeScheduler = duration(3, HOURS); this.contactInformationMaintainer = duration(12, HOURS); this.nameServiceDispatcher = duration(10, SECONDS); this.costReportMaintainer = duration(2, HOURS); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java new file mode 100644 index 00000000000..cb6d95a3300 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -0,0 +1,95 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Automatically schedule OS upgrades. + * + * This is used in clouds where new OS versions regularly become available. + * + * @author mpolden + */ +public class OsUpgradeScheduler extends ControllerMaintainer { + + /** Trigger a new upgrade when the current target version reaches this age */ + private static final Duration MAX_VERSION_AGE = Duration.ofDays(30); + + /** + * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version that + * may not be available yet + */ + private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7); + + private static final DateTimeFormatter VERSION_DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public OsUpgradeScheduler(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected boolean maintain() { + for (var cloud : supportedClouds()) { + Optional<Version> newTarget = newTargetIn(cloud); + if (newTarget.isEmpty()) continue; + controller().upgradeOsIn(cloud, newTarget.get(), Optional.of(upgradeBudget()), false); + } + return true; + } + + /** Returns the new target version for given cloud, if any */ + private Optional<Version> newTargetIn(CloudName cloud) { + Optional<Version> currentTarget = controller().osVersionTarget(cloud) + .map(OsVersionTarget::osVersion) + .map(OsVersion::version); + if (currentTarget.isEmpty()) return Optional.empty(); + if (!hasExpired(currentTarget.get())) return Optional.empty(); + + Instant now = controller().clock().instant(); + String qualifier = LocalDate.ofInstant(now.minus(AVAILABILITY_INTERVAL), ZoneOffset.UTC) + .format(VERSION_DATE_PATTERN); + return Optional.of(new Version(currentTarget.get().getMajor(), + currentTarget.get().getMinor(), + currentTarget.get().getMicro(), + qualifier)); + } + + /** Returns whether we should upgrade from given version */ + private boolean hasExpired(Version version) { + String qualifier = version.getQualifier(); + if (!qualifier.matches("^\\d{8,}")) return false; + + String dateString = qualifier.substring(0, 8); + Instant now = controller().clock().instant(); + Instant versionDate = LocalDate.parse(dateString, VERSION_DATE_PATTERN) + .atStartOfDay(ZoneOffset.UTC) + .toInstant(); + return versionDate.isBefore(now.minus(MAX_VERSION_AGE)); + } + + /** Returns the clouds where we can safely schedule OS upgrades */ + private Set<CloudName> supportedClouds() { + return controller().zoneRegistry().zones().reprovisionToUpgradeOs().zones().stream() + .map(ZoneApi::getCloudName) + .collect(Collectors.toUnmodifiableSet()); + } + + private Duration upgradeBudget() { + return controller().system().isCd() ? Duration.ofHours(1) : Duration.ofDays(14); + } + +} |