diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-08-15 13:34:37 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2018-08-16 10:40:30 +0200 |
commit | 29836deb2e651c948dffc0699bb858a975c3faf7 (patch) | |
tree | 230f882f94500f1852616447b32a69f8adb39d55 | |
parent | 145965c150859771595078f010740a002ce1942c (diff) |
Write OS version status to ZK
7 files changed, 125 insertions, 7 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6bd08d3d21c..95542141e47 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; @@ -227,6 +228,18 @@ public class Controller extends AbstractComponent { } } + /** Returns the current OS version status */ + public OsVersionStatus osVersionStatus() { + return curator.readOsVersionStatus(); + } + + /** Replace the current OS version status with a new one */ + public void updateOsVersionStatus(OsVersionStatus newStatus) { + try (Lock lock = curator.lockOsVersionStatus()) { + curator.writeOsVersionStatus(newStatus); + } + } + /** Returns the hostname of this controller */ public HostName hostname() { return HostName.from(hostnameSupplier.get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index a466fcc03cc..0214eb3cd6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,12 +4,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; -import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -41,6 +41,7 @@ public class ControllerMaintenance extends AbstractComponent { private final DnsMaintainer dnsMaintainer; private final SystemUpgrader systemUpgrader; private final OsUpgrader osUpgrader; + private final OsVersionStatusUpdater osVersionStatusUpdater; private final JobRunner jobRunner; @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -65,6 +66,7 @@ public class ControllerMaintenance extends AbstractComponent { systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud)); osUpgrader = new OsUpgrader(controller, Duration.ofMinutes(1), jobControl); + osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); } public Upgrader upgrader() { return upgrader; } @@ -88,6 +90,7 @@ public class ControllerMaintenance extends AbstractComponent { dnsMaintainer.deconstruct(); systemUpgrader.deconstruct(); osUpgrader.deconstruct(); + osVersionStatusUpdater.deconstruct(); jobRunner.deconstruct(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java new file mode 100644 index 00000000000..cf06489b8c1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -0,0 +1,31 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; + +/** + * @author mpolden + */ +public class OsVersionStatusUpdater extends Maintainer { + + public OsVersionStatusUpdater(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + protected void maintain() { + try { + OsVersionStatus newStatus = OsVersionStatus.compute(controller()); + controller().updateOsVersionStatus(newStatus); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + + ". Retrying in " + maintenanceInterval()); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 9c0d715ec57..8bc0f5786c6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -70,6 +71,7 @@ public class CuratorDb { private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); private final RunSerializer runSerializer = new RunSerializer(); + private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(); private final Curator curator; @@ -153,6 +155,10 @@ public class CuratorDb { return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } + public Lock lockOsVersionStatus() { + return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); + } + // -------------- Helpers ------------------------------------------ /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. @@ -249,6 +255,14 @@ public class CuratorDb { return readSlime(osTargetVersionPath()).map(versionSerializer::fromSlime); } + public void writeOsVersionStatus(OsVersionStatus status) { + curator.set(osVersionStatusPath(), asJson(osVersionStatusSerializer.toSlime(status))); + } + + public OsVersionStatus readOsVersionStatus() { + return readSlime(osVersionStatusPath()).map(osVersionStatusSerializer::fromSlime).orElse(OsVersionStatus.empty); + } + // -------------- Tenant -------------------------------------------------- public void writeTenant(UserTenant tenant) { @@ -453,6 +467,10 @@ public class CuratorDb { return root.append("osUpgrader").append("targetVersion"); } + private static Path osVersionStatusPath() { + return root.append("osVersionStatus"); + } + private static Path versionStatusPath() { return root.append("versionStatus"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index cfcbabc7e0a..518394b46fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -10,7 +11,9 @@ import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -38,7 +41,9 @@ public class OsVersionStatus { * must be queried. */ public static OsVersionStatus compute(Controller controller) { - List<OsVersion.Node> versions = new ArrayList<>(); + Map<Version, List<OsVersion.Node>> versions = new HashMap<>(); + // Always include target version, if set + controller.osVersion().ifPresent(version -> versions.put(version, new ArrayList<>())); for (SystemApplication application : SystemApplication.all()) { if (application.nodeTypesWithUpgradableOs().isEmpty()) { continue; // Avoid querying applications that do not have nodes with upgradable OS @@ -47,12 +52,16 @@ public class OsVersionStatus { controller.configServer().nodeRepository().list(zone, application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) .map(node -> new OsVersion.Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region())) - .forEach(versions::add); + .forEach(node -> versions.compute(node.version(), (ignored, nodes) -> { + if (nodes == null) { + nodes = new ArrayList<>(); + } + nodes.add(node); + return nodes; + })); } } - return new OsVersionStatus(versions.stream() - .collect(Collectors.groupingBy(OsVersion.Node::version)) - .entrySet() + return new OsVersionStatus(versions.entrySet() .stream() .map(kv -> new OsVersion(kv.getKey(), kv.getValue())) .sorted(Comparator.comparing(OsVersion::version)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java new file mode 100644 index 00000000000..48e1308a892 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; + +/** + * @author mpolden + */ +public class OsVersionStatusUpdaterTest { + + @Test + public void test_update() { + ControllerTester tester = new ControllerTester(); + OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), + new JobControl(new MockCuratorDb())); + + // Initially empty + assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus()); + + // Setting a new target adds it to current status + Version version1 = Version.fromString("7.1"); + tester.controller().upgradeOs(version1); + statusUpdater.maintain(); + List<OsVersion> osVersions = tester.controller().osVersionStatus().versions(); + assertFalse(osVersions.isEmpty()); + assertEquals(version1, osVersions.get(0).version()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 8d369c97338..5449e6387a9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -31,6 +31,9 @@ "name": "OsUpgrader" }, { + "name": "OsVersionStatusUpdater" + }, + { "name": "OutstandingChangeDeployer" }, { |