Write OS version status to ZK

7 files changed, 125 insertions, 7 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 6bd08d3d21c..95542141e47 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.Rotation;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
@@ -227,6 +228,18 @@ public class Controller extends AbstractComponent {
         }
     }
 
+    /** Returns the current OS version status */
+    public OsVersionStatus osVersionStatus() {
+        return curator.readOsVersionStatus();
+    }
+
+    /** Replace the current OS version status with a new one */
+    public void updateOsVersionStatus(OsVersionStatus newStatus) {
+        try (Lock lock = curator.lockOsVersionStatus()) {
+            curator.writeOsVersionStatus(newStatus);
+        }
+    }
+
     /** Returns the hostname of this controller */
     public HostName hostname() {
         return HostName.from(hostnameSupplier.get());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index a466fcc03cc..0214eb3cd6c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -4,12 +4,12 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
-import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -41,6 +41,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final DnsMaintainer dnsMaintainer;
     private final SystemUpgrader systemUpgrader;
     private final OsUpgrader osUpgrader;
+    private final OsVersionStatusUpdater osVersionStatusUpdater;
     private final JobRunner jobRunner;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
@@ -65,6 +66,7 @@ public class ControllerMaintenance extends AbstractComponent {
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
         jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud));
         osUpgrader = new OsUpgrader(controller, Duration.ofMinutes(1), jobControl);
+        osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -88,6 +90,7 @@ public class ControllerMaintenance extends AbstractComponent {
         dnsMaintainer.deconstruct();
         systemUpgrader.deconstruct();
         osUpgrader.deconstruct();
+        osVersionStatusUpdater.deconstruct();
         jobRunner.deconstruct();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java
new file mode 100644
index 00000000000..cf06489b8c1
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java
@@ -0,0 +1,31 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+
+/**
+ * @author mpolden
+ */
+public class OsVersionStatusUpdater extends Maintainer {
+
+    public OsVersionStatusUpdater(Controller controller, Duration interval, JobControl jobControl) {
+        super(controller, interval, jobControl);
+    }
+
+    @Override
+    protected void maintain() {
+        try {
+            OsVersionStatus newStatus = OsVersionStatus.compute(controller());
+            controller().updateOsVersionStatus(newStatus);
+        } catch (Exception e) {
+            log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) +
+                                      ". Retrying in " + maintenanceInterval());
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 9c0d715ec57..8bc0f5786c6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
@@ -70,6 +71,7 @@ public class CuratorDb {
     private final TenantSerializer tenantSerializer = new TenantSerializer();
     private final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
     private final RunSerializer runSerializer = new RunSerializer();
+    private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer();
 
     private final Curator curator;
 
@@ -153,6 +155,10 @@ public class CuratorDb {
         return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
     }
 
+    public Lock lockOsVersionStatus() {
+        return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout);
+    }
+
     // -------------- Helpers ------------------------------------------
 
     /** Try locking with a low timeout, meaning it is OK to fail lock acquisition.
@@ -249,6 +255,14 @@ public class CuratorDb {
         return readSlime(osTargetVersionPath()).map(versionSerializer::fromSlime);
     }
 
+    public void writeOsVersionStatus(OsVersionStatus status) {
+        curator.set(osVersionStatusPath(), asJson(osVersionStatusSerializer.toSlime(status)));
+    }
+
+    public OsVersionStatus readOsVersionStatus() {
+        return readSlime(osVersionStatusPath()).map(osVersionStatusSerializer::fromSlime).orElse(OsVersionStatus.empty);
+    }
+
     // -------------- Tenant --------------------------------------------------
 
     public void writeTenant(UserTenant tenant) {
@@ -453,6 +467,10 @@ public class CuratorDb {
         return root.append("osUpgrader").append("targetVersion");
     }
 
+    private static Path osVersionStatusPath() {
+        return root.append("osVersionStatus");
+    }
+
     private static Path versionStatusPath() {
         return root.append("versionStatus");
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
index cfcbabc7e0a..518394b46fc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.versions;
 
 import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
@@ -10,7 +11,9 @@ import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -38,7 +41,9 @@ public class OsVersionStatus {
      * must be queried.
      */
     public static OsVersionStatus compute(Controller controller) {
-        List<OsVersion.Node> versions = new ArrayList<>();
+        Map<Version, List<OsVersion.Node>> versions = new HashMap<>();
+        // Always include target version, if set
+        controller.osVersion().ifPresent(version -> versions.put(version, new ArrayList<>()));
         for (SystemApplication application : SystemApplication.all()) {
             if (application.nodeTypesWithUpgradableOs().isEmpty()) {
                 continue; // Avoid querying applications that do not have nodes with upgradable OS
@@ -47,12 +52,16 @@ public class OsVersionStatus {
                 controller.configServer().nodeRepository().list(zone, application.id()).stream()
                           .filter(node -> OsUpgrader.eligibleForUpgrade(node, application))
                           .map(node -> new OsVersion.Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region()))
-                          .forEach(versions::add);
+                          .forEach(node -> versions.compute(node.version(), (ignored, nodes) -> {
+                              if (nodes == null) {
+                                  nodes = new ArrayList<>();
+                              }
+                              nodes.add(node);
+                              return nodes;
+                          }));
             }
         }
-        return new OsVersionStatus(versions.stream()
-                                           .collect(Collectors.groupingBy(OsVersion.Node::version))
-                                           .entrySet()
+        return new OsVersionStatus(versions.entrySet()
                                            .stream()
                                            .map(kv -> new OsVersion(kv.getKey(), kv.getValue()))
                                            .sorted(Comparator.comparing(OsVersion::version))
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
new file mode 100644
index 00000000000..48e1308a892
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -0,0 +1,41 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author mpolden
+ */
+public class OsVersionStatusUpdaterTest {
+
+    @Test
+    public void test_update() {
+        ControllerTester tester = new ControllerTester();
+        OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1),
+                                                                          new JobControl(new MockCuratorDb()));
+
+        // Initially empty
+        assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus());
+
+        // Setting a new target adds it to current status
+        Version version1 = Version.fromString("7.1");
+        tester.controller().upgradeOs(version1);
+        statusUpdater.maintain();
+        List<OsVersion> osVersions = tester.controller().osVersionStatus().versions();
+        assertFalse(osVersions.isEmpty());
+        assertEquals(version1, osVersions.get(0).version());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 8d369c97338..5449e6387a9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -31,6 +31,9 @@
       "name": "OsUpgrader"
     },
     {
+      "name": "OsVersionStatusUpdater"
+    },
+    {
       "name": "OutstandingChangeDeployer"
     },
     {