Implement OS version management

7 files changed, 260 insertions, 23 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 5db79beac3c..69b31f506e5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
 import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver;
 import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
+import com.yahoo.vespa.hosted.provision.provisioning.OsVersions;
 import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException;
 
 import java.time.Clock;
@@ -78,6 +79,7 @@ public class NodeRepository extends AbstractComponent {
     private final NodeFlavors flavors;
     private final NameResolver nameResolver;
     private final DockerImage dockerImage;
+    private final OsVersions osVersions;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -100,6 +102,7 @@ public class NodeRepository extends AbstractComponent {
         this.flavors = flavors;
         this.nameResolver = nameResolver;
         this.dockerImage = dockerImage;
+        this.osVersions = new OsVersions(this.db);
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (Node.State state : Node.State.values())
@@ -115,6 +118,9 @@ public class NodeRepository extends AbstractComponent {
     /** @return The name resolver used to resolve hostname and ip addresses */
     public NameResolver nameResolver() { return nameResolver; }
 
+    /** Returns the OS versions to use for nodes in this */
+    public OsVersions osVersions() { return osVersions; }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index f559ec0037b..a5dfc616302 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -71,6 +71,7 @@ public class CuratorDatabaseClient {
             curatorDatabase.create(toPath(state));
         curatorDatabase.create(inactiveJobsPath());
         curatorDatabase.create(infrastructureVersionsPath());
+        curatorDatabase.create(osVersionsPath());
     }
 
     /**
@@ -374,4 +375,23 @@ public class CuratorDatabaseClient {
         return root.append("infrastructureVersions");
     }
 
+    public Map<NodeType, Version> readOsVersions() {
+        return read(osVersionsPath(), NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new);
+    }
+
+    public void writeOsVersions(Map<NodeType, Version> versions) {
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(osVersionsPath().getAbsolute(),
+                                                         NodeTypeVersionsSerializer.toJson(versions)));
+        transaction.commit();
+    }
+
+    public Lock lockOsVersions() {
+        return lock(lockRoot.append("osVersionsLock"), defaultLockTimeout);
+    }
+
+    private Path osVersionsPath() {
+        return root.append("osVersions");
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java
new file mode 100644
index 00000000000..7e941d58a62
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java
@@ -0,0 +1,91 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * Thread-safe class that manages target OS versions for nodes in this repository.
+ *
+ * The target OS version for each node type is set through the /nodes/v2/upgrade REST API.
+ *
+ * @author mpolden
+ */
+public class OsVersions {
+
+    private static final Duration defaultCacheTtl = Duration.ofMinutes(1);
+    private static final Logger log = Logger.getLogger(OsVersions.class.getName());
+
+    private final CuratorDatabaseClient db;
+    private final Duration cacheTtl;
+
+    /**
+     * Target OS version is read on every request to /nodes/v2/node/[fqdn]. Cache current targets to avoid
+     * unnecessary ZK reads. When targets change, some nodes may need to wait for TTL until they see the new target,
+     * this is fine.
+     */
+    private volatile Supplier<Map<NodeType, Version>> currentTargets;
+
+    public OsVersions(CuratorDatabaseClient db) {
+        this(db, defaultCacheTtl);
+    }
+
+    OsVersions(CuratorDatabaseClient db, Duration cacheTtl) {
+        this.db = db;
+        this.cacheTtl = cacheTtl;
+        createCache();
+    }
+
+    private void createCache() {
+        this.currentTargets = Suppliers.memoizeWithExpiration(() -> ImmutableMap.copyOf(db.readOsVersions()),
+                                                              cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    /** Returns the current target versions for each node type */
+    public Map<NodeType, Version> targets() {
+        return currentTargets.get();
+    }
+
+    /** Returns the current target version for given node type, if any */
+    public Optional<Version> targetFor(NodeType type) {
+        return Optional.ofNullable(targets().get(type));
+    }
+
+    /** Set the target OS version for nodes of given type */
+    public void setTarget(NodeType nodeType, Version newTarget, boolean force) {
+        if (!nodeType.isDockerHost()) {
+            throw new IllegalArgumentException("Setting target OS version for " + nodeType + " nodes is unsupported");
+        }
+        try (Lock lock = db.lockOsVersions()) {
+            Map<NodeType, Version> osVersions = db.readOsVersions();
+            Optional<Version> oldTarget = Optional.ofNullable(osVersions.get(nodeType));
+
+            if (oldTarget.filter(v -> v.equals(newTarget)).isPresent()) {
+                return; // Old target matches new target, nothing to do
+            }
+
+            if (!force && oldTarget.filter(v -> v.isAfter(newTarget)).isPresent()) {
+                throw new IllegalArgumentException("Cannot set target OS version to " + newTarget +
+                                                   " without setting 'force', as it's lower than the current version: "
+                                                   + oldTarget.get());
+            }
+
+            osVersions.put(nodeType, newTarget);
+            db.writeOsVersions(osVersions);
+            createCache(); // Throw away current cache
+            log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString());
+        }
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index ed251e57c6d..a8d9266c9b7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
@@ -15,7 +16,6 @@ import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
@@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.yolean.Exceptions;
 
+import javax.inject.Inject;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -37,7 +38,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
-import javax.inject.Inject;
 
 import static com.yahoo.vespa.config.SlimeUtils.optionalString;
 
@@ -99,7 +99,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository);
         if (path.equals(    "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot");
         if (path.equals(    "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl());
-        if (path.equals(    "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions());
+        if (path.equals(    "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions());
         throw new NotFoundException("Nothing at path '" + path + "'");
     }
 
@@ -139,7 +139,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
             return new MessageResponse("Updated " + node.hostname());
         }
         else if (path.startsWith("/nodes/v2/upgrade/")) {
-            return setInfrastructureVersion(request);
+            return setTargetVersions(request);
         }
 
         throw new NotFoundException("Nothing at '" + path + "'");
@@ -256,7 +256,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
             path = path.substring(0, path.length()-1);
         int lastSlash = path.lastIndexOf("/");
         if (lastSlash < 0) return path;
-        return path.substring(lastSlash + 1, path.length());
+        return path.substring(lastSlash + 1);
     }
 
     private static boolean isPatchOverride(HttpRequest request) {
@@ -280,19 +280,33 @@ public class NodesApiHandler extends LoggingRequestHandler {
         return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
     }
 
-    private MessageResponse setInfrastructureVersion(HttpRequest request) {
+    private MessageResponse setTargetVersions(HttpRequest request) {
         NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase());
         Inspector inspector = toSlime(request.getData()).get();
+        List<String> messageParts = new ArrayList<>(2);
 
-        Inspector versionField = inspector.field("version");
-        if (!versionField.valid())
-            throw new IllegalArgumentException("'version' is missing");
-        Version version = Version.fromString(versionField.asString());
         boolean force = inspector.field("force").asBool();
+        Inspector versionField = inspector.field("version");
+        Inspector osVersionField = inspector.field("osVersion");
 
-        maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force);
+        if (versionField.valid()) {
+            Version version = Version.fromString(versionField.asString());
+            maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force);
+            messageParts.add("version to " + version.toFullString());
+        }
+
+        if (osVersionField.valid()) {
+            Version osVersion = Version.fromString(osVersionField.asString());
+            nodeRepository.osVersions().setTarget(nodeType, osVersion, force);
+            messageParts.add("osVersion to " + osVersion.toFullString());
+        }
+
+        if (messageParts.isEmpty()) {
+            throw new IllegalArgumentException("At least one of 'version' and 'osVersion' must be set");
+        }
 
-        return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString());
+        return new MessageResponse("Set " + String.join(", ", messageParts) +
+                                   " for nodes of type " + nodeType);
     }
 
     private static String hostnamesAsString(List<Node> nodes) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
index 3fb712e182f..392cba7baa9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
@@ -6,24 +6,25 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
+import com.yahoo.vespa.hosted.provision.provisioning.OsVersions;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.Comparator;
-import java.util.Map;
 
 /**
- * A response containing infrastructure versions
+ * A response containing targets for infrastructure Vespa version and OS version.
  *
  * @author freva
  */
 public class UpgradeResponse extends HttpResponse {
 
     private final InfrastructureVersions infrastructureVersions;
+    private final OsVersions osVersions;
 
-    public UpgradeResponse(InfrastructureVersions infrastructureVersions) {
+    public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions) {
         super(200);
         this.infrastructureVersions = infrastructureVersions;
+        this.osVersions = osVersions;
     }
 
     @Override
@@ -34,6 +35,9 @@ public class UpgradeResponse extends HttpResponse {
         Cursor versionsObject = root.setObject("versions");
         infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString()));
 
+        Cursor osVersionsObject = root.setObject("osVersions");
+        osVersions.targets().forEach((nodeType, version) -> osVersionsObject.setString(nodeType.name(), version.toFullString()));
+
         new JsonFormat(true).encode(stream, slime);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java
new file mode 100644
index 00000000000..88f5dcb9854
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author mpolden
+ */
+public class OsVersionsTest {
+
+    private OsVersions versions;
+
+    @Before
+    public void before() {
+        versions = new OsVersions(
+                new NodeRepositoryTester().nodeRepository().database(),
+                Duration.ofDays(1) // Long TTL to avoid timed expiry during test
+        );
+    }
+
+    @Test
+    public void test_versions() {
+        assertTrue("No versions set", versions.targets().isEmpty());
+        assertSame("Caches empty target versions", versions.targets(), versions.targets());
+
+        // Upgrade OS
+        Version version1 = Version.fromString("7.1");
+        versions.setTarget(NodeType.host, version1, false);
+        Map<NodeType, Version> targetVersions = versions.targets();
+        assertSame("Caches target versions", targetVersions, versions.targets());
+        assertEquals(version1, versions.targetFor(NodeType.host).get());
+
+        // Upgrade OS again
+        Version version2 = Version.fromString("7.2");
+        versions.setTarget(NodeType.host, version2, false);
+        assertNotSame("Cache invalidated", targetVersions, versions.targets());
+        assertEquals(version2, versions.targetFor(NodeType.host).get());
+
+        // Downgrading fails
+        try {
+            versions.setTarget(NodeType.host, version1, false);
+            fail("Expected exception");
+        } catch (IllegalArgumentException ignored) {}
+
+        // Forcing downgrade succeeds
+        versions.setTarget(NodeType.host, version1, true);
+        assertEquals(version1, versions.targetFor(NodeType.host).get());
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index ef0feecc037..b0053b3e7e6 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
-import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -506,21 +505,21 @@ public class RestApiTest {
     @Test
     public void test_upgrade() throws IOException {
         // Initially, no versions are set
-        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{}}");
 
         // Set version for config and confighost
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config",
                         Utf8.toBytes("{\"version\": \"6.123.456\"}"),
                         Request.Method.PATCH),
-                "{\"message\":\"Set version for config to 6.123.456\"}");
+                "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
                         Utf8.toBytes("{\"version\": \"6.123.456\"}"),
                         Request.Method.PATCH),
-                "{\"message\":\"Set version for confighost to 6.123.456\"}");
+                "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}");
 
         // Verify versions are set
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
-                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}");
+                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"},\"osVersions\":{}}");
 
         // Downgrade without force fails
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
@@ -534,11 +533,51 @@ public class RestApiTest {
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
                         Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"),
                         Request.Method.PATCH),
-                "{\"message\":\"Set version for confighost to 6.123.1\"}");
+                "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}");
 
         // Verify version has been updated
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
-                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}");
+                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{}}");
+
+        // Upgrade OS for confighost and host
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                                   Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+                                   Request.Method.PATCH),
+                       "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host",
+                                   Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+                                   Request.Method.PATCH),
+                       "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}");
+
+        // OS versions are set
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
+                       "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"}}");
+
+        // Upgrade OS and Vespa together
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                                   Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"),
+                                   Request.Method.PATCH),
+                       "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}");
+
+        // Attempt to upgrade unsupported node type
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config",
+                                   Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+                                   Request.Method.PATCH),
+                       400,
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting target OS version for config nodes is unsupported\"}");
+
+        // Attempt to downgrade OS
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                                   Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"),
+                                   Request.Method.PATCH),
+                       400,
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}");
+
+        // Downgrading OS with force succeeds
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                                   Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"),
+                                   Request.Method.PATCH),
+                       "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}");
     }
 
     /** Tests the rendering of each node separately to make it easier to find errors */