Merge branch 'master' into freva/remove-hardware-failure

# Conflicts:
#	node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java

14 files changed, 287 insertions, 76 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index f21231236d4..445d056b8e3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
 import com.google.common.collect.ImmutableList;
@@ -67,6 +67,13 @@ public class NodeList implements Iterable<Node> {
         return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().equals(type));
     }
 
+    /** Returns the subset of nodes that are currently changing their Vespa version */
+    public NodeList changingVersion() {
+        return filter(node -> node.status().vespaVersion().isPresent() &&
+                              node.allocation().isPresent() &&
+                              !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion()));
+    }
+
     /** Returns the subset of nodes assigned to the given cluster */
     public NodeList cluster(ClusterSpec.Id cluster) {
         return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().id().equals(cluster));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 39b0422901e..02161caead6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.google.inject.Inject;
@@ -46,6 +46,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final Optional<LoadBalancerExpirer> loadBalancerExpirer;
     private final Optional<DynamicProvisioningMaintainer> dynamicProvisioningMaintainer;
     private final CapacityReportMaintainer capacityReportMaintainer;
+    private final OsUpgradeActivator osUpgradeActivator;
 
     @Inject
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
@@ -80,6 +81,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner ->
                 new DynamicProvisioningMaintainer(nodeRepository, durationFromEnv("host_provisioner_interval").orElse(defaults.dynamicProvisionerInterval), hostProvisioner, flagSource));
         capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, durationFromEnv("capacity_report_interval").orElse(defaults.capacityReportInterval));
+        osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
         infrastructureProvisioner.maintain();
@@ -102,6 +104,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         infrastructureProvisioner.deconstruct();
         loadBalancerExpirer.ifPresent(Maintainer::deconstruct);
         dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct);
+        osUpgradeActivator.deconstruct();
     }
 
     private static Optional<Duration> durationFromEnv(String envVariable) {
@@ -145,6 +148,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration infrastructureProvisionInterval;
         private final Duration loadBalancerExpirerInterval;
         private final Duration dynamicProvisionerInterval;
+        private final Duration osUpgradeActivatorInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -164,6 +168,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             loadBalancerExpirerInterval = Duration.ofMinutes(10);
             reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions
             dynamicProvisionerInterval = Duration.ofMinutes(5);
+            osUpgradeActivatorInterval = Duration.ofMinutes(5);
 
             if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java
new file mode 100644
index 00000000000..e197689eda2
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java
@@ -0,0 +1,37 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+
+import java.time.Duration;
+
+/**
+ * This maintainer (de)activates OS upgrades according to Vespa upgrade status of nodes in this repository.
+ *
+ * If a node is upgrading to a new Vespa version, any ongoing OS upgrade will be paused for all nodes of that type. OS
+ * upgrades will resume once all nodes of that type have completed their Vespa upgrade.
+ *
+ * @author mpolden
+ */
+public class OsUpgradeActivator extends Maintainer {
+
+    public OsUpgradeActivator(NodeRepository nodeRepository, Duration interval) {
+        super(nodeRepository, interval);
+    }
+
+    @Override
+    protected void maintain() {
+        for (var nodeType : NodeType.values()) {
+            if (!nodeType.isDockerHost()) continue;
+            var active = canUpgradeOsOf(nodeType);
+            nodeRepository().osVersions().setActive(nodeType, active);
+        }
+    }
+
+    /** Returns whether to allow OS upgrade of nodes of given type */
+    private boolean canUpgradeOsOf(NodeType type) {
+        return nodeRepository().list().nodeType(type).changingVersion().asList().isEmpty();
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java
index 571356b0a34..99945ce46e8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java
@@ -44,4 +44,9 @@ public class OsVersion {
         return Objects.hash(version, active);
     }
 
+    @Override
+    public String toString() {
+        return "OS version " + version + " [active: " + active + "]";
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
index bc738400c45..a2d84bc7379 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.os;
 
 import com.google.common.base.Supplier;
@@ -18,6 +18,9 @@ import java.util.logging.Logger;
 /**
  * Thread-safe class that manages target OS versions for nodes in this repository.
  *
+ * A version target is initially inactive. Activation decision is taken by
+ * {@link com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator}.
+ *
  * The target OS version for each node type is set through the /nodes/v2/upgrade REST API.
  *
  * @author mpolden
@@ -45,6 +48,11 @@ public class OsVersions {
         this.db = db;
         this.cacheTtl = cacheTtl;
         createCache();
+
+        // Read and write all versions to make sure they are stored in the latest version of the serialized format
+        try (var lock = db.lockOsVersions()) {
+            db.writeOsVersions(db.readOsVersions());
+        }
     }
 
     private void createCache() {
@@ -65,6 +73,7 @@ public class OsVersions {
     /** Remove OS target for given node type. Nodes of this type will stop receiving wanted OS version in their
      * node object */
     public void removeTarget(NodeType nodeType) {
+        require(nodeType);
         try (Lock lock = db.lockOsVersions()) {
             Map<NodeType, OsVersion> osVersions = db.readOsVersions();
             osVersions.remove(nodeType);
@@ -76,9 +85,7 @@ public class OsVersions {
 
     /** Set the target OS version for nodes of given type */
     public void setTarget(NodeType nodeType, Version newTarget, boolean force) {
-        if (!nodeType.isDockerHost()) {
-            throw new IllegalArgumentException("Setting target OS version for " + nodeType + " nodes is unsupported");
-        }
+        require(nodeType);
         if (newTarget.isEmpty()) {
             throw  new IllegalArgumentException("Invalid target version: " + newTarget.toFullString());
         }
@@ -96,11 +103,33 @@ public class OsVersions {
                                                    + oldTarget.get().version());
             }
 
-            osVersions.put(nodeType, new OsVersion(newTarget, true));
+            osVersions.put(nodeType, new OsVersion(newTarget, false));
             db.writeOsVersions(osVersions);
             createCache(); // Throw away current cache
             log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString());
         }
     }
 
+    /** Activate or deactivate target for given node type. This is used for resuming or pausing an OS upgrade. */
+    public void setActive(NodeType nodeType, boolean active) {
+        require(nodeType);
+        try (Lock lock = db.lockOsVersions()) {
+            var osVersions = db.readOsVersions();
+            var currentVersion = osVersions.get(nodeType);
+            if (currentVersion == null) return; // No target version set for this type
+            if (currentVersion.active() == active) return; // No change
+
+            osVersions.put(nodeType, new OsVersion(currentVersion.version(), active));
+            db.writeOsVersions(osVersions);
+            createCache(); // Throw away current cache
+            log.info((active ? "Activated" : "Deactivated") + " OS target version for " + nodeType + " nodes");
+        }
+    }
+
+    private static void require(NodeType nodeType) {
+        if (!nodeType.isDockerHost()) {
+            throw new IllegalArgumentException("Node type '" + nodeType + "' does not support OS upgrades");
+        }
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
index 4104a31886a..26e59040b95 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
@@ -29,9 +29,11 @@ public class OsVersionsSerializer {
     public static byte[] toJson(Map<NodeType, OsVersion> versions) {
         var slime = new Slime();
         var object = slime.setObject();
-        // TODO(mpolden): Write active status here once all readers can handle it
-        versions.forEach((nodeType, osVersion) -> object.setString(NodeSerializer.toString(nodeType),
-                                                                   osVersion.version().toFullString()));
+        versions.forEach((nodeType, osVersion) -> {
+            var versionObject = object.setObject(NodeSerializer.toString(nodeType));
+            versionObject.setString(VERSION_FIELD, osVersion.version().toFullString());
+            versionObject.setBool(ACTIVE_FIELD, osVersion.active());
+        });
         try {
             return SlimeUtils.toJsonBytes(slime);
         } catch (IOException e) {
@@ -45,11 +47,11 @@ public class OsVersionsSerializer {
         inspector.traverse((ObjectTraverser) (key, value) -> {
             Version version;
             boolean active;
-            // TODO(mpolden): Remove fallback after next version
             if (value.type() == Type.OBJECT) {
                 version = Version.fromString(value.field(VERSION_FIELD).asString());
                 active = value.field(ACTIVE_FIELD).asBool();
             } else {
+                // TODO(mpolden): Remove support for legacy format after September 2019
                 version = Version.fromString(value.asString());
                 active = true;
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 5bc3703c11c..98d06f7e01a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -6,10 +6,6 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.flags.FetchVector;
-import com.yahoo.vespa.flags.FlagSource;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.JacksonFlag;
 
 import java.util.Arrays;
 import java.util.Locale;
@@ -23,11 +19,12 @@ import java.util.Optional;
 public class CapacityPolicies {
 
     private final Zone zone;
-    private final JacksonFlag<com.yahoo.vespa.flags.custom.NodeResources> defaultResourcesFlag;
+    /* Deployments must match 1-to-1 the advertised resources of a physical host */
+    private final boolean isUsingAdvertisedResources;
 
-    public CapacityPolicies(Zone zone, FlagSource flagSource) {
+    public CapacityPolicies(Zone zone) {
         this.zone = zone;
-        this.defaultResourcesFlag = Flags.DEFAULT_RESOURCES.bindTo(flagSource);
+        this.isUsingAdvertisedResources = zone.region().value().contains("aws-");
     }
 
     public int decideSize(Capacity requestedCapacity, ClusterSpec.Type clusterType) {
@@ -46,9 +43,7 @@ public class CapacityPolicies {
     public NodeResources decideNodeResources(Optional<NodeResources> requestedResources, ClusterSpec cluster) {
         if (requestedResources.isPresent()) assertMinimumResources(requestedResources.get(), cluster);
 
-        NodeResources resources = requestedResources
-                .or(() -> flagNodeResources(cluster.type()))
-                .orElse(defaultNodeResources(cluster.type()));
+        NodeResources resources = requestedResources.orElse(defaultNodeResources(cluster.type()));
 
         // Allow slow disks in zones which are not performance sensitive
         if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test)
@@ -71,16 +66,16 @@ public class CapacityPolicies {
                 minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb()));
     }
 
-    private Optional<NodeResources> flagNodeResources(ClusterSpec.Type clusterType) {
-        return Optional.ofNullable(defaultResourcesFlag.with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name()).value())
-                .map(r -> new NodeResources(r.vcpu(), r.memoryGb(), r.diskGb(), r.bandwidthGbps(), NodeResources.DiskSpeed.valueOf(r.diskSpeed())));
-    }
-
     private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) {
-        if (clusterType == ClusterSpec.Type.admin)
-            return new NodeResources(0.5, 2, 50, 0.3);
+        if (clusterType == ClusterSpec.Type.admin) {
+            return isUsingAdvertisedResources ?
+                    new NodeResources(0.5, 4, 50, 0.3) :
+                    new NodeResources(0.5, 2, 50, 0.3);
+        }
 
-        return new NodeResources(1.5, 8, 50, 0.3);
+        return isUsingAdvertisedResources ?
+                new NodeResources(2.0, 8, 50, 0.3) :
+                new NodeResources(1.5, 8, 50, 0.3);
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index 7c6fdbe6fa5..97b615d493f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -58,7 +58,7 @@ public class NodeRepositoryProvisioner implements Provisioner {
     public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone,
                                      ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) {
         this.nodeRepository = nodeRepository;
-        this.capacityPolicies = new CapacityPolicies(zone, flagSource);
+        this.capacityPolicies = new CapacityPolicies(zone);
         this.zone = zone;
         this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService, flagSource));
         this.preparer = new Preparer(nodeRepository,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java
new file mode 100644
index 00000000000..158951969eb
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java
@@ -0,0 +1,127 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Status;
+import com.yahoo.vespa.hosted.provision.os.OsVersion;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class OsUpgradeActivatorTest {
+
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
+
+    @Test
+    public void activates_upgrade() {
+        var osVersions = tester.nodeRepository().osVersions();
+        var osUpgradeActivator = new OsUpgradeActivator(tester.nodeRepository(), Duration.ofDays(1));
+        var version0 = Version.fromString("7.0");
+
+        // Create infrastructure nodes
+        var configHostApplication = ApplicationId.from("hosted-vespa", "configserver-host", "default");
+        var configHostNodes = tester.makeReadyNodes(3, "default", NodeType.confighost);
+        tester.prepareAndActivateInfraApplication(configHostApplication, NodeType.confighost, version0);
+
+        var tenantHostApplication = ApplicationId.from("hosted-vespa", "tenant-host", "default");
+        var tenantHostNodes = tester.makeReadyNodes(3, "default", NodeType.host);
+        tester.prepareAndActivateInfraApplication(tenantHostApplication, NodeType.host, version0);
+
+        // All nodes are on initial version
+        assertEquals(version0, minWantedVersion(NodeType.confighost, NodeType.host));
+        completeUpgradeOf(configHostNodes);
+        completeUpgradeOf(tenantHostNodes);
+        assertEquals("All nodes are on initial version", version0, minCurrentVersion(NodeType.confighost, NodeType.host));
+
+        // New OS target version is set
+        var osVersion0 = Version.fromString("8.0");
+        osVersions.setTarget(NodeType.host, osVersion0, false);
+        osVersions.setTarget(NodeType.confighost, osVersion0, false);
+
+        // New OS version is activated as there is no ongoing Vespa upgrade
+        osUpgradeActivator.maintain();
+        assertTrue("OS version " + osVersion0 + " is active", isOsVersionActive(NodeType.confighost, NodeType.host));
+
+        // Tenant hosts start upgrading to next Vespa version
+        var version1 = Version.fromString("7.1");
+        tester.prepareAndActivateInfraApplication(tenantHostApplication, NodeType.host, version1);
+        assertEquals("Wanted version of " + NodeType.host + " is raised", version1,
+                     minWantedVersion(NodeType.host));
+
+        // Activator pauses upgrade for tenant hosts only
+        osUpgradeActivator.maintain();
+        assertTrue("OS version " + osVersion0 + " is active", isOsVersionActive(NodeType.confighost));
+        assertFalse("OS version " + osVersion0 + " is inactive", isOsVersionActive(NodeType.host));
+
+        // Tenant hosts complete their Vespa upgrade
+        completeUpgradeOf(tenantHostNodes);
+        assertEquals("Tenant hosts upgraded", version1, minCurrentVersion(NodeType.host));
+
+        // Activator resumes OS upgrade of tenant hosts
+        osUpgradeActivator.run();
+        assertTrue("OS version " + osVersion0 + " is active", isOsVersionActive(NodeType.confighost, NodeType.host));
+    }
+
+    private boolean isOsVersionActive(NodeType... types) {
+        var active = true;
+        for (var type : types) {
+            active &= tester.nodeRepository().osVersions().targetFor(type).map(OsVersion::active).orElse(false);
+        }
+        return active;
+    }
+
+    private void completeUpgradeOf(List<Node> nodes) {
+        for (var node : nodes) {
+            try (var lock = tester.nodeRepository().lock(node)) {
+                node =  tester.nodeRepository().getNode(node.hostname()).get();
+                node = node.with(node.status().withVespaVersion(node.allocation().get().membership().cluster().vespaVersion()));
+                tester.nodeRepository().write(node, lock);
+            }
+        }
+    }
+
+    private Stream<Node> streamNodes(NodeType... types) {
+        Stream<Node> stream = Stream.empty();
+        for (var type : types) {
+            stream = Stream.concat(stream, tester.nodeRepository().getNodes(type).stream());
+        }
+        return stream;
+    }
+
+    private Version minCurrentVersion(NodeType... types) {
+        return streamNodes(types).map(Node::status)
+                                 .map(Status::vespaVersion)
+                                 .flatMap(Optional::stream)
+                                 .min(Comparator.naturalOrder())
+                                 .orElse(Version.emptyVersion);
+    }
+
+    private Version minWantedVersion(NodeType... types) {
+        return streamNodes(types).map(Node::allocation)
+                                 .flatMap(Optional::stream)
+                                 .map(Allocation::membership)
+                                 .map(ClusterMembership::cluster)
+                                 .map(ClusterSpec::vespaVersion)
+                                 .min(Comparator.naturalOrder())
+                                 .orElse(Version.emptyVersion);
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
index 06f9dcfae68..070db08f090 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.os;
 
 import com.yahoo.component.Version;
@@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
 import org.junit.Test;
 
 import java.time.Duration;
-import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -21,38 +20,41 @@ import static org.junit.Assert.fail;
  */
 public class OsVersionsTest {
 
-    private final OsVersions versions = new OsVersions(
-                new NodeRepositoryTester().nodeRepository().database(),
-                Duration.ofDays(1) // Long TTL to avoid timed expiry during test
-        );
-
     @Test
     public void test_versions() {
+        var versions = new OsVersions(new NodeRepositoryTester().nodeRepository().database(), Duration.ofDays(1));
+
         assertTrue("No versions set", versions.targets().isEmpty());
         assertSame("Caches empty target versions", versions.targets(), versions.targets());
 
         // Upgrade OS
-        Version version1 = Version.fromString("7.1");
-        versions.setTarget(NodeType.host, version1, false);
-        Map<NodeType, OsVersion> targetVersions = versions.targets();
+        var version1 = new OsVersion(Version.fromString("7.1"), false);
+        versions.setTarget(NodeType.host, version1.version(), false);
+        var targetVersions = versions.targets();
         assertSame("Caches target versions", targetVersions, versions.targets());
-        assertEquals(version1, versions.targetFor(NodeType.host).get().version());
+        assertEquals(version1, versions.targetFor(NodeType.host).get());
 
         // Upgrade OS again
-        Version version2 = Version.fromString("7.2");
-        versions.setTarget(NodeType.host, version2, false);
+        var version2 = new OsVersion(Version.fromString("7.2"), false);
+        versions.setTarget(NodeType.host, version2.version(), false);
         assertNotSame("Cache invalidated", targetVersions, versions.targets());
-        assertEquals(version2, versions.targetFor(NodeType.host).get().version());
+        assertEquals(version2, versions.targetFor(NodeType.host).get());
+
+        // Target can be (de)activated
+        versions.setActive(NodeType.host, true);
+        assertTrue("Target version deactivated", versions.targetFor(NodeType.host).get().active());
+        versions.setActive(NodeType.host, false);
+        assertFalse("Target version deactivated", versions.targetFor(NodeType.host).get().active());
 
         // Downgrading fails
         try {
-            versions.setTarget(NodeType.host, version1, false);
+            versions.setTarget(NodeType.host, version1.version(), false);
             fail("Expected exception");
         } catch (IllegalArgumentException ignored) {}
 
         // Forcing downgrade succeeds
-        versions.setTarget(NodeType.host, version1, true);
-        assertEquals(version1, versions.targetFor(NodeType.host).get().version());
+        versions.setTarget(NodeType.host, version1.version(), true);
+        assertEquals(version1, versions.targetFor(NodeType.host).get());
 
         // Target can be removed
         versions.removeTarget(NodeType.host);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java
index 4aec5b8370e..b41958b36db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java
@@ -9,14 +9,13 @@ import org.junit.Test;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 /**
  * @author mpolden
  */
 public class OsVersionsSerializerTest {
 
-    // TODO(mpolden): Remove once no longer supported
     @Test
     public void legacy_format() {
         var json = "{\"host\":\"1.2.3\",\"proxyhost\":\"4.5.6\",\"confighost\":\"7.8.9\"}";
@@ -33,28 +32,14 @@ public class OsVersionsSerializerTest {
     }
 
     @Test
-    public void read_future_format() {
-        var json = "{\n" +
-                   "  \"host\": {\n" +
-                   "    \"version\": \"1.2.3\",\n" +
-                   "    \"active\": false\n" +
-                   "  " +
-                   "},\n" +
-                   "  \"proxyhost\": {\n" +
-                   "    \"version\": \"4.5.6\",\n" +
-                   "    \"active\": true\n" +
-                   "  },\n" +
-                   "  \"confighost\": {\n" +
-                   "    \"version\": \"7.8.9\",\n" +
-                   "    \"active\": true\n" +
-                   "  }\n" +
-                   "}";
-        var versions = OsVersionsSerializer.fromJson(json.getBytes(StandardCharsets.UTF_8));
-        assertEquals(Map.of(
-                NodeType.host, new OsVersion(Version.fromString("1.2.3"), false),
-                NodeType.proxyhost, new OsVersion(Version.fromString("4.5.6"), true),
+    public void serialization() {
+        var versions = Map.of(
+                NodeType.host, new OsVersion(Version.fromString("1.2.3"), true),
+                NodeType.proxyhost, new OsVersion(Version.fromString("4.5.6"), false),
                 NodeType.confighost, new OsVersion(Version.fromString("7.8.9"), true)
-        ), versions);
+        );
+        var serialized = OsVersionsSerializer.fromJson(OsVersionsSerializer.toJson(versions));
+        assertEquals(serialized, versions);
     }
 
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index dee32513457..ef1ad1e76eb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.component.Version;
@@ -91,7 +91,7 @@ public class ProvisioningTester {
         this.orchestrator = orchestrator;
         ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner);
         this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource);
-        this.capacityPolicies = new CapacityPolicies(zone, flagSource);
+        this.capacityPolicies = new CapacityPolicies(zone);
         this.provisionLogger = new NullProvisionLogger();
         this.loadBalancerService = loadBalancerService;
     }
@@ -156,13 +156,17 @@ public class ProvisioningTester {
         assertEquals(toHostNames(hosts), toHostNames(nodeRepository.getNodes(application, Node.State.active)));
     }
 
-    public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType) {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), Version.fromString("6.42"), false);
+    public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) {
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), version, false);
         Capacity capacity = Capacity.fromRequiredNodeType(nodeType);
         List<HostSpec> hostSpecs = prepare(application, cluster, capacity, 1, true);
         activate(application, hostSpecs);
     }
 
+    public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType) {
+        prepareAndActivateInfraApplication(application, nodeType, Version.fromString("6.42"));
+    }
+
     public void deactivate(ApplicationId applicationId) {
         NestedTransaction deactivateTransaction = new NestedTransaction();
         nodeRepository.deactivate(applicationId, deactivateTransaction);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 729ded3234c..cad885104f3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.restapi.v2;
 
 import com.yahoo.application.Networking;
@@ -8,7 +8,11 @@ import com.yahoo.application.container.handler.Response;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.io.IOUtils;
 import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator;
 import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
+import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ComparisonFailure;
@@ -17,6 +21,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -680,7 +685,7 @@ public class RestApiTest {
                                    Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
                                    Request.Method.PATCH),
                        400,
-                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting target OS version for config nodes is unsupported\"}");
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}");
 
         // Attempt to downgrade OS
         assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
@@ -731,6 +736,11 @@ public class RestApiTest {
                                    Request.Method.PATCH),
                        "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}");
 
+        // Activate target
+        var nodeRepository = (NodeRepository) container.components().getComponent(MockNodeRepository.class.getName());
+        var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1));
+        osUpgradeActivator.run();
+
         // Other node type does not return wanted OS version
         Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"));
         assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
index 28881717e7c..cfb39e7e5b1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
@@ -31,6 +31,9 @@
       "name": "OperatorChangeApplicationMaintainer"
     },
     {
+      "name": "OsUpgradeActivator"
+    },
+    {
       "name": "PeriodicApplicationMaintainer"
     },
     {