Merge pull request #5784 from vespa-engine/freva/infra-app-provision

Infrastructure app provision

15 files changed, 611 insertions, 12 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
new file mode 100644
index 00000000000..b6955195dcf
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -0,0 +1,104 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
+import com.yahoo.vespa.service.monitor.application.ConfigServerHostApplication;
+import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
+import com.yahoo.vespa.service.monitor.application.ProxyHostApplication;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * This maintainer makes sure that infrastructure nodes are allocated with correct wanted
+ * version. Source for the wanted version comes from the target version set using
+ * /nodes/v2/upgrade/ endpoint.
+ *
+ * @author freva
+ */
+public class InfrastructureProvisioner extends Maintainer {
+
+    private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName());
+    private static final List<HostedVespaApplication> HOSTED_VESPA_APPLICATIONS = Arrays.asList(
+            ConfigServerApplication.CONFIG_SERVER_APPLICATION,
+            ConfigServerHostApplication.CONFIG_SERVER_HOST_APPLICATION,
+            ProxyHostApplication.PROXY_HOST_APPLICATION);
+
+    private final Provisioner provisioner;
+    private final InfrastructureVersions infrastructureVersions;
+
+    public InfrastructureProvisioner(Provisioner provisioner, NodeRepository nodeRepository,
+                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl) {
+        super(nodeRepository, interval, jobControl);
+        this.provisioner = provisioner;
+        this.infrastructureVersions = infrastructureVersions;
+    }
+
+    @Override
+    protected void maintain() {
+        for (HostedVespaApplication application: HOSTED_VESPA_APPLICATIONS) {
+            try (Mutex lock = nodeRepository().lock(application.getApplicationId())) {
+                Optional<Version> version = getTargetVersion(application.getCapacity().type());
+                if (! version.isPresent()) continue;
+
+                List<HostSpec> hostSpecs = provisioner.prepare(
+                        application.getApplicationId(),
+                        application.getClusterSpecWithVersion(version.get()),
+                        application.getCapacity(),
+                        1, // groups
+                        logger::log);
+
+                NestedTransaction nestedTransaction = new NestedTransaction();
+                provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs);
+                nestedTransaction.commit();
+            }
+        }
+    }
+
+    /**
+     * Returns the version that the given node type should be provisioned to. This is
+     * the version returned by {@link InfrastructureVersions#getTargetVersionFor} unless a provisioning is:
+     * <ul>
+     *   <li>not possible: no nodes of given type in legal state in node-repo</li>
+     *   <li>redundant: all nodes that can be provisioned already have the right wanted Vespa version</li>
+     * </ul>
+     */
+    Optional<Version> getTargetVersion(NodeType nodeType) {
+        Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
+        if (!targetVersion.isPresent()) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
+            return Optional.empty();
+        }
+
+        List<Version> wantedVersions = nodeRepository().getNodes(nodeType,
+                Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive).stream()
+                .map(node -> node.allocation()
+                        .map(allocation -> allocation.membership().cluster().vespaVersion())
+                        .orElse(null))
+                .collect(Collectors.toList());
+        if (wantedVersions.isEmpty()) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
+            return Optional.empty();
+        }
+
+        if (wantedVersions.stream().allMatch(targetVersion.get()::equals)) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
+                    ": Already provisioned to target version " + targetVersion);
+            return Optional.empty();
+        }
+        return targetVersion;
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java
new file mode 100644
index 00000000000..52d7a63dc5e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java
@@ -0,0 +1,63 @@
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Multithread safe class to see and set target versions for infrastructure node types.
+ * {@link InfrastructureProvisioner} maintainer will then allocate the nodes of given node type
+ * with a wanted version equal to the given target version.
+ *
+ * @author freva
+ */
+public class InfrastructureVersions {
+
+    private static Logger logger = Logger.getLogger(InfrastructureVersions.class.getName());
+
+    private final CuratorDatabaseClient db;
+
+    public InfrastructureVersions(CuratorDatabaseClient db) {
+        this.db = db;
+    }
+
+    public void setTargetVersion(NodeType nodeType, Version newTargetVersion, boolean force) {
+        if (nodeType != NodeType.config && nodeType != NodeType.confighost && nodeType != NodeType.proxyhost) {
+            throw new IllegalArgumentException("Cannot set version for type " + nodeType);
+        }
+
+        try (Lock lock = db.lockInfrastructureVersions()) {
+            Map<NodeType, Version> infrastructureVersions = db.readInfrastructureVersions();
+            Optional<Version> currentTargetVersion = Optional.ofNullable(infrastructureVersions.get(nodeType));
+
+            // Trying to set the version to the current version, skip
+            if (currentTargetVersion.equals(Optional.of(newTargetVersion))) return;
+
+            // If we don't force the set, we must set the new version to higher than the already set version
+            if (!force && currentTargetVersion.isPresent()) {
+                if (currentTargetVersion.get().isAfter(newTargetVersion))
+                    throw new IllegalArgumentException(String.format("Cannot downgrade version without setting 'force'. " +
+                            "Current target version: %s, attempted to set target version: %s",
+                            currentTargetVersion.get().toFullString(), newTargetVersion.toFullString()));
+            }
+
+            infrastructureVersions.put(nodeType, newTargetVersion);
+            db.writeInfrastructureVersions(infrastructureVersions);
+            logger.info("Set target version for " + nodeType + " to " + newTargetVersion.toFullString());
+        }
+    }
+
+    public Optional<Version> getTargetVersionFor(NodeType nodeType) {
+        return Optional.ofNullable(db.readInfrastructureVersions().get(nodeType));
+    }
+
+    public Map<NodeType, Version> getTargetVersions() {
+        return Collections.unmodifiableMap(db.readInfrastructureVersions());
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 18af498a38d..32c7a4035d9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostLivenessTracker;
+import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -46,24 +47,27 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final NodeRebooter nodeRebooter;
     private final NodeRetirer nodeRetirer;
     private final MetricsReporter metricsReporter;
+    private final InfrastructureProvisioner infrastructureProvisioner;
 
     private final JobControl jobControl;
+    private final InfrastructureVersions infrastructureVersions;
 
     @Inject
-    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer,
-                                     HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, 
+    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
+                                     HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig) {
-        this(nodeRepository, deployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
+        this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
                 orchestrator, metric, configserverConfig);
     }
 
-    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer,
+    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig) {
         DefaultTimes defaults = new DefaultTimes(zone.environment());
         jobControl = new JobControl(nodeRepository.database());
+        infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
 
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv("throttle_policy").orElse(defaults.throttlePolicy), metric, jobControl, configserverConfig);
         periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
@@ -76,6 +80,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl);
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
+        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl);
+
 
         RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone));
         FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker(
@@ -97,10 +103,15 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         nodeRetirer.deconstruct();
         provisionedExpirer.deconstruct();
         metricsReporter.deconstruct();
+        infrastructureProvisioner.deconstruct();
     }
 
     public JobControl jobControl() { return jobControl; }
 
+    public InfrastructureVersions infrastructureVersions() {
+        return infrastructureVersions;
+    }
+
     private static Optional<Duration> durationFromEnv(String envVariable) {
         return Optional.ofNullable(System.getenv(envPrefix + envVariable)).map(Long::parseLong).map(Duration::ofSeconds);
     }
@@ -136,6 +147,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration nodeRetirerInterval;
         private final Duration metricsInterval;
         private final Duration retiredInterval;
+        private final Duration infrastructureProvisionInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -148,6 +160,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             rebootInterval = Duration.ofDays(30);
             nodeRetirerInterval = Duration.ofMinutes(30);
             metricsInterval = Duration.ofMinutes(1);
+            infrastructureProvisionInterval = Duration.ofMinutes(3);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
 
             if (environment.isTest())
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 7fc305d397f..44eda5e83ec 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -2,9 +2,11 @@
 package com.yahoo.vespa.hosted.provision.persistence;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
@@ -22,6 +24,7 @@ import java.time.Clock;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -70,6 +73,7 @@ public class CuratorDatabaseClient {
         for (Node.State state : Node.State.values())
             curatorDatabase.create(toPath(state));
         curatorDatabase.create(inactiveJobsPath());
+        curatorDatabase.create(infrastructureVersionsPath());
     }
 
     /**
@@ -363,5 +367,27 @@ public class CuratorDatabaseClient {
     private Path inactiveJobsPath() {
         return root.append("inactiveJobs");
     }
-    
+
+
+    public Map<NodeType, Version> readInfrastructureVersions() {
+        byte[] data = curatorDatabase.getData(infrastructureVersionsPath()).get();
+        if (data.length == 0) return new HashMap<>(); // infrastructure versions have never been written
+        return InfrastructureVersionsSerializer.fromJson(data);
+    }
+
+    public void writeInfrastructureVersions(Map<NodeType, Version> infrastructureVersions) {
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath().getAbsolute(),
+                InfrastructureVersionsSerializer.toJson(infrastructureVersions)));
+        transaction.commit();
+    }
+
+    public Lock lockInfrastructureVersions() {
+        return lock(root.append("locks").append("infrastructureVersionsLock"), defaultLockTimeout);
+    }
+
+    private Path infrastructureVersionsPath() {
+        return root.append("infrastructureVersions");
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java
new file mode 100644
index 00000000000..a48888fb4f0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java
@@ -0,0 +1,42 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author freva
+ */
+class InfrastructureVersionsSerializer {
+
+    private InfrastructureVersionsSerializer() {}
+
+    static byte[] toJson(Map<NodeType, Version> versionsByNodeType) {
+        try {
+            Slime slime = new Slime();
+            Cursor object = slime.setObject();
+            versionsByNodeType.forEach((nodeType, version) ->
+                    object.setString(NodeSerializer.toString(nodeType), version.toFullString()));
+            return SlimeUtils.toJsonBytes(slime);
+        } catch (IOException e) {
+            throw new RuntimeException("Serialization of a infrastructure version failed", e);
+        }
+    }
+
+    static Map<NodeType, Version> fromJson(byte[] data) {
+        Map<NodeType, Version> infrastructureVersions = new HashMap<>();
+        Inspector inspector = SlimeUtils.jsonToSlime(data).get();
+        inspector.traverse((ObjectTraverser) (key, value) ->
+                infrastructureVersions.put(NodeSerializer.nodeTypeFromString(key), Version.fromString(value.asString())));
+        return infrastructureVersions;
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index d164252b018..4d753599c4e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -317,7 +317,7 @@ public class NodeSerializer {
         throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
     }
 
-    private NodeType nodeTypeFromString(String typeString) {
+    static NodeType nodeTypeFromString(String typeString) {
         switch (typeString) {
             case "tenant" : return NodeType.tenant;
             case "host" : return NodeType.host;
@@ -329,7 +329,7 @@ public class NodeSerializer {
         }
     }
 
-    private String toString(NodeType type) {
+    static String toString(NodeType type) {
         switch (type) {
             case tenant: return "tenant";
             case host: return "host";
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index 03777078251..54202a15971 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.restapi.v2;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.container.jdisc.HttpRequest;
@@ -91,7 +92,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
 
     private HttpResponse handleGET(HttpRequest request) {
         String path = request.getUri().getPath();
-        if (path.equals(    "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance");
+        if (path.equals(    "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance", "upgrade");
         if (path.equals(    "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository);
         if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository);
         if (path.equals(    "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository);
@@ -99,6 +100,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository);
         if (path.equals(    "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot");
         if (path.equals(    "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl());
+        if (path.equals(    "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions());
         throw new NotFoundException("Nothing at path '" + path + "'");
     }
 
@@ -135,10 +137,16 @@ public class NodesApiHandler extends LoggingRequestHandler {
 
     private HttpResponse handlePATCH(HttpRequest request) {
         String path = request.getUri().getPath();
-        if ( ! path.startsWith("/nodes/v2/node/")) throw new NotFoundException("Nothing at '" + path + "'");
-        Node node = nodeFromRequest(request);
-        nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository).apply());
-        return new MessageResponse("Updated " + node.hostname());
+        if (path.startsWith("/nodes/v2/node/")) {
+            Node node = nodeFromRequest(request);
+            nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository).apply());
+            return new MessageResponse("Updated " + node.hostname());
+        }
+        else if (path.startsWith("/nodes/v2/upgrade/")) {
+            return setInfrastructureVersion(request);
+        }
+
+        throw new NotFoundException("Nothing at '" + path + "'");
     }
 
     private HttpResponse handlePOST(HttpRequest request) {
@@ -276,4 +284,18 @@ public class NodesApiHandler extends LoggingRequestHandler {
         return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
     }
 
+    private MessageResponse setInfrastructureVersion(HttpRequest request) {
+        NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase());
+        Inspector inspector = toSlime(request.getData()).get();
+
+        Inspector versionField = inspector.field("version");
+        if (!versionField.valid())
+            throw new IllegalArgumentException("'version' is missing");
+        Version version = Version.fromString(versionField.asString());
+        boolean force = inspector.field("force").asBool();
+
+        maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force);
+
+        return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString());
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
new file mode 100644
index 00000000000..20a0139a178
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Comparator;
+import java.util.Map;
+
+/**
+ * A response containing infrastructure versions
+ *
+ * @author freva
+ */
+public class UpgradeResponse extends HttpResponse {
+
+    private final InfrastructureVersions infrastructureVersions;
+
+    public UpgradeResponse(InfrastructureVersions infrastructureVersions) {
+        super(200);
+        this.infrastructureVersions = infrastructureVersions;
+    }
+
+    @Override
+    public void render(OutputStream stream) throws IOException {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        Cursor versionsObject = root.setObject("versions");
+        infrastructureVersions.getTargetVersions().entrySet().stream()
+                .sorted(Comparator.comparing(Map.Entry::getKey)) // Sort for stable tests
+                .forEach(entry ->
+                        versionsObject.setString(entry.getKey().name(), entry.getValue().toFullString()));
+
+        new JsonFormat(true).encode(stream, slime);
+    }
+
+    @Override
+    public String getContentType() { return "application/json"; }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index c5067c0f959..6c43ed18645 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -16,6 +16,7 @@ public class ContainerConfig {
                         "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>" +
+                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
new file mode 100644
index 00000000000..af0ca7b6b75
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
@@ -0,0 +1,45 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.google.inject.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.transaction.NestedTransaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author freva
+ */
+public class MockProvisioner implements Provisioner {
+
+    @Inject
+    public MockProvisioner() {}
+
+    @Override
+    public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
+
+    }
+
+    @Override
+    public void remove(NestedTransaction transaction, ApplicationId application) {
+
+    }
+
+    @Override
+    public void restart(ApplicationId application, HostFilter filter) {
+
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
new file mode 100644
index 00000000000..586498619c6
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
@@ -0,0 +1,110 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
+
+import java.time.Duration;
+import java.util.Optional;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class InfrastructureProvisionerTest {
+
+    private final NodeRepositoryTester tester = new NodeRepositoryTester();
+
+    private final Provisioner provisioner = mock(Provisioner.class);
+    private final NodeRepository nodeRepository = tester.nodeRepository();
+    private final InfrastructureVersions infrastructureVersions = mock(InfrastructureVersions.class);
+    private final InfrastructureProvisioner infrastructureProvisioner = new InfrastructureProvisioner(
+            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()));
+
+    @Test
+    public void returns_version_if_usable_nodes_on_old_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.dirty, Optional.empty());
+        addNode(3, Node.State.active, Optional.of(oldVersion));
+
+        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(NodeType.config));
+    }
+
+    @Test
+    public void returns_version_if_has_usable_nodes_without_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.ready, Optional.empty());
+        addNode(3, Node.State.active, Optional.of(target));
+
+        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_usable_nodes_on_target_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(target));
+        addNode(4, Node.State.inactive, Optional.of(target));
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_no_usable_nodes() {
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(Version.fromString("6.123.456")));
+
+        // No nodes in node repo
+        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config));
+
+        // Add nodes in non-provisionable states
+        addNode(1, Node.State.dirty, Optional.empty());
+        addNode(2, Node.State.failed, Optional.empty());
+        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_target_version_not_set() {
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.empty());
+        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config));
+    }
+
+    private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) {
+        Node node = tester.addNode("id-" + id, "node-" + id, "default", NodeType.config);
+        Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> {
+            ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION;
+            ClusterSpec clusterSpec = ClusterSpec.from(application.getClusterType(), application.getClusterId(), ClusterSpec.Group.from(0), version);
+            ClusterMembership membership = ClusterMembership.from(clusterSpec, 1);
+            Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false);
+            return node.with(allocation);
+        });
+        return nodeRepository.database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty());
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java
new file mode 100644
index 00000000000..37b0097b17c
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java
@@ -0,0 +1,83 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author freva
+ */
+public class InfrastructureVersionsTest {
+
+    private final NodeRepositoryTester tester = new NodeRepositoryTester();
+    private final InfrastructureVersions infrastructureVersions =
+            new InfrastructureVersions(tester.nodeRepository().database());
+
+    private final Version version = Version.fromString("6.123.456");
+
+    @Test
+    public void can_only_downgrade_with_force() {
+        assertTrue(infrastructureVersions.getTargetVersions().isEmpty());
+
+        assertEquals(Optional.empty(), infrastructureVersions.getTargetVersionFor(NodeType.config));
+        infrastructureVersions.setTargetVersion(NodeType.config, version, false);
+        assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config));
+
+        // Upgrading to new version without force is fine
+        Version new_version = Version.fromString("6.123.457"); // version + 1
+        infrastructureVersions.setTargetVersion(NodeType.config, new_version, false);
+        assertEquals(Optional.of(new_version), infrastructureVersions.getTargetVersionFor(NodeType.config));
+
+        // Downgrading to old version without force fails
+        try {
+            infrastructureVersions.setTargetVersion(NodeType.config, version, false);
+            fail("Should not be able to downgrade without force");
+        } catch (IllegalArgumentException ignored) { }
+
+        infrastructureVersions.setTargetVersion(NodeType.config, version, true);
+        assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config));
+    }
+
+    @Test
+    public void can_only_set_version_on_certain_node_types() {
+        // We can set version for config
+        infrastructureVersions.setTargetVersion(NodeType.config, version, false);
+
+        try {
+            infrastructureVersions.setTargetVersion(NodeType.tenant, version, false);
+            fail("Should not be able to set version for tenant nodes");
+        } catch (IllegalArgumentException ignored) { }
+
+        try {
+            // Using 'force' does not help, force only applies to version downgrade
+            infrastructureVersions.setTargetVersion(NodeType.tenant, version, true);
+            fail("Should not be able to set version for tenant nodes");
+        } catch (IllegalArgumentException ignored) { }
+    }
+
+    @Test
+    public void can_store_multiple_versions() {
+        Version version2 = Version.fromString("6.456.123");
+
+        infrastructureVersions.setTargetVersion(NodeType.config, version, false);
+        infrastructureVersions.setTargetVersion(NodeType.confighost, version2, false);
+        infrastructureVersions.setTargetVersion(NodeType.proxyhost, version, false);
+
+        Map<NodeType, Version> expected = new HashMap<>();
+        expected.put(NodeType.config, version);
+        expected.put(NodeType.confighost, version2);
+        expected.put(NodeType.proxyhost, version);
+
+        assertEquals(expected, infrastructureVersions.getTargetVersions());
+    }
+}
\ No newline at end of file
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 2e69867e9b1..581b82e5fd5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -495,6 +495,44 @@ public class RestApiTest {
         assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json");
     }
 
+    @Test
+    public void test_upgrade() throws IOException {
+        // Initially, no versions are set
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}");
+
+        // Set version for config and confighost
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config",
+                        Utf8.toBytes("{\"version\": \"6.123.456\"}"),
+                        Request.Method.PATCH),
+                "{\"message\":\"Set version for config to 6.123.456\"}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                        Utf8.toBytes("{\"version\": \"6.123.456\"}"),
+                        Request.Method.PATCH),
+                "{\"message\":\"Set version for confighost to 6.123.456\"}");
+
+        // Verify versions are set
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
+                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}");
+
+        // Downgrade without force fails
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                        Utf8.toBytes("{\"version\": \"6.123.1\"}"),
+                        Request.Method.PATCH),
+                400,
+                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " +
+                        "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}");
+
+        // Downgrade with force is OK
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+                        Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"),
+                        Request.Method.PATCH),
+                "{\"message\":\"Set version for confighost to 6.123.1\"}");
+
+        // Verify version has been updated
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
+                "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}");
+    }
+
     /** Tests the rendering of each node separately to make it easier to find errors */
     @Test
     public void test_single_node_rendering() throws Exception {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
index 28e28f9678e..99cb9fd91f5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
@@ -34,6 +34,9 @@
       "name":"MetricsReporter"
     },
     {
+      "name":"InfrastructureProvisioner"
+    },
+    {
       "name":"NodeFailer"
     }
   ],
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json
index 4224718ab06..86becefb146 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json
@@ -11,6 +11,9 @@
     },
     {
       "url": "http://localhost:8080/nodes/v2/maintenance/"
+    },
+    {
+      "url":"http://localhost:8080/nodes/v2/upgrade/"
     }
   ]
 }
\ No newline at end of file