summaryrefslogtreecommitdiffstats
path: root/node-repository
diff options
context:
space:
mode:
authorMartin Polden <mpolden@mpolden.no>2018-08-13 12:47:04 +0200
committerGitHub <noreply@github.com>2018-08-13 12:47:04 +0200
commitd442582895fcc8d01228cb57199c5ebc1e0768ef (patch)
tree8bde5c2e8b7ff1ad1440d7fc49073f1cbbbc1cf2 /node-repository
parent031e0375afe6c92ad07812019f6eb6045527e9d1 (diff)
parentb56ee11360cf9fd79af8b4bd5a086191af141a71 (diff)
Merge pull request #6545 from vespa-engine/mpolden/node-repo-os-version
Add support for OS versions in node repo
Diffstat (limited to 'node-repository')
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java6
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java47
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java35
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java20
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java5
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java91
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java63
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java12
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java28
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java17
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java63
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java87
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json72
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json71
16 files changed, 567 insertions, 56 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 5db79beac3c..69b31f506e5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver;
import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
+import com.yahoo.vespa.hosted.provision.provisioning.OsVersions;
import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException;
import java.time.Clock;
@@ -78,6 +79,7 @@ public class NodeRepository extends AbstractComponent {
private final NodeFlavors flavors;
private final NameResolver nameResolver;
private final DockerImage dockerImage;
+ private final OsVersions osVersions;
/**
* Creates a node repository from a zookeeper provider.
@@ -100,6 +102,7 @@ public class NodeRepository extends AbstractComponent {
this.flavors = flavors;
this.nameResolver = nameResolver;
this.dockerImage = dockerImage;
+ this.osVersions = new OsVersions(this.db);
// read and write all nodes to make sure they are stored in the latest version of the serialized format
for (Node.State state : Node.State.values())
@@ -115,6 +118,9 @@ public class NodeRepository extends AbstractComponent {
/** @return The name resolver used to resolve hostname and ip addresses */
public NameResolver nameResolver() { return nameResolver; }
+ /** Returns the OS versions to use for nodes in this */
+ public OsVersions osVersions() { return osVersions; }
+
// ---------------- Query API ----------------------------------------------------------------
/**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
index 19e34ccb169..feaa4d8241d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -22,6 +22,7 @@ public class Status {
private final boolean wantToRetire;
private final boolean wantToDeprovision;
private final Optional<String> hardwareDivergence;
+ private final Optional<Version> osVersion;
public Status(Generation generation,
Optional<Version> vespaVersion,
@@ -29,50 +30,49 @@ public class Status {
Optional<String> hardwareFailureDescription,
boolean wantToRetire,
boolean wantToDeprovision,
- Optional<String> hardwareDivergence) {
- Objects.requireNonNull(generation, "Generation must be non-null");
- Objects.requireNonNull(vespaVersion, "Vespa version must be non-null");
- Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null");
+ Optional<String> hardwareDivergence,
+ Optional<Version> osVersion) {
Objects.requireNonNull(hardwareDivergence, "Hardware divergence must be non-null");
hardwareDivergence.ifPresent(s -> requireNonEmptyString(s, "Hardware divergence must be non-empty"));
- this.reboot = generation;
- this.vespaVersion = vespaVersion;
+ this.reboot = Objects.requireNonNull(generation, "Generation must be non-null");
+ this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null");
this.failCount = failCount;
- this.hardwareFailureDescription = hardwareFailureDescription;
+ this.hardwareFailureDescription = Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null");
this.wantToRetire = wantToRetire;
this.wantToDeprovision = wantToDeprovision;
this.hardwareDivergence = hardwareDivergence;
+ this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null");
}
/** Returns a copy of this with the reboot generation changed */
- public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
/** Returns the reboot generation of this node */
public Generation reboot() { return reboot; }
/** Returns a copy of this with the vespa version changed */
- public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
/** Returns the Vespa version installed on the node, if known */
public Optional<Version> vespaVersion() { return vespaVersion; }
- public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
- public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
- public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
/** Returns how many times this node has been moved to the failed state. */
public int failCount() { return failCount; }
- public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); }
+ public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
/** Returns the type of the last hardware failure detected on this node, or empty if none */
public Optional<String> hardwareFailureDescription() { return hardwareFailureDescription; }
/** Returns a copy of this with the want to retire flag changed */
public Status withWantToRetire(boolean wantToRetire) {
- return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence);
+ return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
}
/**
@@ -85,7 +85,7 @@ public class Status {
/** Returns a copy of this with the want to de-provision flag changed */
public Status withWantToDeprovision(boolean wantToDeprovision) {
- return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence);
+ return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
}
/**
@@ -96,14 +96,27 @@ public class Status {
}
public Status withHardwareDivergence(Optional<String> hardwareDivergence) {
- return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence);
+ return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
}
/** Returns hardware divergence report as JSON string, if any */
public Optional<String> hardwareDivergence() { return hardwareDivergence; }
+ /** Returns a copy of this with the current OS version set to version */
+ public Status withOsVersion(Version version) {
+ return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, Optional.of(version));
+ }
+
+ /** Returns the current OS version of this node, if any */
+ public Optional<Version> osVersion() {
+ return osVersion;
+ }
+
/** Returns the initial status of a newly provisioned node */
- public static Status initial() { return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false, false, Optional.empty()); }
+ public static Status initial() {
+ return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false,
+ false, Optional.empty(), Optional.empty());
+ }
private void requireNonEmptyString(String value, String message) {
Objects.requireNonNull(value, message);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java
new file mode 100644
index 00000000000..f7083a6398f
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java
@@ -0,0 +1,35 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node.filter;
+
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Objects;
+
+/**
+ * Filter nodes by their OS version.
+ *
+ * @author mpolden
+ */
+public class NodeOsVersionFilter extends NodeFilter {
+
+ private final Version version;
+
+ private NodeOsVersionFilter(Version version, NodeFilter next) {
+ super(next);
+ this.version = Objects.requireNonNull(version, "version cannot be null");
+ }
+
+ @Override
+ public boolean matches(Node node) {
+ if (!version.isEmpty() && !node.status().osVersion().filter(v -> v.equals(version)).isPresent()) {
+ return false;
+ }
+ return nextMatches(node);
+ }
+
+ public static NodeOsVersionFilter from(String version, NodeFilter filter) {
+ return new NodeOsVersionFilter(Version.fromString(version), filter);
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index f559ec0037b..a5dfc616302 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -71,6 +71,7 @@ public class CuratorDatabaseClient {
curatorDatabase.create(toPath(state));
curatorDatabase.create(inactiveJobsPath());
curatorDatabase.create(infrastructureVersionsPath());
+ curatorDatabase.create(osVersionsPath());
}
/**
@@ -374,4 +375,23 @@ public class CuratorDatabaseClient {
return root.append("infrastructureVersions");
}
+ public Map<NodeType, Version> readOsVersions() {
+ return read(osVersionsPath(), NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new);
+ }
+
+ public void writeOsVersions(Map<NodeType, Version> versions) {
+ NestedTransaction transaction = new NestedTransaction();
+ CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+ curatorTransaction.add(CuratorOperations.setData(osVersionsPath().getAbsolute(),
+ NodeTypeVersionsSerializer.toJson(versions)));
+ transaction.commit();
+ }
+
+ public Lock lockOsVersions() {
+ return lock(lockRoot.append("osVersionsLock"), defaultLockTimeout);
+ }
+
+ private Path osVersionsPath() {
+ return root.append("osVersions");
+ }
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 669f2063ee6..dbe6589dd7f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -58,6 +58,7 @@ public class NodeSerializer {
private static final String wantToRetireKey = "wantToRetire";
private static final String wantToDeprovisionKey = "wantToDeprovision";
private static final String hardwareDivergenceKey = "hardwareDivergence";
+ private static final String osVersionKey = "osVersion";
// Configuration fields
private static final String flavorKey = "flavor";
@@ -114,6 +115,7 @@ public class NodeSerializer {
object.setString(nodeTypeKey, toString(node.type()));
node.status().hardwareDivergence().ifPresent(hardwareDivergence -> object.setString(hardwareDivergenceKey,
hardwareDivergence));
+ node.status().osVersion().ifPresent(version -> object.setString(osVersionKey, version.toString()));
}
private void toSlime(Allocation allocation, Cursor object) {
@@ -169,7 +171,8 @@ public class NodeSerializer {
hardwareFailureDescriptionFromSlime(object),
object.field(wantToRetireKey).asBool(),
object.field(wantToDeprovisionKey).asBool(),
- removeQuotedNulls(hardwareDivergenceFromSlime(object)));
+ removeQuotedNulls(hardwareDivergenceFromSlime(object)),
+ versionFromSlime(object.field(osVersionKey)));
}
private Flavor flavorFromSlime(Inspector object) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java
new file mode 100644
index 00000000000..7e941d58a62
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java
@@ -0,0 +1,91 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * Thread-safe class that manages target OS versions for nodes in this repository.
+ *
+ * The target OS version for each node type is set through the /nodes/v2/upgrade REST API.
+ *
+ * @author mpolden
+ */
+public class OsVersions {
+
+ private static final Duration defaultCacheTtl = Duration.ofMinutes(1);
+ private static final Logger log = Logger.getLogger(OsVersions.class.getName());
+
+ private final CuratorDatabaseClient db;
+ private final Duration cacheTtl;
+
+ /**
+ * Target OS version is read on every request to /nodes/v2/node/[fqdn]. Cache current targets to avoid
+ * unnecessary ZK reads. When targets change, some nodes may need to wait for TTL until they see the new target,
+ * this is fine.
+ */
+ private volatile Supplier<Map<NodeType, Version>> currentTargets;
+
+ public OsVersions(CuratorDatabaseClient db) {
+ this(db, defaultCacheTtl);
+ }
+
+ OsVersions(CuratorDatabaseClient db, Duration cacheTtl) {
+ this.db = db;
+ this.cacheTtl = cacheTtl;
+ createCache();
+ }
+
+ private void createCache() {
+ this.currentTargets = Suppliers.memoizeWithExpiration(() -> ImmutableMap.copyOf(db.readOsVersions()),
+ cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
+ }
+
+ /** Returns the current target versions for each node type */
+ public Map<NodeType, Version> targets() {
+ return currentTargets.get();
+ }
+
+ /** Returns the current target version for given node type, if any */
+ public Optional<Version> targetFor(NodeType type) {
+ return Optional.ofNullable(targets().get(type));
+ }
+
+ /** Set the target OS version for nodes of given type */
+ public void setTarget(NodeType nodeType, Version newTarget, boolean force) {
+ if (!nodeType.isDockerHost()) {
+ throw new IllegalArgumentException("Setting target OS version for " + nodeType + " nodes is unsupported");
+ }
+ try (Lock lock = db.lockOsVersions()) {
+ Map<NodeType, Version> osVersions = db.readOsVersions();
+ Optional<Version> oldTarget = Optional.ofNullable(osVersions.get(nodeType));
+
+ if (oldTarget.filter(v -> v.equals(newTarget)).isPresent()) {
+ return; // Old target matches new target, nothing to do
+ }
+
+ if (!force && oldTarget.filter(v -> v.isAfter(newTarget)).isPresent()) {
+ throw new IllegalArgumentException("Cannot set target OS version to " + newTarget +
+ " without setting 'force', as it's lower than the current version: "
+ + oldTarget.get());
+ }
+
+ osVersions.put(nodeType, newTarget);
+ db.writeOsVersions(osVersions);
+ createCache(); // Throw away current cache
+ log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString());
+ }
+ }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 31d9a606d91..910da4e90bf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -114,6 +114,8 @@ public class NodePatcher {
case "vespaVersion" :
case "currentVespaVersion" :
return node.with(node.status().withVespaVersion(Version.fromString(asString(value))));
+ case "currentOsVersion" :
+ return node.with(node.status().withOsVersion(Version.fromString(asString(value))));
case "failCount" :
return node.with(node.status().setFailCount(asLong(value).intValue()));
case "flavor" :
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index 54202a15971..c282993a466 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2;
import com.yahoo.component.Version;
import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.NodeType;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
@@ -15,12 +16,12 @@ import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter;
import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter;
import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter;
import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
@@ -28,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType;
import com.yahoo.vespa.orchestrator.Orchestrator;
import com.yahoo.yolean.Exceptions;
+import javax.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -37,7 +39,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;
-import javax.inject.Inject;
import static com.yahoo.vespa.config.SlimeUtils.optionalString;
@@ -53,7 +54,6 @@ public class NodesApiHandler extends LoggingRequestHandler {
private final NodeRepository nodeRepository;
private final NodeRepositoryMaintenance maintenance;
private final NodeFlavors nodeFlavors;
- private static final String nodeTypeKey = "type";
@Inject
public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator,
@@ -100,7 +100,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository);
if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot");
if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl());
- if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions());
+ if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions());
throw new NotFoundException("Nothing at path '" + path + "'");
}
@@ -114,18 +114,15 @@ public class NodesApiHandler extends LoggingRequestHandler {
}
else if (path.startsWith("/nodes/v2/state/failed/")) {
List<Node> failedNodes = nodeRepository.failRecursively(lastElement(path), Agent.operator, "Failed through the nodes/v2 API");
- String failedHostnames = failedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", "));
- return new MessageResponse("Moved " + failedHostnames + " to failed");
+ return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed");
}
else if (path.startsWith("/nodes/v2/state/parked/")) {
List<Node> parkedNodes = nodeRepository.parkRecursively(lastElement(path), Agent.operator, "Parked through the nodes/v2 API");
- String parkedHostnames = parkedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", "));
- return new MessageResponse("Moved " + parkedHostnames + " to parked");
+ return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked");
}
else if (path.startsWith("/nodes/v2/state/dirty/")) {
List<Node> dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API");
- String dirtiedHostnames = dirtiedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", "));
- return new MessageResponse("Moved " + dirtiedHostnames + " to dirty");
+ return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty");
}
else if (path.startsWith("/nodes/v2/state/active/")) {
nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API");
@@ -143,7 +140,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
return new MessageResponse("Updated " + node.hostname());
}
else if (path.startsWith("/nodes/v2/upgrade/")) {
- return setInfrastructureVersion(request);
+ return setTargetVersions(request);
}
throw new NotFoundException("Nothing at '" + path + "'");
@@ -227,10 +224,10 @@ public class NodesApiHandler extends LoggingRequestHandler {
additionalIpAddresses,
parentHostname,
nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString()),
- nodeTypeFromSlime(inspector.field(nodeTypeKey)));
+ nodeTypeFromSlime(inspector.field("type")));
}
- private NodeType nodeTypeFromSlime(Inspector object) {
+ private static NodeType nodeTypeFromSlime(Inspector object) {
if (! object.valid()) return NodeType.tenant; // default
switch (object.asString()) {
case "tenant" : return NodeType.tenant;
@@ -252,18 +249,19 @@ public class NodesApiHandler extends LoggingRequestHandler {
filter = StateFilter.from(request.getProperty("state"), filter);
filter = NodeTypeFilter.from(request.getProperty("type"), filter);
filter = ParentHostFilter.from(request.getProperty("parentHost"), filter);
+ filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter);
return filter;
}
- private String lastElement(String path) {
+ private static String lastElement(String path) {
if (path.endsWith("/"))
path = path.substring(0, path.length()-1);
int lastSlash = path.lastIndexOf("/");
if (lastSlash < 0) return path;
- return path.substring(lastSlash + 1, path.length());
+ return path.substring(lastSlash + 1);
}
- private boolean isPatchOverride(HttpRequest request) {
+ private static boolean isPatchOverride(HttpRequest request) {
// Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests.
String override = request.getHeader("X-HTTP-Method-Override");
if (override != null) {
@@ -284,18 +282,37 @@ public class NodesApiHandler extends LoggingRequestHandler {
return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
}
- private MessageResponse setInfrastructureVersion(HttpRequest request) {
+ private MessageResponse setTargetVersions(HttpRequest request) {
NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase());
Inspector inspector = toSlime(request.getData()).get();
+ List<String> messageParts = new ArrayList<>(2);
- Inspector versionField = inspector.field("version");
- if (!versionField.valid())
- throw new IllegalArgumentException("'version' is missing");
- Version version = Version.fromString(versionField.asString());
boolean force = inspector.field("force").asBool();
+ Inspector versionField = inspector.field("version");
+ Inspector osVersionField = inspector.field("osVersion");
+
+ if (versionField.valid()) {
+ Version version = Version.fromString(versionField.asString());
+ maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force);
+ messageParts.add("version to " + version.toFullString());
+ }
+
+ if (osVersionField.valid()) {
+ Version osVersion = Version.fromString(osVersionField.asString());
+ nodeRepository.osVersions().setTarget(nodeType, osVersion, force);
+ messageParts.add("osVersion to " + osVersion.toFullString());
+ }
- maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force);
+ if (messageParts.isEmpty()) {
+ throw new IllegalArgumentException("At least one of 'version' and 'osVersion' must be set");
+ }
- return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString());
+ return new MessageResponse("Set " + String.join(", ", messageParts) +
+ " for nodes of type " + nodeType);
}
+
+ private static String hostnamesAsString(List<Node> nodes) {
+ return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", "));
+ }
+
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 99ebb3e517b..970871a4d05 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -169,6 +169,8 @@ class NodesResponse extends HttpResponse {
}
object.setLong("rebootGeneration", node.status().reboot().wanted());
object.setLong("currentRebootGeneration", node.status().reboot().current());
+ node.status().osVersion().ifPresent(version -> object.setString("currentOsVersion", version.toFullString()));
+ nodeRepository.osVersions().targetFor(node.type()).ifPresent(version -> object.setString("wantedOsVersion", version.toFullString()));
node.status().vespaVersion()
.filter(version -> !version.isEmpty())
.ifPresent(version -> {
@@ -231,7 +233,7 @@ class NodesResponse extends HttpResponse {
path = path.substring(0, path.length()-1);
int lastSlash = path.lastIndexOf("/");
if (lastSlash < 0) return path;
- return path.substring(lastSlash+1, path.length());
+ return path.substring(lastSlash+1);
}
private static Node.State stateFromString(String stateString) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
index 3fb712e182f..392cba7baa9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java
@@ -6,24 +6,25 @@ import com.yahoo.slime.Cursor;
import com.yahoo.slime.JsonFormat;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
+import com.yahoo.vespa.hosted.provision.provisioning.OsVersions;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.Comparator;
-import java.util.Map;
/**
- * A response containing infrastructure versions
+ * A response containing targets for infrastructure Vespa version and OS version.
*
* @author freva
*/
public class UpgradeResponse extends HttpResponse {
private final InfrastructureVersions infrastructureVersions;
+ private final OsVersions osVersions;
- public UpgradeResponse(InfrastructureVersions infrastructureVersions) {
+ public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions) {
super(200);
this.infrastructureVersions = infrastructureVersions;
+ this.osVersions = osVersions;
}
@Override
@@ -34,6 +35,9 @@ public class UpgradeResponse extends HttpResponse {
Cursor versionsObject = root.setObject("versions");
infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString()));
+ Cursor osVersionsObject = root.setObject("osVersions");
+ osVersions.targets().forEach((nodeType, version) -> osVersionsObject.setString(nodeType.name(), version.toFullString()));
+
new JsonFormat(true).encode(stream, slime);
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java
new file mode 100644
index 00000000000..4639a86aeec
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java
@@ -0,0 +1,28 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class NodeTypeVersionsSerializerTest {
+
+ @Test
+ public void test_serialization() {
+ Map<NodeType, Version> versions = new TreeMap<>();
+ versions.put(NodeType.host, Version.fromString("7.1"));
+ versions.put(NodeType.confighost, Version.fromString("7.2"));
+
+ Map<NodeType, Version> serialized = NodeTypeVersionsSerializer.fromJson(NodeTypeVersionsSerializer.toJson(versions));
+ assertEquals(versions, serialized);
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 0c32c13f387..6c9d0be69b2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.provision.node.History;
import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
import org.junit.Test;
-import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Arrays;
@@ -240,7 +239,7 @@ public class SerializationTest {
}
@Test
- public void serialize_additional_ip_addresses() throws IOException {
+ public void serialize_additional_ip_addresses() {
Node node = createNode();
// Test round-trip with additional addresses
@@ -326,7 +325,7 @@ public class SerializationTest {
}
@Test
- public void vespa_version_serialization() throws Exception {
+ public void vespa_version_serialization() {
String nodeWithWantedVespaVersion =
"{\n" +
" \"type\" : \"tenant\",\n" +
@@ -343,6 +342,18 @@ public class SerializationTest {
assertEquals("6.42.2", node.allocation().get().membership().cluster().vespaVersion().toString());
}
+ @Test
+ public void os_version_serialization() {
+ Node serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(createNode()));
+ assertFalse(serialized.status().osVersion().isPresent());
+
+ // Update OS version
+ serialized = serialized.with(serialized.status()
+ .withOsVersion(Version.fromString("7.1")));
+ serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized));
+ assertEquals(Version.fromString("7.1"), serialized.status().osVersion().get());
+ }
+
private byte[] createNodeJson(String hostname, String... ipAddress) {
String ipAddressJsonPart = "";
if (ipAddress.length > 0) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java
new file mode 100644
index 00000000000..88f5dcb9854
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author mpolden
+ */
+public class OsVersionsTest {
+
+ private OsVersions versions;
+
+ @Before
+ public void before() {
+ versions = new OsVersions(
+ new NodeRepositoryTester().nodeRepository().database(),
+ Duration.ofDays(1) // Long TTL to avoid timed expiry during test
+ );
+ }
+
+ @Test
+ public void test_versions() {
+ assertTrue("No versions set", versions.targets().isEmpty());
+ assertSame("Caches empty target versions", versions.targets(), versions.targets());
+
+ // Upgrade OS
+ Version version1 = Version.fromString("7.1");
+ versions.setTarget(NodeType.host, version1, false);
+ Map<NodeType, Version> targetVersions = versions.targets();
+ assertSame("Caches target versions", targetVersions, versions.targets());
+ assertEquals(version1, versions.targetFor(NodeType.host).get());
+
+ // Upgrade OS again
+ Version version2 = Version.fromString("7.2");
+ versions.setTarget(NodeType.host, version2, false);
+ assertNotSame("Cache invalidated", targetVersions, versions.targets());
+ assertEquals(version2, versions.targetFor(NodeType.host).get());
+
+ // Downgrading fails
+ try {
+ versions.setTarget(NodeType.host, version1, false);
+ fail("Expected exception");
+ } catch (IllegalArgumentException ignored) {}
+
+ // Forcing downgrade succeeds
+ versions.setTarget(NodeType.host, version1, true);
+ assertEquals(version1, versions.targetFor(NodeType.host).get());
+ }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index ef0feecc037..448b64d1e78 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -21,12 +21,12 @@ import java.io.IOException;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
-import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@@ -506,21 +506,21 @@ public class RestApiTest {
@Test
public void test_upgrade() throws IOException {
// Initially, no versions are set
- assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{}}");
// Set version for config and confighost
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config",
Utf8.toBytes("{\"version\": \"6.123.456\"}"),
Request.Method.PATCH),
- "{\"message\":\"Set version for config to 6.123.456\"}");
+ "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}");
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
Utf8.toBytes("{\"version\": \"6.123.456\"}"),
Request.Method.PATCH),
- "{\"message\":\"Set version for confighost to 6.123.456\"}");
+ "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}");
// Verify versions are set
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
- "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}");
+ "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"},\"osVersions\":{}}");
// Downgrade without force fails
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
@@ -534,11 +534,84 @@ public class RestApiTest {
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"),
Request.Method.PATCH),
- "{\"message\":\"Set version for confighost to 6.123.1\"}");
+ "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}");
// Verify version has been updated
assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
- "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}");
+ "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{}}");
+
+ // Upgrade OS for confighost and host
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+ Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}");
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host",
+ Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}");
+
+ // OS versions are set
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"),
+ "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"}}");
+
+ // Upgrade OS and Vespa together
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+ Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}");
+
+ // Attempt to upgrade unsupported node type
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config",
+ Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ 400,
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting target OS version for config nodes is unsupported\"}");
+
+ // Attempt to downgrade OS
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+ Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"),
+ Request.Method.PATCH),
+ 400,
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}");
+
+ // Downgrading OS with force succeeds
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost",
+ Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}");
+ }
+
+ @Test
+ public void test_os_version() throws Exception {
+ // Schedule OS upgrade
+ assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host",
+ Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}");
+
+ // Other node type does not return wanted OS version
+ Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"));
+ assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion"));
+
+ // Node updates its node object after upgrading OS
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+ Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
+ assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json");
+
+ // Another node upgrades
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com",
+ Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"),
+ Request.Method.PATCH),
+ "{\"message\":\"Updated dockerhost2.yahoo.com\"}");
+
+ // Filter nodes by osVersion
+ assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"),
+ "{\"nodes\":[" +
+ "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," +
+ "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" +
+ "]}");
}
/** Tests the rendering of each node separately to make it easier to find errors */
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json
new file mode 100644
index 00000000000..2e8092012fb
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json
@@ -0,0 +1,72 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+ "id": "dockerhost1.yahoo.com",
+ "state": "active",
+ "type": "host",
+ "hostname": "dockerhost1.yahoo.com",
+ "openStackId": "dockerhost1",
+ "flavor": "large",
+ "canonicalFlavor": "large",
+ "minDiskAvailableGb": 1600.0,
+ "minMainMemoryAvailableGb": 32.0,
+ "description": "Flavor-name-is-large",
+ "minCpuCores": 4.0,
+ "fastDisk": true,
+ "environment": "BARE_METAL",
+ "owner": {
+ "tenant": "zoneapp",
+ "application": "zoneapp",
+ "instance": "zoneapp"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "node-admin",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion": "6.42.0",
+ "allowedToBeDown": false,
+ "rebootGeneration": 1,
+ "currentRebootGeneration": 0,
+ "currentOsVersion": "7.5.2",
+ "wantedOsVersion": "7.5.2",
+ "failCount": 0,
+ "hardwareFailure": false,
+ "wantToRetire": false,
+ "wantToDeprovision": false,
+ "history": [
+ {
+ "event": "provisioned",
+ "at": 123,
+ "agent": "system"
+ },
+ {
+ "event": "readied",
+ "at": 123,
+ "agent": "system"
+ },
+ {
+ "event": "reserved",
+ "at": 123,
+ "agent": "application"
+ },
+ {
+ "event": "activated",
+ "at": 123,
+ "agent": "application"
+ }
+ ],
+ "ipAddresses": [
+ "::1",
+ "127.0.0.1"
+ ],
+ "additionalIpAddresses": [
+ "::2",
+ "::3",
+ "::4"
+ ]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json
new file mode 100644
index 00000000000..88bda7544d9
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json
@@ -0,0 +1,71 @@
+{
+ "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+ "id": "dockerhost1.yahoo.com",
+ "state": "active",
+ "type": "host",
+ "hostname": "dockerhost1.yahoo.com",
+ "openStackId": "dockerhost1",
+ "flavor": "large",
+ "canonicalFlavor": "large",
+ "minDiskAvailableGb": 1600.0,
+ "minMainMemoryAvailableGb": 32.0,
+ "description": "Flavor-name-is-large",
+ "minCpuCores": 4.0,
+ "fastDisk": true,
+ "environment": "BARE_METAL",
+ "owner": {
+ "tenant": "zoneapp",
+ "application": "zoneapp",
+ "instance": "zoneapp"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "node-admin",
+ "group": "0",
+ "index": 0,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+ "wantedVespaVersion": "6.42.0",
+ "allowedToBeDown": false,
+ "rebootGeneration": 1,
+ "currentRebootGeneration": 0,
+ "wantedOsVersion": "7.5.2",
+ "failCount": 0,
+ "hardwareFailure": false,
+ "wantToRetire": false,
+ "wantToDeprovision": false,
+ "history": [
+ {
+ "event": "provisioned",
+ "at": 123,
+ "agent": "system"
+ },
+ {
+ "event": "readied",
+ "at": 123,
+ "agent": "system"
+ },
+ {
+ "event": "reserved",
+ "at": 123,
+ "agent": "application"
+ },
+ {
+ "event": "activated",
+ "at": 123,
+ "agent": "application"
+ }
+ ],
+ "ipAddresses": [
+ "::1",
+ "127.0.0.1"
+ ],
+ "additionalIpAddresses": [
+ "::2",
+ "::3",
+ "::4"
+ ]
+}