diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-08-13 12:47:04 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-13 12:47:04 +0200 |
commit | d442582895fcc8d01228cb57199c5ebc1e0768ef (patch) | |
tree | 8bde5c2e8b7ff1ad1440d7fc49073f1cbbbc1cf2 | |
parent | 031e0375afe6c92ad07812019f6eb6045527e9d1 (diff) | |
parent | b56ee11360cf9fd79af8b4bd5a086191af141a71 (diff) |
Merge pull request #6545 from vespa-engine/mpolden/node-repo-os-version
Add support for OS versions in node repo
16 files changed, 567 insertions, 56 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 5db79beac3c..69b31f506e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.provisioning.OsVersions; import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; import java.time.Clock; @@ -78,6 +79,7 @@ public class NodeRepository extends AbstractComponent { private final NodeFlavors flavors; private final NameResolver nameResolver; private final DockerImage dockerImage; + private final OsVersions osVersions; /** * Creates a node repository from a zookeeper provider. @@ -100,6 +102,7 @@ public class NodeRepository extends AbstractComponent { this.flavors = flavors; this.nameResolver = nameResolver; this.dockerImage = dockerImage; + this.osVersions = new OsVersions(this.db); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (Node.State state : Node.State.values()) @@ -115,6 +118,9 @@ public class NodeRepository extends AbstractComponent { /** @return The name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } + /** Returns the OS versions to use for nodes in this */ + public OsVersions osVersions() { return osVersions; } + // ---------------- Query API ---------------------------------------------------------------- /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 19e34ccb169..feaa4d8241d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -22,6 +22,7 @@ public class Status { private final boolean wantToRetire; private final boolean wantToDeprovision; private final Optional<String> hardwareDivergence; + private final Optional<Version> osVersion; public Status(Generation generation, Optional<Version> vespaVersion, @@ -29,50 +30,49 @@ public class Status { Optional<String> hardwareFailureDescription, boolean wantToRetire, boolean wantToDeprovision, - Optional<String> hardwareDivergence) { - Objects.requireNonNull(generation, "Generation must be non-null"); - Objects.requireNonNull(vespaVersion, "Vespa version must be non-null"); - Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null"); + Optional<String> hardwareDivergence, + Optional<Version> osVersion) { Objects.requireNonNull(hardwareDivergence, "Hardware divergence must be non-null"); hardwareDivergence.ifPresent(s -> requireNonEmptyString(s, "Hardware divergence must be non-empty")); - this.reboot = generation; - this.vespaVersion = vespaVersion; + this.reboot = Objects.requireNonNull(generation, "Generation must be non-null"); + this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null"); this.failCount = failCount; - this.hardwareFailureDescription = hardwareFailureDescription; + this.hardwareFailureDescription = Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null"); this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.hardwareDivergence = hardwareDivergence; + this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); } /** Returns a copy of this with the reboot generation changed */ - public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the reboot generation of this node */ public Generation reboot() { return reboot; } /** Returns a copy of this with the vespa version changed */ - public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the Vespa version installed on the node, if known */ public Optional<Version> vespaVersion() { return vespaVersion; } - public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } - public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } - public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the type of the last hardware failure detected on this node, or empty if none */ public Optional<String> hardwareFailureDescription() { return hardwareFailureDescription; } /** Returns a copy of this with the want to retire flag changed */ public Status withWantToRetire(boolean wantToRetire) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** @@ -85,7 +85,7 @@ public class Status { /** Returns a copy of this with the want to de-provision flag changed */ public Status withWantToDeprovision(boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** @@ -96,14 +96,27 @@ public class Status { } public Status withHardwareDivergence(Optional<String> hardwareDivergence) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns hardware divergence report as JSON string, if any */ public Optional<String> hardwareDivergence() { return hardwareDivergence; } + /** Returns a copy of this with the current OS version set to version */ + public Status withOsVersion(Version version) { + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, Optional.of(version)); + } + + /** Returns the current OS version of this node, if any */ + public Optional<Version> osVersion() { + return osVersion; + } + /** Returns the initial status of a newly provisioned node */ - public static Status initial() { return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false, false, Optional.empty()); } + public static Status initial() { + return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false, + false, Optional.empty(), Optional.empty()); + } private void requireNonEmptyString(String value, String message) { Objects.requireNonNull(value, message); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java new file mode 100644 index 00000000000..f7083a6398f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Objects; + +/** + * Filter nodes by their OS version. + * + * @author mpolden + */ +public class NodeOsVersionFilter extends NodeFilter { + + private final Version version; + + private NodeOsVersionFilter(Version version, NodeFilter next) { + super(next); + this.version = Objects.requireNonNull(version, "version cannot be null"); + } + + @Override + public boolean matches(Node node) { + if (!version.isEmpty() && !node.status().osVersion().filter(v -> v.equals(version)).isPresent()) { + return false; + } + return nextMatches(node); + } + + public static NodeOsVersionFilter from(String version, NodeFilter filter) { + return new NodeOsVersionFilter(Version.fromString(version), filter); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index f559ec0037b..a5dfc616302 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -71,6 +71,7 @@ public class CuratorDatabaseClient { curatorDatabase.create(toPath(state)); curatorDatabase.create(inactiveJobsPath()); curatorDatabase.create(infrastructureVersionsPath()); + curatorDatabase.create(osVersionsPath()); } /** @@ -374,4 +375,23 @@ public class CuratorDatabaseClient { return root.append("infrastructureVersions"); } + public Map<NodeType, Version> readOsVersions() { + return read(osVersionsPath(), NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new); + } + + public void writeOsVersions(Map<NodeType, Version> versions) { + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.setData(osVersionsPath().getAbsolute(), + NodeTypeVersionsSerializer.toJson(versions))); + transaction.commit(); + } + + public Lock lockOsVersions() { + return lock(lockRoot.append("osVersionsLock"), defaultLockTimeout); + } + + private Path osVersionsPath() { + return root.append("osVersions"); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 669f2063ee6..dbe6589dd7f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -58,6 +58,7 @@ public class NodeSerializer { private static final String wantToRetireKey = "wantToRetire"; private static final String wantToDeprovisionKey = "wantToDeprovision"; private static final String hardwareDivergenceKey = "hardwareDivergence"; + private static final String osVersionKey = "osVersion"; // Configuration fields private static final String flavorKey = "flavor"; @@ -114,6 +115,7 @@ public class NodeSerializer { object.setString(nodeTypeKey, toString(node.type())); node.status().hardwareDivergence().ifPresent(hardwareDivergence -> object.setString(hardwareDivergenceKey, hardwareDivergence)); + node.status().osVersion().ifPresent(version -> object.setString(osVersionKey, version.toString())); } private void toSlime(Allocation allocation, Cursor object) { @@ -169,7 +171,8 @@ public class NodeSerializer { hardwareFailureDescriptionFromSlime(object), object.field(wantToRetireKey).asBool(), object.field(wantToDeprovisionKey).asBool(), - removeQuotedNulls(hardwareDivergenceFromSlime(object))); + removeQuotedNulls(hardwareDivergenceFromSlime(object)), + versionFromSlime(object.field(osVersionKey))); } private Flavor flavorFromSlime(Inspector object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java new file mode 100644 index 00000000000..7e941d58a62 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java @@ -0,0 +1,91 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Thread-safe class that manages target OS versions for nodes in this repository. + * + * The target OS version for each node type is set through the /nodes/v2/upgrade REST API. + * + * @author mpolden + */ +public class OsVersions { + + private static final Duration defaultCacheTtl = Duration.ofMinutes(1); + private static final Logger log = Logger.getLogger(OsVersions.class.getName()); + + private final CuratorDatabaseClient db; + private final Duration cacheTtl; + + /** + * Target OS version is read on every request to /nodes/v2/node/[fqdn]. Cache current targets to avoid + * unnecessary ZK reads. When targets change, some nodes may need to wait for TTL until they see the new target, + * this is fine. + */ + private volatile Supplier<Map<NodeType, Version>> currentTargets; + + public OsVersions(CuratorDatabaseClient db) { + this(db, defaultCacheTtl); + } + + OsVersions(CuratorDatabaseClient db, Duration cacheTtl) { + this.db = db; + this.cacheTtl = cacheTtl; + createCache(); + } + + private void createCache() { + this.currentTargets = Suppliers.memoizeWithExpiration(() -> ImmutableMap.copyOf(db.readOsVersions()), + cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the current target versions for each node type */ + public Map<NodeType, Version> targets() { + return currentTargets.get(); + } + + /** Returns the current target version for given node type, if any */ + public Optional<Version> targetFor(NodeType type) { + return Optional.ofNullable(targets().get(type)); + } + + /** Set the target OS version for nodes of given type */ + public void setTarget(NodeType nodeType, Version newTarget, boolean force) { + if (!nodeType.isDockerHost()) { + throw new IllegalArgumentException("Setting target OS version for " + nodeType + " nodes is unsupported"); + } + try (Lock lock = db.lockOsVersions()) { + Map<NodeType, Version> osVersions = db.readOsVersions(); + Optional<Version> oldTarget = Optional.ofNullable(osVersions.get(nodeType)); + + if (oldTarget.filter(v -> v.equals(newTarget)).isPresent()) { + return; // Old target matches new target, nothing to do + } + + if (!force && oldTarget.filter(v -> v.isAfter(newTarget)).isPresent()) { + throw new IllegalArgumentException("Cannot set target OS version to " + newTarget + + " without setting 'force', as it's lower than the current version: " + + oldTarget.get()); + } + + osVersions.put(nodeType, newTarget); + db.writeOsVersions(osVersions); + createCache(); // Throw away current cache + log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString()); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 31d9a606d91..910da4e90bf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -114,6 +114,8 @@ public class NodePatcher { case "vespaVersion" : case "currentVespaVersion" : return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); + case "currentOsVersion" : + return node.with(node.status().withOsVersion(Version.fromString(asString(value)))); case "failCount" : return node.with(node.status().setFailCount(asLong(value).intValue())); case "flavor" : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 54202a15971..c282993a466 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.component.Version; import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -15,12 +16,12 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; @@ -28,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; +import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -37,7 +39,6 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; -import javax.inject.Inject; import static com.yahoo.vespa.config.SlimeUtils.optionalString; @@ -53,7 +54,6 @@ public class NodesApiHandler extends LoggingRequestHandler { private final NodeRepository nodeRepository; private final NodeRepositoryMaintenance maintenance; private final NodeFlavors nodeFlavors; - private static final String nodeTypeKey = "type"; @Inject public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, @@ -100,7 +100,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); - if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions()); + if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -114,18 +114,15 @@ public class NodesApiHandler extends LoggingRequestHandler { } else if (path.startsWith("/nodes/v2/state/failed/")) { List<Node> failedNodes = nodeRepository.failRecursively(lastElement(path), Agent.operator, "Failed through the nodes/v2 API"); - String failedHostnames = failedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + failedHostnames + " to failed"); + return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed"); } else if (path.startsWith("/nodes/v2/state/parked/")) { List<Node> parkedNodes = nodeRepository.parkRecursively(lastElement(path), Agent.operator, "Parked through the nodes/v2 API"); - String parkedHostnames = parkedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + parkedHostnames + " to parked"); + return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked"); } else if (path.startsWith("/nodes/v2/state/dirty/")) { List<Node> dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API"); - String dirtiedHostnames = dirtiedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + dirtiedHostnames + " to dirty"); + return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty"); } else if (path.startsWith("/nodes/v2/state/active/")) { nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API"); @@ -143,7 +140,7 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Updated " + node.hostname()); } else if (path.startsWith("/nodes/v2/upgrade/")) { - return setInfrastructureVersion(request); + return setTargetVersions(request); } throw new NotFoundException("Nothing at '" + path + "'"); @@ -227,10 +224,10 @@ public class NodesApiHandler extends LoggingRequestHandler { additionalIpAddresses, parentHostname, nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString()), - nodeTypeFromSlime(inspector.field(nodeTypeKey))); + nodeTypeFromSlime(inspector.field("type"))); } - private NodeType nodeTypeFromSlime(Inspector object) { + private static NodeType nodeTypeFromSlime(Inspector object) { if (! object.valid()) return NodeType.tenant; // default switch (object.asString()) { case "tenant" : return NodeType.tenant; @@ -252,18 +249,19 @@ public class NodesApiHandler extends LoggingRequestHandler { filter = StateFilter.from(request.getProperty("state"), filter); filter = NodeTypeFilter.from(request.getProperty("type"), filter); filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); + filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter); return filter; } - private String lastElement(String path) { + private static String lastElement(String path) { if (path.endsWith("/")) path = path.substring(0, path.length()-1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash + 1, path.length()); + return path.substring(lastSlash + 1); } - private boolean isPatchOverride(HttpRequest request) { + private static boolean isPatchOverride(HttpRequest request) { // Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests. String override = request.getHeader("X-HTTP-Method-Override"); if (override != null) { @@ -284,18 +282,37 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); } - private MessageResponse setInfrastructureVersion(HttpRequest request) { + private MessageResponse setTargetVersions(HttpRequest request) { NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); Inspector inspector = toSlime(request.getData()).get(); + List<String> messageParts = new ArrayList<>(2); - Inspector versionField = inspector.field("version"); - if (!versionField.valid()) - throw new IllegalArgumentException("'version' is missing"); - Version version = Version.fromString(versionField.asString()); boolean force = inspector.field("force").asBool(); + Inspector versionField = inspector.field("version"); + Inspector osVersionField = inspector.field("osVersion"); + + if (versionField.valid()) { + Version version = Version.fromString(versionField.asString()); + maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force); + messageParts.add("version to " + version.toFullString()); + } + + if (osVersionField.valid()) { + Version osVersion = Version.fromString(osVersionField.asString()); + nodeRepository.osVersions().setTarget(nodeType, osVersion, force); + messageParts.add("osVersion to " + osVersion.toFullString()); + } - maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force); + if (messageParts.isEmpty()) { + throw new IllegalArgumentException("At least one of 'version' and 'osVersion' must be set"); + } - return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString()); + return new MessageResponse("Set " + String.join(", ", messageParts) + + " for nodes of type " + nodeType); } + + private static String hostnamesAsString(List<Node> nodes) { + return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 99ebb3e517b..970871a4d05 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -169,6 +169,8 @@ class NodesResponse extends HttpResponse { } object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); + node.status().osVersion().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); + nodeRepository.osVersions().targetFor(node.type()).ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); node.status().vespaVersion() .filter(version -> !version.isEmpty()) .ifPresent(version -> { @@ -231,7 +233,7 @@ class NodesResponse extends HttpResponse { path = path.substring(0, path.length()-1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash+1, path.length()); + return path.substring(lastSlash+1); } private static Node.State stateFromString(String stateString) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java index 3fb712e182f..392cba7baa9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java @@ -6,24 +6,25 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; +import com.yahoo.vespa.hosted.provision.provisioning.OsVersions; import java.io.IOException; import java.io.OutputStream; -import java.util.Comparator; -import java.util.Map; /** - * A response containing infrastructure versions + * A response containing targets for infrastructure Vespa version and OS version. * * @author freva */ public class UpgradeResponse extends HttpResponse { private final InfrastructureVersions infrastructureVersions; + private final OsVersions osVersions; - public UpgradeResponse(InfrastructureVersions infrastructureVersions) { + public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions) { super(200); this.infrastructureVersions = infrastructureVersions; + this.osVersions = osVersions; } @Override @@ -34,6 +35,9 @@ public class UpgradeResponse extends HttpResponse { Cursor versionsObject = root.setObject("versions"); infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString())); + Cursor osVersionsObject = root.setObject("osVersions"); + osVersions.targets().forEach((nodeType, version) -> osVersionsObject.setString(nodeType.name(), version.toFullString())); + new JsonFormat(true).encode(stream, slime); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java new file mode 100644 index 00000000000..4639a86aeec --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import org.junit.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class NodeTypeVersionsSerializerTest { + + @Test + public void test_serialization() { + Map<NodeType, Version> versions = new TreeMap<>(); + versions.put(NodeType.host, Version.fromString("7.1")); + versions.put(NodeType.confighost, Version.fromString("7.2")); + + Map<NodeType, Version> serialized = NodeTypeVersionsSerializer.fromJson(NodeTypeVersionsSerializer.toJson(versions)); + assertEquals(versions, serialized); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 0c32c13f387..6c9d0be69b2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Arrays; @@ -240,7 +239,7 @@ public class SerializationTest { } @Test - public void serialize_additional_ip_addresses() throws IOException { + public void serialize_additional_ip_addresses() { Node node = createNode(); // Test round-trip with additional addresses @@ -326,7 +325,7 @@ public class SerializationTest { } @Test - public void vespa_version_serialization() throws Exception { + public void vespa_version_serialization() { String nodeWithWantedVespaVersion = "{\n" + " \"type\" : \"tenant\",\n" + @@ -343,6 +342,18 @@ public class SerializationTest { assertEquals("6.42.2", node.allocation().get().membership().cluster().vespaVersion().toString()); } + @Test + public void os_version_serialization() { + Node serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(createNode())); + assertFalse(serialized.status().osVersion().isPresent()); + + // Update OS version + serialized = serialized.with(serialized.status() + .withOsVersion(Version.fromString("7.1"))); + serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized)); + assertEquals(Version.fromString("7.1"), serialized.status().osVersion().get()); + } + private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; if (ipAddress.length > 0) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java new file mode 100644 index 00000000000..88f5dcb9854 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author mpolden + */ +public class OsVersionsTest { + + private OsVersions versions; + + @Before + public void before() { + versions = new OsVersions( + new NodeRepositoryTester().nodeRepository().database(), + Duration.ofDays(1) // Long TTL to avoid timed expiry during test + ); + } + + @Test + public void test_versions() { + assertTrue("No versions set", versions.targets().isEmpty()); + assertSame("Caches empty target versions", versions.targets(), versions.targets()); + + // Upgrade OS + Version version1 = Version.fromString("7.1"); + versions.setTarget(NodeType.host, version1, false); + Map<NodeType, Version> targetVersions = versions.targets(); + assertSame("Caches target versions", targetVersions, versions.targets()); + assertEquals(version1, versions.targetFor(NodeType.host).get()); + + // Upgrade OS again + Version version2 = Version.fromString("7.2"); + versions.setTarget(NodeType.host, version2, false); + assertNotSame("Cache invalidated", targetVersions, versions.targets()); + assertEquals(version2, versions.targetFor(NodeType.host).get()); + + // Downgrading fails + try { + versions.setTarget(NodeType.host, version1, false); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + // Forcing downgrade succeeds + versions.setTarget(NodeType.host, version1, true); + assertEquals(version1, versions.targetFor(NodeType.host).get()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index ef0feecc037..448b64d1e78 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -506,21 +506,21 @@ public class RestApiTest { @Test public void test_upgrade() throws IOException { // Initially, no versions are set - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{}}"); // Set version for config and confighost assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", Utf8.toBytes("{\"version\": \"6.123.456\"}"), Request.Method.PATCH), - "{\"message\":\"Set version for config to 6.123.456\"}"); + "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": \"6.123.456\"}"), Request.Method.PATCH), - "{\"message\":\"Set version for confighost to 6.123.456\"}"); + "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}"); // Verify versions are set assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"},\"osVersions\":{}}"); // Downgrade without force fails assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -534,11 +534,84 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"), Request.Method.PATCH), - "{\"message\":\"Set version for confighost to 6.123.1\"}"); + "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}"); // Verify version has been updated assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{}}"); + + // Upgrade OS for confighost and host + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // OS versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"}}"); + + // Upgrade OS and Vespa together + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); + + // Attempt to upgrade unsupported node type + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting target OS version for config nodes is unsupported\"}"); + + // Attempt to downgrade OS + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); + + // Downgrading OS with force succeeds + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); + } + + @Test + public void test_os_version() throws Exception { + // Schedule OS upgrade + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // Other node type does not return wanted OS version + Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); + assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); + + // Node updates its node object after upgrading OS + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); + + // Another node upgrades + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); + + // Filter nodes by osVersion + assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"), + "{\"nodes\":[" + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + + "]}"); } /** Tests the rendering of each node separately to make it easier to find errors */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json new file mode 100644 index 00000000000..2e8092012fb --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -0,0 +1,72 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "currentOsVersion": "7.5.2", + "wantedOsVersion": "7.5.2", + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json new file mode 100644 index 00000000000..88bda7544d9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json @@ -0,0 +1,71 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "wantedOsVersion": "7.5.2", + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} |