From dbed0aaf832b9dc5b1376aa2724ddcf1125fe31f Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 29 May 2020 15:20:00 +0200 Subject: Require wantToRetire when setting wantToDeprovision --- .../com/yahoo/vespa/hosted/provision/Node.java | 18 +++- .../vespa/hosted/provision/NodeRepository.java | 2 +- .../yahoo/vespa/hosted/provision/node/Status.java | 14 +-- .../hosted/provision/os/RetiringUpgrader.java | 8 +- .../hosted/provision/restapi/NodePatcher.java | 38 ++++---- .../provision/testutils/MockNodeRepository.java | 2 +- .../vespa/hosted/provision/NodeRepositoryTest.java | 5 +- .../maintenance/InactiveAndFailedExpirerTest.java | 6 +- .../provisioning/InPlaceResizeProvisionTest.java | 2 +- .../provisioning/NodeTypeProvisioningTest.java | 4 +- .../provision/provisioning/ProvisioningTest.java | 6 +- .../hosted/provision/restapi/NodesV2ApiTest.java | 43 +++++---- .../restapi/responses/docker-node1-reports-2.json | 102 ++++++++++++++++++++ .../restapi/responses/docker-node1-reports-3.json | 90 ++++++++++++++++++ .../restapi/responses/docker-node1-reports-4.json | 85 +++++++++++++++++ .../restapi/responses/docker-node1-reports.json | 105 +++++++++++++++++++++ .../restapi/responses/node6-reports-2.json | 78 --------------- .../restapi/responses/node6-reports-3.json | 66 ------------- .../provision/restapi/responses/node6-reports.json | 81 ---------------- 19 files changed, 460 insertions(+), 295 deletions(-) create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json (limited to 'node-repository') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 45ed8db3491..a9861497ca3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -168,17 +168,25 @@ public final class Node { public Optional reservedTo() { return reservedTo; } /** - * Returns a copy of this node with wantToRetire set to the given value and updated history. - * If given wantToRetire is equal to the current, the method is no-op. + * Returns a copy of this node with wantToRetire and wantToDeprovision set to the given values and updated history. + * + * If both given wantToRetire and wantToDeprovision are equal to the current values, the method is no-op. */ - public Node withWantToRetire(boolean wantToRetire, Agent agent, Instant at) { - if (wantToRetire == status.wantToRetire()) return this; - Node node = this.with(status.withWantToRetire(wantToRetire)); + public Node withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, Agent agent, Instant at) { + if (!type.isDockerHost() && wantToDeprovision) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); + if (wantToRetire == status.wantToRetire() && + wantToDeprovision == status.wantToDeprovision()) return this; + Node node = this.with(status.withWantToRetire(wantToRetire, wantToDeprovision)); if (wantToRetire) node = node.with(history.with(new History.Event(History.Event.Type.wantToRetire, agent, at))); return node; } + public Node withWantToRetire(boolean wantToRetire, Agent agent, Instant at) { + return withWantToRetire(wantToRetire, status.wantToDeprovision(), agent, at); + } + /** * Returns a copy of this node which is retired. * If the node was already retired it is returned as-is. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index cc485374340..b5e36abd076 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -434,7 +434,7 @@ public class NodeRepository extends AbstractComponent { .map(node -> { if (node.state() != State.provisioned && node.state() != State.dirty) illegal("Can not set " + node + " ready. It is not provisioned or dirty."); - return node.with(node.status().withWantToRetire(false).withWantToDeprovision(false)); + return node.withWantToRetire(false, false, Agent.system, clock.instant()); }) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 6b52bd68e73..c289edfc19e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -36,6 +36,11 @@ public class Status { this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null").filter(v -> !Version.emptyVersion.equals(v)); this.dockerImage = Objects.requireNonNull(dockerImage, "Docker image must be non-null").filter(d -> !DockerImage.EMPTY.equals(d)); this.failCount = failCount; + if (wantToDeprovision && !wantToRetire) { + // TODO(mpolden): Throw when persisted nodes have been rewritten + wantToRetire = true; + //throw new IllegalArgumentException("Node cannot be marked wantToDeprovision unless it's also marked wantToRetire"); + } this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); @@ -69,8 +74,8 @@ public class Status { /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - /** Returns a copy of this with the want to retire flag changed */ - public Status withWantToRetire(boolean wantToRetire) { + /** Returns a copy of this with the want to retire/deprovision flags changed */ + public Status withWantToRetire(boolean wantToRetire, boolean wantToDeprovision) { return new Status(reboot, vespaVersion, dockerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } @@ -82,11 +87,6 @@ public class Status { return wantToRetire; } - /** Returns a copy of this with the want to de-provision flag changed */ - public Status withWantToDeprovision(boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, dockerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); - } - /** * Returns whether this node should be de-provisioned when possible. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java index 6bd2545b153..b2b83b6d064 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java @@ -72,10 +72,10 @@ public class RetiringUpgrader implements Upgrader { LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " + host.status().osVersion().current().map(Version::toFullString).orElse("") + ", want " + target); - nodesToRetire.add(host.with(host.status() - .withWantToDeprovision(true) - .withOsVersion(host.status().osVersion().withWanted(Optional.of(target)))) - .withWantToRetire(true, Agent.RetiringUpgrader, now)); + + host = host.withWantToRetire(true, true, Agent.RetiringUpgrader, now); + host = host.with(host.status().withOsVersion(host.status().osVersion().withWanted(Optional.of(target)))); + nodesToRetire.add(host); nodeRepository.write(nodesToRetire, lock); nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, nodeType)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 8b5639cc514..897af634d49 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -6,13 +6,12 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Type; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -48,6 +47,7 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class NodePatcher { private static final String WANT_TO_RETIRE = "wantToRetire"; + private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -77,13 +77,13 @@ public class NodePatcher { List patchedNodes = new ArrayList<>(); inspector.traverse((String name, Inspector value) -> { try { - node = applyField(node, name, value); + node = applyField(node, name, value, inspector); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not set field '" + name + "'", e); } try { - patchedNodes.addAll(applyFieldRecursive(name, value)); + patchedNodes.addAll(applyFieldRecursive(name, value, inspector)); } catch (IllegalArgumentException e) { // Non recursive field, ignore } @@ -93,12 +93,12 @@ public class NodePatcher { return patchedNodes; } - private List applyFieldRecursive(String name, Inspector value) { + private List applyFieldRecursive(String name, Inspector value, Inspector root) { switch (name) { case WANT_TO_RETIRE: List childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of(); return childNodes.stream() - .map(child -> applyField(child, name, value)) + .map(child -> applyField(child, name, value, root)) .collect(Collectors.toList()); default : @@ -106,7 +106,7 @@ public class NodePatcher { } } - private Node applyField(Node node, String name, Inspector value) { + private Node applyField(Node node, String name, Inspector value, Inspector root) { switch (name) { case "currentRebootGeneration" : return node.withCurrentRebootGeneration(asLong(value), clock.instant()); @@ -134,11 +134,10 @@ public class NodePatcher { case "additionalIpAddresses" : return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes.get()); case WANT_TO_RETIRE : - return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case "wantToDeprovision" : - if (node.type() != NodeType.host && asBoolean(value)) - throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); - return node.with(node.status().withWantToDeprovision(asBoolean(value))); + case WANT_TO_DEPROVISION : + boolean wantToRetire = asOptionalBoolean(root.field(WANT_TO_RETIRE)).orElse(node.status().wantToRetire()); + boolean wantToDeprovision = asOptionalBoolean(root.field(WANT_TO_DEPROVISION)).orElse(node.status().wantToDeprovision()); + return node.withWantToRetire(wantToRetire, wantToDeprovision, Agent.operator, clock.instant()); case "reports" : return nodeWithPatchedReports(node, value); case "openStackId" : @@ -202,7 +201,7 @@ public class NodePatcher { if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) return patchedNode; - patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + patchedNode = patchedNode.withWantToRetire(hasHardFailReports, hasHardFailReports, Agent.system, clock.instant()); } return patchedNode; @@ -252,19 +251,14 @@ public class NodePatcher { return field.asString(); } - private Optional asOptionalString(Inspector field) { - return field.type().equals(Type.NIX) ? Optional.empty() : Optional.of(asString(field)); - } - - // Allows us to clear optional flags by passing "null" as slime does not have an empty (but present) representation - private Optional removeQuotedNulls(Optional value) { - return value.filter(v -> !v.equals("null")); - } - private boolean asBoolean(Inspector field) { if ( ! field.type().equals(Type.BOOL)) throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type()); return field.asBool(); } + private Optional asOptionalBoolean(Inspector field) { + return Optional.of(field).filter(Inspector::valid).map(this::asBoolean); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 94b97d91312..151bd80a7b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -117,7 +117,7 @@ public class MockNodeRepository extends NodeRepository { Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); - nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); + nodes.add(node55.with(node55.status().withWantToRetire(true, true))); /* Setup docker hosts (two of these will be reserved for spares */ nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 7bb80fe2a21..97f1eda866a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.NodeType; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; @@ -13,7 +12,6 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Arrays; -import java.util.EnumSet; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; @@ -165,8 +163,7 @@ public class NodeRepositoryTest { // Set host 1 properties and deprovision it Node host1 = tester.nodeRepository().getNode("host1").get(); - host1 = host1.withWantToRetire(true, Agent.system, tester.nodeRepository().clock().instant()); - host1 = host1.with(host1.status().withWantToDeprovision(true)); + host1 = host1.withWantToRetire(true, true, Agent.system, tester.nodeRepository().clock().instant()); host1 = host1.withFirmwareVerifiedAt(tester.clock().instant()); host1 = host1.with(host1.status().withIncreasedFailCount()); host1 = host1.with(host1.reports().withReport(Report.basicReport("id", Report.Type.HARD_FAIL, tester.clock().instant(), "Test report"))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index e42e8e57b8c..89e43f80479 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -191,11 +191,11 @@ public class InactiveAndFailedExpirerTest { @Test public void nodes_marked_for_deprovisioning_move_to_parked() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(5, nodeResources); + tester.makeReadyHosts(2, nodeResources); // Activate and deallocate ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - List preparedNodes = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); + List preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromRequiredNodeType(NodeType.host)); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); tester.deactivate(applicationId); @@ -204,7 +204,7 @@ public class InactiveAndFailedExpirerTest { // Nodes marked for deprovisioning are moved to parked tester.nodeRepository().write(inactiveNodes.stream() - .map(node -> node.with(node.status().withWantToDeprovision(true))) + .map(node -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())) .collect(Collectors.toList()), () -> {}); tester.advanceTime(Duration.ofMinutes(11)); new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 16fba824300..b2ee298c19d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -205,7 +205,7 @@ public class InPlaceResizeProvisionTest { // ... same with setting a node to want to retire Node nodeToWantoToRetire = listCluster(content1).not().retired().asList().get(0); - tester.nodeRepository().write(nodeToWantoToRetire.with(nodeToWantoToRetire.status().withWantToRetire(true)), + tester.nodeRepository().write(nodeToWantoToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), tester.nodeRepository().lock(nodeToWantoToRetire)); new PrepareHelper(tester, app).prepare(content1, 8, 1, halvedResources).activate(); assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 98133898cf6..11e7af512c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -113,7 +113,7 @@ public class NodeTypeProvisioningTest { Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5); { // Pick out a node and retire it - tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)), () -> {}); + tester.nodeRepository().write(nodeToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), () -> {}); List hosts = deployProxies(application, tester); assertEquals(11, hosts.size()); @@ -186,7 +186,7 @@ public class NodeTypeProvisioningTest { String currentyRetiringHostname; { nodesToRetire.forEach(nodeToRetire -> - tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)), () -> {})); + tester.nodeRepository().write(nodeToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), () -> {})); List hosts = deployProxies(application, tester); assertEquals(11, hosts.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 7a7f8a7d891..607ca963cef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -633,7 +633,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Flag all nodes for retirement List readyNodes = tester.makeReadyNodes(5, defaultResources); - readyNodes.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + readyNodes.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); try { prepare(application, 2, 0, 2, 0, defaultResources, tester); @@ -661,7 +661,7 @@ public class ProvisioningTest { assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire - tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); // redeploy without allow failing tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); @@ -724,7 +724,7 @@ public class ProvisioningTest { // Retire some nodes and redeploy { List nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); - nodesToRetire.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + nodesToRetire.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); tester.activate(application, state.allHosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 29dac58e10c..1e788e2c70e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -207,8 +207,17 @@ public class NodesV2ApiTest { Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": false, \"wantToRetire\": false}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true, \"wantToRetire\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), @@ -379,7 +388,7 @@ public class NodesV2ApiTest { @Test public void fails_to_ready_node_with_hard_fail() throws Exception { assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host12.yahoo.com", "default") + "]"). + ("[" + asHostJson("host12.yahoo.com", "default", Optional.empty()) + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); @@ -563,7 +572,7 @@ public class NodesV2ApiTest { @Test public void test_reports_patching() throws IOException { // Add report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{" + " \"reports\": {" + " \"actualCpuCores\": {" + @@ -584,19 +593,19 @@ public class NodesV2ApiTest { " }" + "}"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching with an empty reports is no-op - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": {}}"), Request.Method.PATCH), 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching existing report overwrites - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{" + " \"reports\": {" + " \"actualCpuCores\": {" + @@ -606,22 +615,22 @@ public class NodesV2ApiTest { "}"), Request.Method.PATCH), 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-2.json"); // Clearing one report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": { \"diskSpace\": null } }"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-3.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-3.json"); // Clearing all reports - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": null }"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json new file mode 100644 index 00000000000..220fdbd8654 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json @@ -0,0 +1,102 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": true, + "wantToDeprovision": true, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ], + "reports": { + "actualCpuCores": { + "createdMillis": 3 + }, + "diskSpace": { + "createdMillis": 2, + "description": "Actual disk space (2TB) differs from spec (3TB)", + "type": "HARD_FAIL", + "details": { + "inGib": 3, + "disks": [ + "/dev/sda1", + "/dev/sdb3" + ] + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json new file mode 100644 index 00000000000..d2474f21c55 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json @@ -0,0 +1,90 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ], + "reports": { + "actualCpuCores": { + "createdMillis": 3 + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json new file mode 100644 index 00000000000..cbf02795d73 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json @@ -0,0 +1,85 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json new file mode 100644 index 00000000000..c00c06634b5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json @@ -0,0 +1,105 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": true, + "wantToDeprovision": true, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ], + "reports": { + "actualCpuCores": { + "createdMillis": 1, + "description": "Actual number of CPU cores (2) differs from spec (4)", + "type": "HARD_FAIL", + "value": 2 + }, + "diskSpace": { + "createdMillis": 2, + "description": "Actual disk space (2TB) differs from spec (3TB)", + "type": "HARD_FAIL", + "details": { + "inGib": 3, + "disks": [ + "/dev/sda1", + "/dev/sdb3" + ] + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json deleted file mode 100644 index a3d53798d7c..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": true, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 3 - }, - "diskSpace": { - "createdMillis": 2, - "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", - "details": { - "inGib": 3, - "disks": [ - "/dev/sda1", - "/dev/sdb3" - ] - } - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json deleted file mode 100644 index 7f0c3a5f706..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 3 - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json deleted file mode 100644 index 67b8d67c7f1..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": true, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 1, - "description": "Actual number of CPU cores (2) differs from spec (4)", - "type":"HARD_FAIL", - "value": 2 - }, - "diskSpace": { - "createdMillis": 2, - "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", - "details": { - "inGib": 3, - "disks": [ - "/dev/sda1", - "/dev/sdb3" - ] - } - } - } -} -- cgit v1.2.3