From 4376d536eb6c7f2b9c78d650933a11d672781a36 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 21 Apr 2023 11:23:04 +0200 Subject: Create new /nodes/v2 API to initiate drop of documents --- .../yahoo/vespa/hosted/provision/node/Nodes.java | 20 +++++++++++++ .../hosted/provision/restapi/NodePatcher.java | 2 +- .../provision/restapi/NodesV2ApiHandler.java | 10 +++++-- .../hosted/provision/restapi/NodesV2ApiTest.java | 35 +++++++++++++++++----- 4 files changed, 57 insertions(+), 10 deletions(-) (limited to 'node-repository') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 6dc3b2b3193..fd6b15609d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -5,6 +5,7 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; @@ -42,6 +43,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.provision.restapi.NodePatcher.DROP_DOCUMENTS_REPORT; + /** * The nodes in the node repo and their state transitions * @@ -727,6 +730,23 @@ public class Nodes { return resultingNodes; } + public List dropDocuments(ApplicationId applicationId, Optional clusterId) { + try (Mutex lock = applications.lock(applicationId)) { + Instant now = clock.instant(); + List nodes = list(Node.State.active, Node.State.reserved) + .owner(applicationId) + .matching(node -> { + ClusterSpec cluster = node.allocation().get().membership().cluster(); + if (!cluster.type().isContent()) return false; + return clusterId.isEmpty() || clusterId.get().equals(cluster.id()); + }) + .mapToList(node -> node.with(node.reports().withReport(Report.basicReport(DROP_DOCUMENTS_REPORT, Report.Type.UNSPECIFIED, now, "")))); + if (nodes.isEmpty()) + throw new NoSuchNodeException("No content nodes found for " + applicationId + clusterId.map(id -> " and cluster " + id).orElse("")); + return db.writeTo(nodes, Agent.operator, Optional.empty()); + } + } + public boolean canAllocateTenantNodeTo(Node host) { return canAllocateTenantNodeTo(host, zone.cloud().dynamicProvisioning()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index bbe287fc034..4dc48459ec9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -58,7 +58,7 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class NodePatcher { // Same as in DropDocumentsReport.java - private static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; + public static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; private static final String WANT_TO_RETIRE = "wantToRetire"; private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index eb935ba6a5c..e04d21d3012 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; @@ -222,6 +223,11 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job")); if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); + if (path.matches("/nodes/v2/application/{applicationId}/drop-documents")) { + int count = nodeRepository.nodes().dropDocuments(ApplicationId.fromFullString(path.get("applicationId")), + Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)).size(); + return new MessageResponse("Triggered dropping of documents on " + count + " nodes"); + } throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } @@ -482,14 +488,14 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Load load, Cursor object) { + private static void toSlime(Load load, Cursor object) { object.setDouble("cpu", load.cpu()); object.setDouble("memory", load.memory()); object.setDouble("disk", load.disk()); } /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ - private URI withPath(String newPath, URI uri) { + private static URI withPath(String newPath, URI uri) { try { return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 7affcfebdb3..022822fd3ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -647,22 +647,43 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); + } + + @Test + public void drop_documents() throws IOException { + // Initially no reports + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "reports", false); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "reports", false); + + // Initiating drop documents will set the report on all nodes + assertResponse(new Request("http://localhost:8080/nodes/v2/application/tenant3.application3.instance3/drop-documents?clusterId=id3", new byte[0], Request.Method.POST), + "{\"message\":\"Triggered dropping of documents on 2 nodes\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com", + // Host admin of the first node finishes dropping + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host1.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "{\"message\":\"Updated host4.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com", + // Host admin of the second node finishes dropping, node-repo will update report on both nodes to start phase 2 + assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host10.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com"), + "{\"message\":\"Updated test-node-pool-102-2\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "{\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456,\"readiedAt\":123}}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36,\"readiedAt\":123}}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/application/does.not.exist/drop-documents", new byte[0], Request.Method.POST), + 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No content nodes found for does.not.exist\"}"); } @Test -- cgit v1.2.3