Merge pull request #26772 from vespa-engine/freva/drop-docs

Add support for node-agent dropping all container docs

5 files changed, 196 insertions, 21 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java
new file mode 100644
index 00000000000..0d88f10ebf9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java
@@ -0,0 +1,55 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports;
+
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class DropDocumentsReport extends BaseReport {
+    private static final String REPORT_ID = "dropDocuments";
+    private static final String DROPPED_AT_FIELD = "droppedAt";
+    private static final String READIED_AT_FIELD = "readiedAt";
+    private static final String STARTED_AT_FIELD = "startedAt";
+
+    private final Long droppedAt;
+    private final Long readiedAt;
+    private final Long startedAt;
+
+    public DropDocumentsReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull,
+                               @JsonProperty(DROPPED_AT_FIELD) Long droppedAtOrNull,
+                               @JsonProperty(READIED_AT_FIELD) Long readiedAtOrNull,
+                               @JsonProperty(STARTED_AT_FIELD) Long startedAtOrNull) {
+        super(createdMillisOrNull, null);
+        this.droppedAt = droppedAtOrNull;
+        this.readiedAt = readiedAtOrNull;
+        this.startedAt = startedAtOrNull;
+    }
+
+    @JsonGetter(DROPPED_AT_FIELD)
+    public Long droppedAt() { return droppedAt; }
+
+    @JsonGetter(READIED_AT_FIELD)
+    public Long readiedAt() { return readiedAt; }
+
+    @JsonGetter(STARTED_AT_FIELD)
+    public Long startedAt() { return startedAt; }
+
+    public DropDocumentsReport withDroppedAt(long droppedAt) {
+        return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt);
+    }
+
+    public DropDocumentsReport withStartedAt(long startedAt) {
+        return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt);
+    }
+
+    public static String reportId() {
+        return REPORT_ID;
+    }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 20359410321..f2f690106fa 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembers
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException;
 import com.yahoo.vespa.hosted.node.admin.container.Container;
@@ -29,6 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -228,6 +230,12 @@ public class NodeAgentImpl implements NodeAgent {
             changed = true;
         }
 
+        Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class);
+        if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) {
+            newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(clock.millis()).toJsonNode());
+            changed = true;
+        }
+
         if (changed) {
             context.log(logger, "Publishing new set of attributes to node repo: %s -> %s",
                     currentNodeAttributes, newNodeAttributes);
@@ -433,6 +441,21 @@ public class NodeAgentImpl implements NodeAgent {
                    .orElse(false);
     }
 
+    private void dropDocsIfNeeded(NodeAgentContext context, Optional<Container> container) {
+        Optional<DropDocumentsReport> report = context.node().reports()
+                .getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class);
+        if (report.isEmpty() || report.get().readiedAt() != null) return;
+
+        if (report.get().droppedAt() == null) {
+            container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true));
+            FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context);
+            nodeRepository.updateNodeAttributes(context.node().hostname(),
+                    new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(clock.millis()).toJsonNode()));
+        }
+
+        throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container");
+    }
+
     public void converge(NodeAgentContext context) {
         try {
             doConverge(context);
@@ -494,6 +517,7 @@ public class NodeAgentImpl implements NodeAgent {
                     context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString());
                     return;
                 }
+                dropDocsIfNeeded(context, container);
                 container = removeContainerIfNeededUpdateContainerState(context, container);
                 credentialsMaintainers.forEach(maintainer -> maintainer.converge(context));
                 if (container.isEmpty()) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index b8b72308bdd..2db5314dbf2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeReposit
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException;
 import com.yahoo.vespa.hosted.node.admin.container.Container;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -38,8 +40,11 @@ import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.BiFunction;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -739,6 +744,56 @@ public class NodeAgentImplTest {
         inOrder.verify(orchestrator, times(1)).resume(eq(hostName));
     }
 
+    @Test
+    void drop_all_documents() {
+        InOrder inOrder = inOrder(orchestrator, nodeRepository);
+        BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ?
+                nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode()))
+                .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+                .build();
+        NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+
+        NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null));
+        UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile();
+        mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true);
+        assertTrue(indexPath.exists());
+
+        // Initially no changes, index is not dropped
+        nodeAgent.converge(context);
+        assertTrue(indexPath.exists());
+        inOrder.verifyNoMoreInteractions();
+
+        context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null)));
+        nodeAgent.converge(context);
+        verify(containerOperations).removeContainer(eq(context), any());
+        assertFalse(indexPath.exists());
+        inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, clock.millis(), null, null).toJsonNode())));
+        inOrder.verifyNoMoreInteractions();
+
+        // After droppedAt and before readiedAt are set, we cannot proceed
+        mockGetContainer(null, false);
+        context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null)));
+        nodeAgent.converge(context);
+        verify(containerOperations, never()).removeContainer(eq(context), any());
+        verify(containerOperations, never()).startContainer(eq(context));
+        inOrder.verifyNoMoreInteractions();
+
+        context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null)));
+        nodeAgent.converge(context);
+        verify(containerOperations).startContainer(eq(context));
+        inOrder.verifyNoMoreInteractions();
+
+        mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true);
+        clock.advance(Duration.ofSeconds(31));
+        nodeAgent.converge(context);
+        verify(containerOperations, times(1)).startContainer(eq(context));
+        verify(containerOperations, never()).removeContainer(eq(context), any());
+        inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes()
+                .withRebootGeneration(0)
+                .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, clock.millis()).toJsonNode())));
+        inOrder.verifyNoMoreInteractions();
+    }
+
     private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) {
         NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState)
                 .type(NodeType.tenant)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java
index dfe01f5f1c3..bbe287fc034 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java
@@ -11,8 +11,10 @@ import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.WireguardKey;
+import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.slime.Type;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
@@ -40,6 +42,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Stream;
 
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
@@ -54,9 +57,13 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote;
  */
 public class NodePatcher {
 
+    // Same as in DropDocumentsReport.java
+    private static final String DROP_DOCUMENTS_REPORT = "dropDocuments";
+
     private static final String WANT_TO_RETIRE = "wantToRetire";
     private static final String WANT_TO_DEPROVISION = "wantToDeprovision";
     private static final String WANT_TO_REBUILD = "wantToRebuild";
+    private static final String REPORTS = "reports";
     private static final Set<String> RECURSIVE_FIELDS = Set.of(WANT_TO_RETIRE, WANT_TO_DEPROVISION);
     private static final Set<String> IP_CONFIG_FIELDS = Set.of("ipAddresses",
                                                                "additionalIpAddresses",
@@ -133,7 +140,29 @@ public class NodePatcher {
                     throw new IllegalArgumentException("Could not set field '" + name + "'", e);
                 }
             }
-            nodeRepository.nodes().write(node, lock);
+            List<Node> nodes = List.of(node);
+            if (node.state() == Node.State.active && isInDocumentsDroppedState(root.field(REPORTS).field(DROP_DOCUMENTS_REPORT))) {
+                NodeList clusterNodes = nodeRepository.nodes()
+                        .list(Node.State.active)
+                        .except(node)
+                        .owner(node.allocation().get().owner())
+                        .cluster(node.allocation().get().membership().cluster().id());
+                boolean allNodesDroppedDocuments = clusterNodes.stream().allMatch(cNode ->
+                        cNode.reports().getReport(DROP_DOCUMENTS_REPORT).map(report -> isInDocumentsDroppedState(report.getInspector())).orElse(false));
+                if (allNodesDroppedDocuments) {
+                    nodes = Stream.concat(nodes.stream(), clusterNodes.stream())
+                            .map(cNode -> {
+                                Cursor reportRoot = new Slime().setObject();
+                                Report report = cNode.reports().getReport(DROP_DOCUMENTS_REPORT).get();
+                                report.toSlime(reportRoot);
+                                reportRoot.setLong("readiedAt", clock.millis());
+
+                                return cNode.with(cNode.reports().withReport(Report.fromSlime(DROP_DOCUMENTS_REPORT, reportRoot)));
+                            })
+                            .toList();
+                }
+            }
+            nodeRepository.nodes().write(nodes, lock);
         }
     }
 
@@ -202,18 +231,15 @@ public class NodePatcher {
                                                      .orElseGet(node.status()::wantToRebuild),
                                              Agent.operator,
                                              clock.instant());
-            case "reports" :
+            case REPORTS:
                 return nodeWithPatchedReports(node, value);
-            case "id" :
+            case "id":
                 return node.withId(asString(value));
             case "diskGb":
-            case "minDiskAvailableGb":
                 return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble())), Agent.operator, clock.instant());
             case "memoryGb":
-            case "minMainMemoryAvailableGb":
                 return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble())), Agent.operator, clock.instant());
             case "vcpu":
-            case "minCpuCores":
                 return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble())), Agent.operator, clock.instant());
             case "fastDisk":
                 return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow)), Agent.operator, clock.instant());
@@ -244,18 +270,12 @@ public class NodePatcher {
     }
 
     private Node applyIpconfigField(Node node, String name, Inspector value, LockedNodeList nodes) {
-        switch (name) {
-            case "ipAddresses" -> {
-                return IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes);
-            }
-            case "additionalIpAddresses" -> {
-                return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes);
-            }
-            case "additionalHostnames" -> {
-                return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes);
-            }
-        }
-        throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field");
+        return switch (name) {
+            case "ipAddresses" -> IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes);
+            case "additionalIpAddresses" -> IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes);
+            case "additionalHostnames" -> IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes);
+            default -> throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field");
+        };
     }
 
     private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) {
@@ -374,4 +394,9 @@ public class NodePatcher {
         return Optional.of(field).filter(Inspector::valid).map(this::asBoolean);
     }
 
+    private static boolean isInDocumentsDroppedState(Inspector report) {
+        if (!report.valid()) return false;
+        return report.field("droppedAt").valid() && !report.field("readiedAt").valid();
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
index c9e57c22d11..7affcfebdb3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
@@ -647,6 +647,22 @@ public class NodesV2ApiTest {
                         Request.Method.PATCH),
                 "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
         assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json");
+
+        assertResponse(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com",
+                        Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}}"),
+                        Request.Method.PATCH),
+                "{\"message\":\"Updated host1.yahoo.com\"}");
+        tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"),
+                "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}");
+
+        assertResponse(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com",
+                        Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456}}}"),
+                        Request.Method.PATCH),
+                "{\"message\":\"Updated host10.yahoo.com\"}");
+        tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com"),
+                "{\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456,\"readiedAt\":123}}");
+        tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"),
+                "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36,\"readiedAt\":123}}");
     }
 
     @Test
@@ -906,13 +922,13 @@ public class NodesV2ApiTest {
 
         // Test patching with overrides
         tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host,
-                                          "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8),
+                                          "{\"diskGb\":5432,\"memoryGb\":2345}".getBytes(StandardCharsets.UTF_8),
                                           Request.Method.PATCH),
                               400,
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}");
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'memoryGb': Can only override disk GB for configured flavor\"}");
 
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host,
-                        "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8),
+                        "{\"diskGb\":5432}".getBytes(StandardCharsets.UTF_8),
                         Request.Method.PATCH),
                 "{\"message\":\"Updated " + host + "\"}");
         tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host),