diff options
author | Valerij Fredriksen <valerijf@yahooinc.com> | 2023-04-18 12:00:26 +0200 |
---|---|---|
committer | Valerij Fredriksen <valerijf@yahooinc.com> | 2023-04-18 15:54:02 +0200 |
commit | 2aa24e270099d7fe2060452e807ec2d728c6d06b (patch) | |
tree | c92723b13e63723bf2e7a4a20fe5366362a93562 /node-admin | |
parent | 38040357527c9bab9951c59ee1687520e677b422 (diff) |
Drop documents in node-admin if report is set
Diffstat (limited to 'node-admin')
3 files changed, 134 insertions, 0 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java new file mode 100644 index 00000000000..0d88f10ebf9 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author freva + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DropDocumentsReport extends BaseReport { + private static final String REPORT_ID = "dropDocuments"; + private static final String DROPPED_AT_FIELD = "droppedAt"; + private static final String READIED_AT_FIELD = "readiedAt"; + private static final String STARTED_AT_FIELD = "startedAt"; + + private final Long droppedAt; + private final Long readiedAt; + private final Long startedAt; + + public DropDocumentsReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull, + @JsonProperty(DROPPED_AT_FIELD) Long droppedAtOrNull, + @JsonProperty(READIED_AT_FIELD) Long readiedAtOrNull, + @JsonProperty(STARTED_AT_FIELD) Long startedAtOrNull) { + super(createdMillisOrNull, null); + this.droppedAt = droppedAtOrNull; + this.readiedAt = readiedAtOrNull; + this.startedAt = startedAtOrNull; + } + + @JsonGetter(DROPPED_AT_FIELD) + public Long droppedAt() { return droppedAt; } + + @JsonGetter(READIED_AT_FIELD) + public Long readiedAt() { return readiedAt; } + + @JsonGetter(STARTED_AT_FIELD) + public Long startedAt() { return startedAt; } + + public DropDocumentsReport withDroppedAt(long droppedAt) { + return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); + } + + public DropDocumentsReport withStartedAt(long startedAt) { + return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); + } + + public static String reportId() { + return REPORT_ID; + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 20359410321..f2f690106fa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembers import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.container.Container; @@ -29,6 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import java.time.Clock; import java.time.Duration; @@ -228,6 +230,12 @@ public class NodeAgentImpl implements NodeAgent { changed = true; } + Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); + if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) { + newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(clock.millis()).toJsonNode()); + changed = true; + } + if (changed) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", currentNodeAttributes, newNodeAttributes); @@ -433,6 +441,21 @@ public class NodeAgentImpl implements NodeAgent { .orElse(false); } + private void dropDocsIfNeeded(NodeAgentContext context, Optional<Container> container) { + Optional<DropDocumentsReport> report = context.node().reports() + .getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); + if (report.isEmpty() || report.get().readiedAt() != null) return; + + if (report.get().droppedAt() == null) { + container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true)); + FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context); + nodeRepository.updateNodeAttributes(context.node().hostname(), + new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(clock.millis()).toJsonNode())); + } + + throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container"); + } + public void converge(NodeAgentContext context) { try { doConverge(context); @@ -494,6 +517,7 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); return; } + dropDocsIfNeeded(context, container); container = removeContainerIfNeededUpdateContainerState(context, container); credentialsMaintainers.forEach(maintainer -> maintainer.converge(context)); if (container.isEmpty()) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index b8b72308bdd..2db5314dbf2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeReposit import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.container.Container; @@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,8 +40,11 @@ import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.BiFunction; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -739,6 +744,56 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); } + @Test + void drop_all_documents() { + InOrder inOrder = inOrder(orchestrator, nodeRepository); + BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ? + nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode())) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .build(); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + + NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null)); + UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile(); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + assertTrue(indexPath.exists()); + + // Initially no changes, index is not dropped + nodeAgent.converge(context); + assertTrue(indexPath.exists()); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null))); + nodeAgent.converge(context); + verify(containerOperations).removeContainer(eq(context), any()); + assertFalse(indexPath.exists()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, clock.millis(), null, null).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + + // After droppedAt and before readiedAt are set, we cannot proceed + mockGetContainer(null, false); + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null))); + nodeAgent.converge(context); + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null))); + nodeAgent.converge(context); + verify(containerOperations).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + clock.advance(Duration.ofSeconds(31)); + nodeAgent.converge(context); + verify(containerOperations, times(1)).startContainer(eq(context)); + verify(containerOperations, never()).removeContainer(eq(context), any()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(0) + .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, clock.millis()).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + } + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) .type(NodeType.tenant) |