From f40b14666735820d6276ba453ec6fa3b72a91720 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 11 Nov 2021 14:41:39 +0100 Subject: Update rebooted event for nodes everytime container is recreated --- .../configserver/noderepository/NodeSpec.java | 3 ++- .../hosted/node/admin/nodeagent/NodeAgentImpl.java | 31 +++++++++++++--------- .../node/admin/nodeagent/NodeAgentImplTest.java | 8 +++--- 3 files changed, 25 insertions(+), 17 deletions(-) (limited to 'node-admin') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 7ac8d43f881..acb6ece6fc1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -791,7 +791,8 @@ public class NodeSpec { .type(NodeType.tenant) .flavor("d-2-8-50") .resources(new NodeResources(2, 8, 50, 10)) - .realResources(new NodeResources(2, 8, 50, 10)); + .realResources(new NodeResources(2, 8, 50, 10)) + .events(List.of(new Event("operator", "rebooted", Instant.EPOCH))); // Set the required allocated fields if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 3ab196a052e..92aacf8827b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -189,19 +189,29 @@ public class NodeAgentImpl implements NodeAgent { } } - private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context) { + private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context, Optional containerCreatedAt) { final NodeAttributes currentNodeAttributes = new NodeAttributes(); final NodeAttributes newNodeAttributes = new NodeAttributes(); + boolean changed = false; if (context.node().wantedRestartGeneration().isPresent() && !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) { currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration()); newNodeAttributes.withRestartGeneration(currentRestartGeneration); + changed = true; } - if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration)) { + boolean createdAtAfterRebootedEvent = context.node().events().stream() + .filter(event -> event.type().equals("rebooted")) + .map(event -> containerCreatedAt + .map(createdAt -> createdAt.isAfter(event.at())) + .orElse(false)) // Container not created + .findFirst() + .orElse(containerCreatedAt.isPresent()); // No rebooted event + if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration) || createdAtAfterRebootedEvent) { currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration()); newNodeAttributes.withRebootGeneration(currentRebootGeneration); + changed = true; } Optional actualDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); @@ -213,16 +223,13 @@ public class NodeAgentImpl implements NodeAgent { currentNodeAttributes.withVespaVersion(currentImage.tagAsVersion()); newNodeAttributes.withDockerImage(newImage); newNodeAttributes.withVespaVersion(newImage.tagAsVersion()); + changed = true; } - publishStateToNodeRepoIfChanged(context, currentNodeAttributes, newNodeAttributes); - } - - private void publishStateToNodeRepoIfChanged(NodeAgentContext context, NodeAttributes currentAttributes, NodeAttributes newAttributes) { - if (!currentAttributes.equals(newAttributes)) { + if (changed) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", - currentAttributes, newAttributes); - nodeRepository.updateNodeAttributes(context.hostname().value(), newAttributes); + currentNodeAttributes, newNodeAttributes); + nodeRepository.updateNodeAttributes(context.hostname().value(), newNodeAttributes); } } @@ -454,7 +461,7 @@ public class NodeAgentImpl implements NodeAgent { case inactive: case parked: removeContainerIfNeededUpdateContainerState(context, container); - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, Optional.empty()); stopServicesIfNeeded(context); break; case active: @@ -501,7 +508,7 @@ public class NodeAgentImpl implements NodeAgent { // has been successfully rolled out. // - Slobrok and internal orchestrator state is used to determine whether // to allow upgrade (suspend). - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, container.map(Container::createdAt)); if (suspendedInOrchestrator || node.orchestratorStatus().isSuspended()) { context.log(logger, "Call resume against Orchestrator"); orchestrator.resume(context.hostname().value()); @@ -517,7 +524,7 @@ public class NodeAgentImpl implements NodeAgent { credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context)); storageMaintainer.syncLogs(context, false); storageMaintainer.archiveNodeStorage(context); - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, Optional.empty()); nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); break; default: diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 29411b8069f..13f101be6e8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -187,7 +187,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); inOrder.verify(orchestrator, never()).resume(hostName); } @@ -285,7 +285,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations).removeContainer(eq(secondContext), any()); inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); inOrder.verify(containerOperations, never()).restartVespa(any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2))); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0))); nodeAgent.doConverge(secondContext); inOrder.verify(orchestrator).resume(any(String.class)); @@ -610,7 +610,7 @@ public class NodeAgentImplTest { inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); inOrder.verify(orchestrator).resume(hostName); } @@ -765,7 +765,7 @@ public class NodeAgentImplTest { Optional.of(new Container( containerId, ContainerName.fromHostname(hostName), - Instant.EPOCH, + clock.instant(), isRunning ? Container.State.running : Container.State.exited, "image-id-1", dockerImage, -- cgit v1.2.3