Update rebooted event for nodes everytime container is recreated

3 files changed, 25 insertions, 17 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
index 7ac8d43f881..acb6ece6fc1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
@@ -791,7 +791,8 @@ public class NodeSpec {
                     .type(NodeType.tenant)
                     .flavor("d-2-8-50")
                     .resources(new NodeResources(2, 8, 50, 10))
-                    .realResources(new NodeResources(2, 8, 50, 10));
+                    .realResources(new NodeResources(2, 8, 50, 10))
+                    .events(List.of(new Event("operator", "rebooted", Instant.EPOCH)));
 
             // Set the required allocated fields
             if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 3ab196a052e..92aacf8827b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -189,19 +189,29 @@ public class NodeAgentImpl implements NodeAgent {
         }
     }
 
-    private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context) {
+    private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context, Optional<Instant> containerCreatedAt) {
         final NodeAttributes currentNodeAttributes = new NodeAttributes();
         final NodeAttributes newNodeAttributes = new NodeAttributes();
+        boolean changed = false;
 
         if (context.node().wantedRestartGeneration().isPresent() &&
                 !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) {
             currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration());
             newNodeAttributes.withRestartGeneration(currentRestartGeneration);
+            changed = true;
         }
 
-        if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration)) {
+        boolean createdAtAfterRebootedEvent = context.node().events().stream()
+                .filter(event -> event.type().equals("rebooted"))
+                .map(event -> containerCreatedAt
+                        .map(createdAt -> createdAt.isAfter(event.at()))
+                        .orElse(false)) // Container not created
+                .findFirst()
+                .orElse(containerCreatedAt.isPresent()); // No rebooted event
+        if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration) || createdAtAfterRebootedEvent) {
             currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration());
             newNodeAttributes.withRebootGeneration(currentRebootGeneration);
+            changed = true;
         }
 
         Optional<DockerImage> actualDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN);
@@ -213,16 +223,13 @@ public class NodeAgentImpl implements NodeAgent {
             currentNodeAttributes.withVespaVersion(currentImage.tagAsVersion());
             newNodeAttributes.withDockerImage(newImage);
             newNodeAttributes.withVespaVersion(newImage.tagAsVersion());
+            changed = true;
         }
 
-        publishStateToNodeRepoIfChanged(context, currentNodeAttributes, newNodeAttributes);
-    }
-
-    private void publishStateToNodeRepoIfChanged(NodeAgentContext context, NodeAttributes currentAttributes, NodeAttributes newAttributes) {
-        if (!currentAttributes.equals(newAttributes)) {
+        if (changed) {
             context.log(logger, "Publishing new set of attributes to node repo: %s -> %s",
-                    currentAttributes, newAttributes);
-            nodeRepository.updateNodeAttributes(context.hostname().value(), newAttributes);
+                    currentNodeAttributes, newNodeAttributes);
+            nodeRepository.updateNodeAttributes(context.hostname().value(), newNodeAttributes);
         }
     }
 
@@ -454,7 +461,7 @@ public class NodeAgentImpl implements NodeAgent {
             case inactive:
             case parked:
                 removeContainerIfNeededUpdateContainerState(context, container);
-                updateNodeRepoWithCurrentAttributes(context);
+                updateNodeRepoWithCurrentAttributes(context, Optional.empty());
                 stopServicesIfNeeded(context);
                 break;
             case active:
@@ -501,7 +508,7 @@ public class NodeAgentImpl implements NodeAgent {
                 //    has been successfully rolled out.
                 //  - Slobrok and internal orchestrator state is used to determine whether
                 //    to allow upgrade (suspend).
-                updateNodeRepoWithCurrentAttributes(context);
+                updateNodeRepoWithCurrentAttributes(context, container.map(Container::createdAt));
                 if (suspendedInOrchestrator || node.orchestratorStatus().isSuspended()) {
                     context.log(logger, "Call resume against Orchestrator");
                     orchestrator.resume(context.hostname().value());
@@ -517,7 +524,7 @@ public class NodeAgentImpl implements NodeAgent {
                 credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context));
                 storageMaintainer.syncLogs(context, false);
                 storageMaintainer.archiveNodeStorage(context);
-                updateNodeRepoWithCurrentAttributes(context);
+                updateNodeRepoWithCurrentAttributes(context, Optional.empty());
                 nodeRepository.setNodeState(context.hostname().value(), NodeState.ready);
                 break;
             default:
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 29411b8069f..13f101be6e8 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -187,7 +187,7 @@ public class NodeAgentImplTest {
         inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
         inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context));
         inOrder.verify(nodeRepository).updateNodeAttributes(
-                hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()));
+                hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0));
         inOrder.verify(orchestrator, never()).resume(hostName);
     }
 
@@ -285,7 +285,7 @@ public class NodeAgentImplTest {
         inOrder.verify(containerOperations).removeContainer(eq(secondContext), any());
         inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any());
         inOrder.verify(containerOperations, never()).restartVespa(any());
-        inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2)));
+        inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0)));
 
         nodeAgent.doConverge(secondContext);
         inOrder.verify(orchestrator).resume(any(String.class));
@@ -610,7 +610,7 @@ public class NodeAgentImplTest {
         inOrder.verify(aclMaintainer, times(1)).converge(eq(context));
         inOrder.verify(containerOperations, times(1)).resumeNode(eq(context));
         inOrder.verify(nodeRepository).updateNodeAttributes(
-                hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()));
+                hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0));
         inOrder.verify(orchestrator).resume(hostName);
     }
 
@@ -765,7 +765,7 @@ public class NodeAgentImplTest {
                     Optional.of(new Container(
                             containerId,
                             ContainerName.fromHostname(hostName),
-                            Instant.EPOCH,
+                            clock.instant(),
                             isRunning ? Container.State.running : Container.State.exited,
                             "image-id-1",
                             dockerImage,