Preserve all node events

Node events are now limited by a total size limit, instead of one per type.

14 files changed, 169 insertions, 104 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index 3db68a27234..2522241bc84 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -347,19 +347,24 @@ public final class Node implements Nodelike {
                         allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems);
     }
 
-    /** Returns a copy of this with a history record saying it was detected to be down at this instant */
+    /** Returns a copy of this with a history record saying it was detected to be down at given instant */
     public Node downAt(Instant instant, Agent agent) {
         return with(history.with(new History.Event(History.Event.Type.down, agent, instant)));
     }
 
-    /** Returns a copy of this with any history record saying it has been detected down removed */
-    public Node up() {
-        return with(history.without(History.Event.Type.down));
+    /** Returns a copy of this with a history record saying it was detected to be up at given instant */
+    public Node upAt(Instant instant, Agent agent) {
+        return with(history.with(new History.Event(History.Event.Type.up, agent, instant)));
     }
 
-    /** Returns whether this node has a record of being down */
+    /** Returns whether this node is down, according to its recorded 'down' and 'up' events */
     public boolean isDown() {
-        return history().event(History.Event.Type.down).isPresent();
+        Optional<Instant> downAt = history().event(History.Event.Type.down).map(History.Event::at);
+        if (downAt.isEmpty()) return false;
+
+        Optional<Instant> upAt = history().event(History.Event.Type.up).map(History.Event::at);
+        if (upAt.isEmpty()) return true;
+        return !downAt.get().isBefore(upAt.get());
     }
 
     /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 6eaee7b33de..459ab6a3e1c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -6,9 +6,9 @@ import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.NodeAllocationException;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.NodeAllocationException;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.lang.MutableInteger;
 import com.yahoo.transaction.Mutex;
@@ -164,8 +164,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
         }
 
         return candidatesForRemoval(nodes).stream()
-                .sorted(Comparator.comparing(node -> node.history().events().stream()
-                                                         .map(History.Event::at).min(Comparator.naturalOrder()).orElse(Instant.MIN)))
+                .sorted(Comparator.comparing(node -> node.history().asList().stream().findFirst().map(History.Event::at).orElse(Instant.MIN)))
                 .filter(node -> {
                     if (!sharedHosts.containsKey(node.hostname()) || sharedHosts.size() > minCount) {
                         sharedHosts.remove(node.hostname());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
index fa6b201def4..ced0a161e59 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
@@ -78,11 +78,11 @@ public class ExpeditedChangeApplicationMaintainer extends ApplicationMaintainer
 
         List<String> reasons = nodes.stream()
                                     .flatMap(node -> node.history()
-                                                            .events()
-                                                            .stream()
-                                                            .filter(event -> expediteChangeBy(event.agent()))
-                                                            .filter(event -> lastDeployTime.get().isBefore(event.at()))
-                                                            .map(event -> event.type() + (event.agent() == Agent.system ? "" : " by " + event.agent())))
+                                                         .asList()
+                                                         .stream()
+                                                         .filter(event -> expediteChangeBy(event.agent()))
+                                                         .filter(event -> lastDeployTime.get().isBefore(event.at()))
+                                                         .map(event -> event.type() + (event.agent() == Agent.system ? "" : " by " + event.agent())))
                                     .sorted()
                                     .distinct()
                                     .collect(Collectors.toList());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 1fe29c8b162..b117523dce0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.ClusterId;
-import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.persistence.CacheStats;
 import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
@@ -248,8 +247,8 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
             boolean down = NodeHealthTracker.allDown(services);
             metric.set("nodeFailerBadNode", (down ? 1 : 0), context);
 
-            boolean nodeDownInNodeRepo = node.history().event(History.Event.Type.down).isPresent();
-            metric.set("downInNodeRepo", (nodeDownInNodeRepo ? 1 : 0), context);
+            boolean recordedDown = node.isDown();
+            metric.set("downInNodeRepo", (recordedDown ? 1 : 0), context);
         }
 
         metric.set("numberOfServices", numberOfServices, context);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 3900d10a53e..1f6862a1dc4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -155,7 +155,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
 
         for (Node node : activeNodes) {
             Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit);
-            if (node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node)) {
+            if (downBefore(graceTimeStart, node) && !applicationSuspended(node)) {
                 // Allow a grace period after node re-activation
                 if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart))
                     failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit));
@@ -278,6 +278,11 @@ public class NodeFailer extends NodeRepositoryMaintainer {
         }
     }
 
+    /** Returns whether node is down, and has been down since before given instant */
+    private static boolean downBefore(Instant instant, Node node) {
+        return node.isDown() && node.history().event(History.Event.Type.down).get().at().isBefore(instant);
+    }
+
     private void wantToFail(Node node, boolean wantToFail, Mutex lock) {
         nodeRepository().nodes().write(node.withWantToFail(wantToFail, Agent.NodeFailer, clock().instant()), lock);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
index 874ff91d8a4..1eb651598e8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
@@ -96,7 +96,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer {
                 if (isDown) {
                     recordAsDown(node.get(), lock);
                 } else {
-                    clearDownRecord(node.get(), lock);
+                    recordAsUp(node.get(), lock);
                 }
             } catch (ApplicationLockException e) {
                 // Fine, carry on with other nodes. We'll try updating this one in the next run
@@ -129,14 +129,14 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer {
 
     /** Record a node as down if not already recorded */
     private void recordAsDown(Node node, Mutex lock) {
-        if (node.history().event(History.Event.Type.down).isPresent()) return; // already down: Don't change down timestamp
+        if (node.isDown()) return; // already down: Don't change down timestamp
         nodeRepository().nodes().write(node.downAt(clock().instant(), Agent.NodeHealthTracker), lock);
     }
 
     /** Clear down record for node, if any */
-    private void clearDownRecord(Node node, Mutex lock) {
-        if (node.history().event(History.Event.Type.down).isEmpty()) return;
-        nodeRepository().nodes().write(node.up(), lock);
+    private void recordAsUp(Node node, Mutex lock) {
+        if (!node.isDown()) return; // already up: Don't change up timestamp
+        nodeRepository().nodes().write(node.upAt(clock().instant(), Agent.NodeHealthTracker), lock);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
index 3cd97c64e4d..6093977bc9d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
@@ -55,11 +55,11 @@ public class NodeRebooter extends NodeRepositoryMaintainer {
         var rebootEvents = EnumSet.of(History.Event.Type.provisioned, History.Event.Type.rebooted, History.Event.Type.osUpgraded);
         var rebootInterval = Duration.ofDays(rebootIntervalInDays.value());
 
-        Optional<Duration> overdue = node.history().events().stream()
-                .filter(event -> rebootEvents.contains(event.type()))
-                .map(History.Event::at)
-                .max(Comparator.naturalOrder())
-                .map(lastReboot -> Duration.between(lastReboot, clock().instant()).minus(rebootInterval));
+        Optional<Duration> overdue = node.history().asList().stream()
+                                         .filter(event -> rebootEvents.contains(event.type()))
+                                         .map(History.Event::at)
+                                         .max(Comparator.naturalOrder())
+                                         .map(lastReboot -> Duration.between(lastReboot, clock().instant()).minus(rebootInterval));
 
         if (overdue.isEmpty()) // should never happen as all hosts should have provisioned timestamp
             return random.nextDouble() < interval().getSeconds() / (double) rebootInterval.getSeconds();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
index f1e62634235..2515e570dbb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
@@ -1,17 +1,19 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.node;
 
-import com.google.common.collect.ImmutableMap;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.time.Instant;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
- * An immutable record of the last event of each type happening to this node.
+ * An immutable list of events happening to this node, in chronological order.
+ *
  * Note that the history cannot be used to find the nodes current state - it will have a record of some
  * event happening in the past even if that event is later undone.
  *
@@ -19,67 +21,63 @@ import java.util.stream.Collectors;
  */
 public class History {
 
-    private final ImmutableMap<Event.Type, Event> events;
+    /** The maximum number of events to keep for a node */
+    private static final int MAX_SIZE = 15;
 
-    public History(Collection<Event> events) {
-        this(toImmutableMap(events));
-    }
+    private final List<Event> events;
 
-    private History(ImmutableMap<Event.Type, Event> events) {
-        this.events = events;
+    public History(List<Event> events) {
+        this(events, MAX_SIZE);
     }
 
-    private static ImmutableMap<Event.Type, Event> toImmutableMap(Collection<Event> events) {
-        ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>();
-        for (Event event : events)
-            builder.put(event.type(), event);
-        return builder.build();
+    History(List<Event> events, int maxSize) {
+        this.events = Objects.requireNonNull(events, "events must be non-null")
+                             .stream()
+                             .sorted(Comparator.comparing(Event::at))
+                             .skip(Math.max(events.size() - maxSize, 0))
+                             .collect(Collectors.toUnmodifiableList());
     }
 
-    /** Returns this event if it is present in this history */
-    public Optional<Event> event(Event.Type type) { return Optional.ofNullable(events.get(type)); }
+    /** Returns the latest event of given type, if it is present in this history */
+    public Optional<Event> event(Event.Type type) {
+        return events.stream().filter(event -> event.type() == type).max(Comparator.comparing(Event::at));
+    }
 
     /** Returns true if a given event is registered in this history at the given time */
     public boolean hasEventAt(Event.Type type, Instant time) {
-        return event(type)
-                       .map(event -> event.at().equals(time))
-                       .orElse(false);
+        return event(type).map(event -> event.at().equals(time))
+                          .orElse(false);
     }
 
     /** Returns true if a given event is registered in this history after the given time */
     public boolean hasEventAfter(Event.Type type, Instant time) {
-        return event(type)
-                .map(event -> event.at().isAfter(time))
-                .orElse(false);
+        return event(type).map(event -> event.at().isAfter(time))
+                          .orElse(false);
     }
 
     /** Returns true if a given event is registered in this history before the given time */
     public boolean hasEventBefore(Event.Type type, Instant time) {
-        return event(type)
-                .map(event -> event.at().isBefore(time))
-                .orElse(false);
+        return event(type).map(event -> event.at().isBefore(time))
+                          .orElse(false);
     }
 
-    public Collection<Event> events() { return events.values(); }
+    public List<Event> asList() {
+        return events;
+    }
 
     /** Returns a copy of this history with the given event added */
     public History with(Event event) {
-        ImmutableMap.Builder<Event.Type, Event> builder = builderWithout(event.type());
-        builder.put(event.type(), event);
-        return new History(builder.build());
-    }
-
-    /** Returns a copy of this history with the given event type removed (or an identical history if it was not present) */
-    public History without(Event.Type type) {
-        return new History(builderWithout(type).build());
-    }
-
-    private ImmutableMap.Builder<Event.Type, Event> builderWithout(Event.Type type) {
-        ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>();
-        for (Event event : events.values())
-            if (event.type() != type)
-                builder.put(event.type(), event);
-        return builder;
+        List<Event> copy = new ArrayList<>(events);
+        if (!copy.isEmpty()) {
+            // Let given event overwrite the latest if they're of the same type. Some events may be repeated, such as
+            // 'reserved'
+            Event last = copy.get(copy.size() - 1);
+            if (last.type() == event.type()) {
+                copy.remove(last);
+            }
+        }
+        copy.add(event);
+        return new History(copy);
     }
 
     /** Returns a copy of this history with a record of this state transition added, if applicable */
@@ -106,17 +104,17 @@ public class History {
      * This returns a copy of this history with all application level events removed. 
      */
     private History withoutApplicationEvents() {
-        return new History(events().stream().filter(e -> ! e.type().isApplicationLevel()).collect(Collectors.toList()));
+        return new History(asList().stream().filter(e -> ! e.type().isApplicationLevel()).collect(Collectors.toList()));
     }
 
     /** Returns the empty history */
-    public static History empty() { return new History(Collections.emptyList()); }
+    public static History empty() { return new History(List.of()); }
 
     @Override
     public String toString() {
         if (events.isEmpty()) return "history: (empty)";
         StringBuilder b = new StringBuilder("history: ");
-        for (Event e : events.values())
+        for (Event e : events)
             b.append(e).append(", ");
          b.setLength(b.length() - 2); // remove last comma
         return b.toString();
@@ -148,27 +146,27 @@ public class History {
             readied,
             reserved,
 
-            // The node was scheduled for retirement (hard)
+            /** The node was scheduled for retirement (hard) */
             wantToRetire(false),
-            // The node was scheduled for retirement (soft)
+            /** The node was scheduled for retirement (soft) */
             preferToRetire(false),
-            // This node was scheduled for failing
+            /** This node was scheduled for failing */
             wantToFail,
-            // The active node was retired
+            /** The active node was retired */
             retired,
-            // The active node went down according to the service monitor
+            /** The active node went down according to the service monitor */
             down,
-            // The active node came up according to the service monitor
+            /** The active node came up according to the service monitor */
             up,
-            // The node made a config request, indicating it is live
+            /** The node made a config request, indicating it is live */
             requested,
-            // The node resources/flavor were changed
+            /** The node resources/flavor were changed */
             resized(false),
-            // The node was rebooted
+            /** The node was rebooted */
             rebooted(false),
-            // The node upgraded its OS (implies a reboot)
+            /** The node upgraded its OS (implies a reboot) */
             osUpgraded(false),
-            // The node verified its firmware (whether this resulted in a reboot depends on the node model)
+            /** The node verified its firmware (whether this resulted in a reboot depends on the node model) */
             firmwareVerified(false);
 
             private final boolean applicationLevel;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 4990c1e9db8..551a15aa804 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -219,7 +219,7 @@ public class NodeSerializer {
     }
 
     private void toSlime(History history, Cursor array) {
-        for (History.Event event : history.events())
+        for (History.Event event : history.asList())
             toSlime(event, array.addObject());
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
index 922c8bc8e20..2822114375f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
@@ -197,7 +197,7 @@ class NodesResponse extends SlimeJsonResponse {
     }
 
     private void toSlime(History history, Cursor array) {
-        for (History.Event event : history.events()) {
+        for (History.Event event : history.asList()) {
             Cursor object = array.addObject();
             object.setString("event", event.type().name());
             object.setLong("at", event.at().toEpochMilli());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index 5211b855fff..4972e9f6f20 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -78,7 +78,7 @@ public class InactiveAndFailedExpirerTest {
         Node ready = tester.nodeRepository().nodes().setReady(List.of(dirty.asList().get(0)), Agent.system, getClass().getSimpleName()).get(0);
         assertEquals("Allocated history is removed on readying",
                 List.of(History.Event.Type.provisioned, History.Event.Type.readied),
-                ready.history().events().stream().map(History.Event::type).collect(Collectors.toList()));
+                ready.history().asList().stream().map(History.Event::type).collect(Collectors.toList()));
 
         // Dirty times out for the other one
         tester.advanceTime(Duration.ofMinutes(14));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/HistoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/HistoryTest.java
new file mode 100644
index 00000000000..90162cc3ea3
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/HistoryTest.java
@@ -0,0 +1,59 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.node;
+
+import com.yahoo.vespa.hosted.provision.node.History.Event;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class HistoryTest {
+
+    @Test
+    public void truncate_events() {
+        assertEquals(0, new History(List.of(), 2).asList().size());
+        assertEquals(1, new History(shuffledEvents(1), 2).asList().size());
+        assertEquals(2, new History(shuffledEvents(2), 2).asList().size());
+
+        History history = new History(shuffledEvents(5), 3);
+        assertEquals(3, history.asList().size());
+        assertEquals("Most recent events are kept",
+                     List.of(2L, 3L, 4L),
+                     history.asList().stream().map(e -> e.at().toEpochMilli()).collect(Collectors.toList()));
+    }
+
+    @Test
+    public void repeating_event_overwrites_existing() {
+        Instant i0 = Instant.ofEpochMilli(1);
+        History history = new History(List.of(new Event(Event.Type.readied, Agent.system, i0)));
+
+        Instant i1 = Instant.ofEpochMilli(2);
+        history = history.with(new Event(Event.Type.reserved, Agent.system, i1));
+        assertEquals(2, history.asList().size());
+
+        Instant i2 = Instant.ofEpochMilli(3);
+        history = history.with(new Event(Event.Type.reserved, Agent.system, i2));
+
+        assertEquals(2, history.asList().size());
+        assertEquals(i2, history.asList().get(1).at());
+    }
+
+    private static List<Event> shuffledEvents(int count) {
+        Instant start = Instant.ofEpochMilli(0);
+        List<Event> events = new ArrayList<>();
+        for (int i = 0; i < count; i++) {
+            events.add(new Event(Event.Type.values()[i], Agent.system, start.plusMillis(i)));
+        }
+        Collections.shuffle(events);
+        return events;
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java
index 48ee23c7b60..44c02e49d13 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java
@@ -71,7 +71,7 @@ public class NodeSerializerTest {
         assertEquals(node.id(), copy.id());
         assertEquals(node.state(), copy.state());
         assertFalse(copy.allocation().isPresent());
-        assertEquals(0, copy.history().events().size());
+        assertEquals(0, copy.history().asList().size());
     }
 
     @Test
@@ -81,14 +81,14 @@ public class NodeSerializerTest {
                                                              DiskSpeed.any, StorageType.any, Architecture.arm64);
 
         clock.advance(Duration.ofMinutes(3));
-        assertEquals(0, node.history().events().size());
+        assertEquals(0, node.history().asList().size());
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                                                 ApplicationName.from("myApplication"),
                                                 InstanceName.from("myInstance")),
                              ClusterMembership.from("content/myId/0/0/stateful", Vtag.currentVersion, Optional.empty()),
                              requestedResources,
                              clock.instant());
-        assertEquals(1, node.history().events().size());
+        assertEquals(1, node.history().asList().size());
         node = node.withRestart(new Generation(1, 2));
         node = node.withReboot(new Generation(3, 4));
         node = node.with(FlavorConfigBuilder.createDummies("arm64").getFlavorOrThrow("arm64"), Agent.system, clock.instant());
@@ -112,7 +112,7 @@ public class NodeSerializerTest {
         assertEquals(node.allocation().get().membership(), copy.allocation().get().membership());
         assertEquals(node.allocation().get().requestedResources(), copy.allocation().get().requestedResources());
         assertEquals(node.allocation().get().isRemovable(), copy.allocation().get().isRemovable());
-        assertEquals(2, copy.history().events().size());
+        assertEquals(2, copy.history().asList().size());
         assertEquals(clock.instant().truncatedTo(MILLIS), copy.history().event(History.Event.Type.reserved).get().at());
         assertEquals(NodeType.tenant, copy.type());
     }
@@ -160,7 +160,7 @@ public class NodeSerializerTest {
         assertEquals(3, node.allocation().get().restartGeneration().wanted());
         assertEquals(4, node.allocation().get().restartGeneration().current());
         assertEquals(Arrays.asList(History.Event.Type.provisioned, History.Event.Type.reserved),
-                node.history().events().stream().map(History.Event::type).collect(Collectors.toList()));
+                node.history().asList().stream().map(History.Event::type).collect(Collectors.toList()));
         assertTrue(node.allocation().get().isRemovable());
         assertEquals(NodeType.tenant, node.type());
     }
@@ -170,18 +170,18 @@ public class NodeSerializerTest {
         Node node = createNode();
 
         clock.advance(Duration.ofMinutes(3));
-        assertEquals(0, node.history().events().size());
+        assertEquals(0, node.history().asList().size());
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                                                 ApplicationName.from("myApplication"),
                                                 InstanceName.from("myInstance")),
                              ClusterMembership.from("content/myId/0/0/stateful", Vtag.currentVersion, Optional.empty()),
                              node.flavor().resources(),
                              clock.instant());
-        assertEquals(1, node.history().events().size());
+        assertEquals(1, node.history().asList().size());
         clock.advance(Duration.ofMinutes(2));
         node = node.retire(Agent.application, clock.instant());
         Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
-        assertEquals(2, copy.history().events().size());
+        assertEquals(2, copy.history().asList().size());
         assertEquals(clock.instant().truncatedTo(MILLIS), copy.history().event(History.Event.Type.retired).get().at());
         assertEquals(Agent.application,
                      (copy.history().event(History.Event.Type.retired).get()).agent());
@@ -209,13 +209,13 @@ public class NodeSerializerTest {
                 "    \"wantedVespaVersion\": \"6.42.2\"\n" +
                 "  }\n" +
                 "}\n").getBytes());
-        assertEquals(0, node.history().events().size());
+        assertEquals(0, node.history().asList().size());
         assertTrue(node.allocation().isPresent());
         assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value());
         assertEquals("container", node.allocation().get().membership().cluster().type().name());
         assertEquals(0, node.allocation().get().membership().cluster().group().get().index());
         Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
-        assertEquals(0, copy.history().events().size());
+        assertEquals(0, copy.history().asList().size());
     }
 
     @Test
@@ -356,7 +356,7 @@ public class NodeSerializerTest {
                                .withCurrentOsVersion(Version.fromString("7.1"), Instant.ofEpochMilli(456));
         serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized));
         assertEquals(Version.fromString("7.1"), serialized.status().osVersion().current().get());
-        var osUpgradedEvents = serialized.history().events().stream()
+        var osUpgradedEvents = serialized.history().asList().stream()
                                          .filter(event -> event.type() == History.Event.Type.osUpgraded)
                                          .collect(Collectors.toList());
         assertEquals("OS upgraded event is added", 1, osUpgradedEvents.size());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json
index a73e9a7bade..3e8b19135c0 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json
@@ -37,6 +37,11 @@
   "wantToRebuild": false,
   "history": [
     {
+      "event": "firmwareVerified",
+      "at": 100,
+      "agent": "system"
+    },
+    {
       "event": "provisioned",
       "at": 123,
       "agent": "system"
@@ -55,11 +60,6 @@
       "event": "activated",
       "at": 123,
       "agent": "application"
-    },
-    {
-      "event": "firmwareVerified",
-      "at": 100,
-      "agent": "system"
     }
   ],
   "ipAddresses": [