Firmware check timestamps in node-repo, accessible through REST API

11 files changed, 252 insertions, 14 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index dca4ea331fd..c870d7d830c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter;
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
 import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver;
 import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
+import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks;
 import com.yahoo.vespa.hosted.provision.provisioning.OsVersions;
 import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException;
 
@@ -81,6 +82,7 @@ public class NodeRepository extends AbstractComponent {
     private final NameResolver nameResolver;
     private final DockerImage dockerImage;
     private final OsVersions osVersions;
+    private final FirmwareChecks firmwareChecks;
     private final Flags flags;
 
     /**
@@ -105,6 +107,7 @@ public class NodeRepository extends AbstractComponent {
         this.nameResolver = nameResolver;
         this.dockerImage = dockerImage;
         this.osVersions = new OsVersions(this.db);
+        this.firmwareChecks = new FirmwareChecks(db, clock);
         this.flags = new Flags(this.db);
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
@@ -124,6 +127,9 @@ public class NodeRepository extends AbstractComponent {
     /** Returns the OS versions to use for nodes in this */
     public OsVersions osVersions() { return osVersions; }
 
+    /** Returns the status of firmware checks for hosts managed by this. */
+    public FirmwareChecks firmwareChecks() { return firmwareChecks; }
+
     /** Returns feature flags of this node repository */
     public Flags flags() {
         return flags;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
index feaa4d8241d..82470787f5b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.node;
 import com.yahoo.component.Version;
 
 import javax.annotation.concurrent.Immutable;
+import java.time.Instant;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -23,6 +24,7 @@ public class Status {
     private final boolean wantToDeprovision;
     private final Optional<String> hardwareDivergence;
     private final Optional<Version> osVersion;
+    private final Optional<Instant> firmwareVerifiedAt;
 
     public Status(Generation generation,
                   Optional<Version> vespaVersion,
@@ -31,7 +33,8 @@ public class Status {
                   boolean wantToRetire,
                   boolean wantToDeprovision,
                   Optional<String> hardwareDivergence,
-                  Optional<Version> osVersion) {
+                  Optional<Version> osVersion,
+                  Optional<Instant> firmwareVerifiedAt) {
         Objects.requireNonNull(hardwareDivergence, "Hardware divergence must be non-null");
         hardwareDivergence.ifPresent(s -> requireNonEmptyString(s, "Hardware divergence must be non-empty"));
         this.reboot = Objects.requireNonNull(generation, "Generation must be non-null");
@@ -42,37 +45,38 @@ public class Status {
         this.wantToDeprovision = wantToDeprovision;
         this.hardwareDivergence = hardwareDivergence;
         this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null");
+        this.firmwareVerifiedAt = Objects.requireNonNull(firmwareVerifiedAt, "Firmware check instant must be non-null");
     }
 
     /** Returns a copy of this with the reboot generation changed */
-    public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
     /** Returns the reboot generation of this node */
     public Generation reboot() { return reboot; }
 
     /** Returns a copy of this with the vespa version changed */
-    public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
     /** Returns the Vespa version installed on the node, if known */
     public Optional<Version> vespaVersion() { return vespaVersion; }
 
-    public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
-    public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
-    public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
     /** Returns how many times this node has been moved to the failed state. */
     public int failCount() { return failCount; }
 
-    public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); }
+    public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt); }
 
     /** Returns the type of the last hardware failure detected on this node, or empty if none */
     public Optional<String> hardwareFailureDescription() { return hardwareFailureDescription; }
 
     /** Returns a copy of this with the want to retire flag changed */
     public Status withWantToRetire(boolean wantToRetire) {
-        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
+        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt);
     }
 
     /**
@@ -85,7 +89,7 @@ public class Status {
 
     /** Returns a copy of this with the want to de-provision flag changed */
     public Status withWantToDeprovision(boolean wantToDeprovision) {
-        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
+        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt);
     }
 
     /**
@@ -96,7 +100,7 @@ public class Status {
     }
 
     public Status withHardwareDivergence(Optional<String> hardwareDivergence) {
-        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion);
+        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, firmwareVerifiedAt);
     }
 
     /** Returns hardware divergence report as JSON string, if any */
@@ -104,7 +108,7 @@ public class Status {
 
     /** Returns a copy of this with the current OS version set to version */
     public Status withOsVersion(Version version) {
-        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, Optional.of(version));
+        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, Optional.of(version), firmwareVerifiedAt);
     }
 
     /** Returns the current OS version of this node, if any */
@@ -112,10 +116,20 @@ public class Status {
         return osVersion;
     }
 
+    /** Returns a copy of this with the firmwareVerifiedAt set to the given instant. */
+    public Status withFirmwareVerifiedAt(Instant instant) {
+        return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion, Optional.of(instant));
+    }
+
+    /** Returns the last time this node had firmware that was verified to be up to date. */
+    public Optional<Instant> firmwareVerifiedAt() {
+        return firmwareVerifiedAt;
+    }
+
     /** Returns the initial status of a newly provisioned node */
     public static Status initial() {
         return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false,
-                          false, Optional.empty(), Optional.empty());
+                          false, Optional.empty(), Optional.empty(), Optional.empty());
     }
 
     private void requireNonEmptyString(String value, String message) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index da4d2a0afb2..139f42b0f37 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -25,8 +25,10 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Status;
 
+import java.nio.charset.StandardCharsets;
 import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -88,6 +90,7 @@ public class CuratorDatabaseClient {
         curatorDatabase.create(inactiveJobsPath());
         curatorDatabase.create(infrastructureVersionsPath());
         curatorDatabase.create(osVersionsPath());
+        curatorDatabase.create(firmwareCheckPath());
         curatorDatabase.create(loadBalancersRoot);
         curatorDatabase.create(flagsRoot);
     }
@@ -424,6 +427,25 @@ public class CuratorDatabaseClient {
         return root.append("osVersions");
     }
 
+    /** Stores the instant after which a firmware check is required, or clears any outstanding ones if empty is given. */
+    public void writeFirmwareCheck(Optional<Instant> after) {
+        byte[] data = after.map(instant -> Long.toString(instant.toEpochMilli()).getBytes())
+                           .orElse(new byte[0]);
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(firmwareCheckPath().getAbsolute(), data));
+        transaction.commit();
+    }
+
+    /** Returns the instant after which a firmware check is required, if any. */
+    public Optional<Instant> readFirmwareCheck() {
+        return read(firmwareCheckPath(), data -> Instant.ofEpochMilli(Long.parseLong(new String(data))));
+    }
+
+    private Path firmwareCheckPath() {
+        return root.append("firmwareCheck");
+    }
+
     public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() {
         return curatorDatabase.getChildren(loadBalancersRoot).stream()
                               .map(LoadBalancerId::fromSerializedForm)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 6374ad9ce91..5175ea27396 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -61,6 +61,7 @@ public class NodeSerializer {
     private static final String wantToDeprovisionKey = "wantToDeprovision";
     private static final String hardwareDivergenceKey = "hardwareDivergence";
     private static final String osVersionKey = "osVersion";
+    private static final String firmwareCheckKey = "firmwareCheck";
 
     // Configuration fields
     private static final String flavorKey = "flavor";
@@ -118,6 +119,7 @@ public class NodeSerializer {
         node.status().hardwareDivergence().ifPresent(hardwareDivergence -> object.setString(hardwareDivergenceKey,
                                                                                             hardwareDivergence));
         node.status().osVersion().ifPresent(version -> object.setString(osVersionKey, version.toString()));
+        node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli()));
     }
 
     private void toSlime(Allocation allocation, Cursor object) {
@@ -175,7 +177,8 @@ public class NodeSerializer {
                           object.field(wantToRetireKey).asBool(),
                           object.field(wantToDeprovisionKey).asBool(),
                           removeQuotedNulls(hardwareDivergenceFromSlime(object)),
-                          versionFromSlime(object.field(osVersionKey)));
+                          versionFromSlime(object.field(osVersionKey)),
+                          instantFromSlime(object.field(firmwareCheckKey)));
     }
 
     private Flavor flavorFromSlime(Inspector object) {
@@ -229,6 +232,12 @@ public class NodeSerializer {
         return Optional.of(Version.fromString(object.asString()));
     }
 
+    private Optional<Instant> instantFromSlime(Inspector object) {
+        if ( ! object.valid())
+            return Optional.empty();
+        return Optional.of(Instant.ofEpochMilli(object.asLong()));
+    }
+
     private Optional<String> parentHostnameFromSlime(Inspector object) {
         if (object.field(parentHostnameKey).valid())
             return Optional.of(object.field(parentHostnameKey).asString());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java
new file mode 100644
index 00000000000..514745a9207
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java
@@ -0,0 +1,60 @@
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Keeps cached data about when to do a firmware check on the hosts managed by a node repository.
+ *
+ * The data kept here is managed through POST and DELETE to <code>/nodes/v2/upgrade/firmware</code>
+ * <p>
+ * Reads and writes are not locked, as writes do not depend on prior state.
+ * <p>
+ * Local cache expires periodically, and on writes from this host, for testing.
+ *
+ * @author jonmv
+ */
+public class FirmwareChecks {
+
+    private static final Duration cacheExpiry = Duration.ofMinutes(1);
+
+    private final CuratorDatabaseClient database;
+    private final Clock clock;
+
+    private Supplier<Optional<Instant>> checkAfter;
+
+    public FirmwareChecks(CuratorDatabaseClient database, Clock clock) {
+        this.database = database;
+        this.clock = clock;
+        createCache();
+    }
+
+    /** Returns the instant after which a firmware check is required, or empty if none currently are. */
+    public Optional<Instant> requiredAfter() {
+        return checkAfter.get();
+    }
+
+    /** Requests a firmware check for all hosts managed by this node repository. */
+    public void request() {
+        database.writeFirmwareCheck(Optional.of(clock.instant()));
+        createCache();
+    }
+
+    /** Clears any outstanding firmware checks for this node repository. */
+    public void cancel() {
+        database.writeFirmwareCheck(Optional.empty());
+        createCache();
+    }
+
+    private void createCache() {
+        checkAfter = Suppliers.memoizeWithExpiration(database::readFirmwareCheck, cacheExpiry.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 3c54c480c44..7d1e6f77e58 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -116,6 +117,8 @@ public class NodePatcher {
                 return node.with(node.status().withVespaVersion(Version.fromString(asString(value))));
             case "currentOsVersion" :
                 return node.with(node.status().withOsVersion(Version.fromString(asString(value))));
+            case "currentFirmwareCheck":
+                return node.with(node.status().withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value))));
             case "failCount" :
                 return node.with(node.status().setFailCount(asLong(value).intValue()));
             case "flavor" :
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index 418a3b16e2d..7f1959d548e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -37,6 +37,7 @@ import javax.inject.Inject;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -170,6 +171,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false);
         if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), true, "", "");
         if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), true, path.get("dimension"), path.get("value"));
+        if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse();
 
         throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
     }
@@ -184,6 +186,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true);
         if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), false, "", "");
         if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), false, path.get("dimension"), path.get("value"));
+        if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse();
 
         throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
     }
@@ -299,7 +302,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
     private MessageResponse setTargetVersions(HttpRequest request) {
         NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase());
         Inspector inspector = toSlime(request.getData()).get();
-        List<String> messageParts = new ArrayList<>(2);
+        List<String> messageParts = new ArrayList<>(3);
 
         boolean force = inspector.field("force").asBool();
         Inspector versionField = inspector.field("version");
@@ -331,6 +334,16 @@ public class NodesApiHandler extends LoggingRequestHandler {
                                    " for nodes of type " + nodeType);
     }
 
+    private MessageResponse cancelFirmwareCheckResponse() {
+        nodeRepository.firmwareChecks().cancel();
+        return new MessageResponse("Cancelled outstanding requests for firmware checks");
+    }
+
+    private MessageResponse requestFirmwareCheckResponse() {
+        nodeRepository.firmwareChecks().request();
+        return new MessageResponse("Will request firmware checks on all hosts.");
+    }
+
     private static String hostnamesAsString(List<Node> nodes) {
         return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", "));
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index d2ab3c20080..258e02eae0e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -168,6 +168,9 @@ class NodesResponse extends HttpResponse {
         object.setLong("currentRebootGeneration", node.status().reboot().current());
         node.status().osVersion().ifPresent(version -> object.setString("currentOsVersion", version.toFullString()));
         nodeRepository.osVersions().targetFor(node.type()).ifPresent(version -> object.setString("wantedOsVersion", version.toFullString()));
+        node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli()));
+        if (node.type().isDockerHost())
+            nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli()));
         node.status().vespaVersion()
                 .filter(version -> !version.isEmpty())
                 .ifPresent(version -> {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 5804add3fc0..603c7ead268 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -23,6 +23,7 @@ import org.junit.Test;
 
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Optional;
@@ -311,6 +312,16 @@ public class SerializationTest {
     }
 
     @Test
+    public void firmware_check_serialization() {
+        Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode()));
+        assertFalse(node.status().firmwareVerifiedAt().isPresent());
+
+        node = node.with(node.status().withFirmwareVerifiedAt(Instant.ofEpochMilli(100)));
+        node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node));
+        assertEquals(100, node.status().firmwareVerifiedAt().get().toEpochMilli());
+    }
+
+    @Test
     public void serialize_node_types() {
         for (NodeType t : NodeType.values()) {
             assertEquals(t, NodeSerializer.nodeTypeFromString(NodeSerializer.toString(t)));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index bee9bf3625a..840cfa7ab5f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -665,6 +665,30 @@ public class RestApiTest {
     }
 
     @Test
+    public void test_firmware_upgrades() throws IOException {
+        // dockerhost1 checks firmware at time 100.
+        assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+                                   Utf8.toBytes("{\"currentFirmwareCheck\":100}"), Request.Method.PATCH),
+                       "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
+
+        // Schedule a firmware check at time 123 (the mock default).
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.POST),
+                       "{\"message\":\"Will request firmware checks on all hosts.\"}");
+
+        // dockerhost1 displays both values.
+        assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"),
+                       "dockerhost1-with-firmware-data.json");
+
+        // host1 has no wantedFirmwareCheck, as it's not a docker host.
+        assertFile(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"),
+                   "node1.json");
+
+        // Cancel the firmware check.
+        assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.DELETE),
+                       "{\"message\":\"Cancelled outstanding requests for firmware checks\"}");
+    }
+
+    @Test
     public void test_flags() throws Exception {
         assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags1.json");
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json
new file mode 100644
index 00000000000..7d3e0b43b53
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json
@@ -0,0 +1,73 @@
+{
+  "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+  "id": "dockerhost1.yahoo.com",
+  "state": "active",
+  "type": "host",
+  "hostname": "dockerhost1.yahoo.com",
+  "openStackId": "dockerhost1",
+  "flavor": "large",
+  "canonicalFlavor": "large",
+  "minDiskAvailableGb": 1600.0,
+  "minMainMemoryAvailableGb": 32.0,
+  "description": "Flavor-name-is-large",
+  "minCpuCores": 4.0,
+  "fastDisk": true,
+  "bandwidth": 0.0,
+  "environment": "BARE_METAL",
+  "owner": {
+    "tenant": "zoneapp",
+    "application": "zoneapp",
+    "instance": "zoneapp"
+  },
+  "membership": {
+    "clustertype": "container",
+    "clusterid": "node-admin",
+    "group": "0",
+    "index": 0,
+    "retired": false
+  },
+  "restartGeneration": 0,
+  "currentRestartGeneration": 0,
+  "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+  "wantedVespaVersion": "6.42.0",
+  "allowedToBeDown": false,
+  "rebootGeneration": 0,
+  "currentRebootGeneration": 0,
+  "currentFirmwareCheck": 100,
+  "wantedFirmwareCheck": 123,
+  "failCount": 0,
+  "hardwareFailure": false,
+  "wantToRetire": false,
+  "wantToDeprovision": false,
+  "history": [
+    {
+      "event": "provisioned",
+      "at": 123,
+      "agent": "system"
+    },
+    {
+      "event": "readied",
+      "at": 123,
+      "agent": "system"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "activated",
+      "at": 123,
+      "agent": "application"
+    }
+  ],
+  "ipAddresses": [
+    "127.0.0.1",
+    "::1"
+  ],
+  "additionalIpAddresses": [
+    "::2",
+    "::3",
+    "::4"
+  ]
+}