Change hardware failure from a boolean to an enum

13 files changed, 150 insertions, 98 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 0533538b016..7751abb1d4b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -195,10 +195,8 @@ public class NodeRepository extends AbstractComponent {
         Optional<Node> nodeToDeallocate = getNode(hostname, Node.State.failed, Node.State.parked);
         if ( ! nodeToDeallocate.isPresent())
             throw new IllegalArgumentException("Could not deallocate " + hostname + ": No such node in the failed or parked state");
-        if (nodeToDeallocate.get().status().hardwareFailure()) {
-            throw new IllegalArgumentException(String.format("Could not deallocate %s: Hardware failure flag is set",
-                    hostname));
-        }
+        if (nodeToDeallocate.get().status().hardwareFailure().isPresent())
+            throw new IllegalArgumentException("Could not deallocate " + hostname + ": It has a hardware failure");
         return deallocate(Collections.singletonList(nodeToDeallocate.get())).get(0);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index ef9c65c8a01..c040f7225f3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -48,7 +48,7 @@ public class FailedExpirer extends Expirer {
     protected void expire(List<Node> expired) {
         List<Node> nodesToRecycle = new ArrayList<>();
         for (Node recycleCandidate : expired) {
-            if (recycleCandidate.status().hardwareFailure()) continue;
+            if (recycleCandidate.status().hardwareFailure().isPresent()) continue;
             if (failCountIndicatesHwFail(zone) && recycleCandidate.status().failCount() >= 5) continue;
             nodesToRecycle.add(recycleCandidate);
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 9a116478882..c4840fb9f44 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -72,7 +72,7 @@ public class NodeFailer extends Maintainer {
 
     private List<Node> readyNodesWithHardwareFailure() {
         return nodeRepository().getNodes(Node.Type.tenant, Node.State.ready).stream()
-                .filter(n -> n.status().hardwareFailure())
+                .filter(n -> n.status().hardwareFailure().isPresent())
                 .collect(Collectors.toList());
     }
     
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
index a26f02a53dc..3b1619c34c9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -20,7 +20,21 @@ public class Status {
     private final Optional<String> stateVersion;
     private final Optional<String> dockerImage;
     private final int failCount;
-    private final boolean hardwareFailure;
+    private final Optional<HardwareFailureType> hardwareFailure;
+    
+    public enum HardwareFailureType {
+        
+        /** There are mce log error messages */
+        mce,
+        /** There are SMART log error messages */
+        smart,
+        /** There are kernel log error messages */
+        kernel,
+        /** There is an error but its type is unknown */ 
+        // TODO: Remove this when all hosts in the node repo has a failure type
+        unknown
+        
+    }
 
     public Status(Generation generation,
                   Optional<Version> vespaVersion,
@@ -28,7 +42,7 @@ public class Status {
                   Optional<String> stateVersion,
                   Optional<String> dockerImage,
                   int failCount,
-                  boolean hardwareFailure) {
+                  Optional<HardwareFailureType> hardwareFailure) {
         this.reboot = generation;
         this.vespaVersion = vespaVersion;
         this.hostedVersion = hostedVersion;
@@ -82,12 +96,12 @@ public class Status {
     /** Returns how many times this node has been moved to the failed state. */
     public int failCount() { return failCount; }
 
-    /** Returns whether a hardware failure has been detected on this node */
-    public boolean hardwareFailure() { return hardwareFailure; }
+    /** Returns the type of the last hardware failure detected on this node, or empty if none */
+    public Optional<HardwareFailureType> hardwareFailure() { return hardwareFailure; }
 
-    public Status setHardwareFailure(boolean hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); }
+    public Status setHardwareFailure(Optional<HardwareFailureType> hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, dockerImage, failCount, hardwareFailure); }
 
     /** Returns the initial status of a newly provisioned node */
-    public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, false); }
+    public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, Optional.empty()); }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 00282ccb53d..40f95021751 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -11,6 +11,7 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
+import com.yahoo.slime.Type;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
@@ -42,8 +43,6 @@ public class NodeSerializer {
     // Node fields
     private static final String hostnameKey = "hostname";
     private static final String openStackIdKey = "openStackId";
-    // TODO Legacy name. Remove when 5.120 is released everywhere
-    private static final String dockerHostHostNameKey = "dockerHostHostName";
     private static final String parentHostnameKey = "parentHostname";
     private static final String configurationKey ="configuration";
     private static final String historyKey = "history";
@@ -102,10 +101,10 @@ public class NodeSerializer {
         object.setLong(currentRebootGenerationKey, node.status().reboot().current());
         node.status().vespaVersion().ifPresent(version -> object.setString(vespaVersionKey, version.toString()));
         node.status().hostedVersion().ifPresent(version -> object.setString(hostedVersionKey, version.toString()));
-        node.status().stateVersion().ifPresent(version -> object.setString(stateVersionKey, version.toString()));
+        node.status().stateVersion().ifPresent(version -> object.setString(stateVersionKey, version));
         node.status().dockerImage().ifPresent(image -> object.setString(dockerImageKey, image));
         object.setLong(failCountKey, node.status().failCount());
-        object.setBool(hardwareFailureKey, node.status().hardwareFailure());
+        node.status().hardwareFailure().ifPresent(failure -> object.setString(hardwareFailureKey, toString(failure)));
         node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey)));
         toSlime(node.history(), object.setArray(historyKey));
         object.setString(nodeTypeKey, toString(node.type()));
@@ -154,18 +153,17 @@ public class NodeSerializer {
                         state,
                         allocationFromSlime(object.field(instanceKey)),
                         historyFromSlime(object.field(historyKey)),
-                        typeFromSlime(object.field(nodeTypeKey)));
+                        nodeTypeFromString(object.field(nodeTypeKey).asString()));
     }
 
     private Status statusFromSlime(Inspector object) {
-        return new Status(
-                generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey),
-                softwareVersionFromSlime(object.field(vespaVersionKey)),
-                softwareVersionFromSlime(object.field(hostedVersionKey)),
-                optionalString(object.field(stateVersionKey)),
-                optionalString(object.field(dockerImageKey)),
-                (int)object.field(failCountKey).asLong(),
-                object.field(hardwareFailureKey).asBool());
+        return new Status(generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey),
+                          softwareVersionFromSlime(object.field(vespaVersionKey)),
+                          softwareVersionFromSlime(object.field(hostedVersionKey)),
+                          optionalString(object.field(stateVersionKey)),
+                          optionalString(object.field(dockerImageKey)),
+                          (int)object.field(failCountKey).asLong(),
+                          hardwareFailureFromSlime(object.field(hardwareFailureKey)));
     }
 
     private Configuration configurationFromSlime(Inspector object) {
@@ -174,17 +172,17 @@ public class NodeSerializer {
 
     private Optional<Allocation> allocationFromSlime(Inspector object) {
         if ( ! object.valid()) return Optional.empty();
-        return Optional.of(new Allocation(
-                applicationIdFromSlime(object),
-                ClusterMembership.from(object.field(serviceIdKey).asString(), optionalString(object.field(dockerImageKey))),
-                generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey),
-                object.field(removableKey).asBool()));
+        return Optional.of(new Allocation(applicationIdFromSlime(object),
+                                          ClusterMembership.from(object.field(serviceIdKey).asString(), 
+                                                                 optionalString(object.field(dockerImageKey))),
+                                          generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey),
+                                          object.field(removableKey).asBool()));
     }
 
     private ApplicationId applicationIdFromSlime(Inspector object) {
         return ApplicationId.from(TenantName.from(object.field(tenantIdKey).asString()),
-                ApplicationName.from(object.field(applicationIdKey).asString()),
-                InstanceName.from(object.field(instanceIdKey).asString()));
+                                  ApplicationName.from(object.field(applicationIdKey).asString()),
+                                  InstanceName.from(object.field(instanceIdKey).asString()));
     }
 
     private History historyFromSlime(Inspector array) {
@@ -221,12 +219,15 @@ public class NodeSerializer {
     private Optional<String> parentHostnameFromSlime(Inspector object) {
         if (object.field(parentHostnameKey).valid())
             return Optional.of(object.field(parentHostnameKey).asString());
-        // TODO Remove when 5.120 is released everywhere
-        else if (object.field(dockerHostHostNameKey).valid())
-            return Optional.of(object.field(dockerHostHostNameKey).asString());
         else
             return Optional.empty();
     }
+    
+    private Optional<Status.HardwareFailureType> hardwareFailureFromSlime(Inspector object) {
+        if ( ! object.valid()) return Optional.empty();
+        if (object.type() == Type.BOOL) return Optional.of(Status.HardwareFailureType.unknown); // TODO: Remove this line when 6.28 is deployed everywhere
+        return Optional.of(hardwareFailureFromString(object.asString()));
+    }
 
     // Enum <-> string mappings
     
@@ -273,12 +274,11 @@ public class NodeSerializer {
         throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
     }
 
-    private Node.Type typeFromSlime(Inspector object) {
-        if ( ! object.valid()) return Node.Type.tenant; // TODO: Remove this and change to pass string line when 6.13 is released everywhere
-        switch (object.asString()) {
+    private Node.Type nodeTypeFromString(String typeString) {
+        switch (typeString) {
             case "tenant" : return Node.Type.tenant;
             case "host" : return Node.Type.host;
-            default : throw new IllegalArgumentException("Unknown node type '" + object.asString() + "'");
+            default : throw new IllegalArgumentException("Unknown node type '" + typeString + "'");
         }
     }
     private String toString(Node.Type type) {
@@ -286,7 +286,26 @@ public class NodeSerializer {
             case tenant: return "tenant";
             case host: return "host";
         }
-        throw new IllegalArgumentException("Unknown node type '" + type.toString() + "'");
+        throw new IllegalArgumentException("Serialized form of '" + type + "' not defined");
+    }
+
+    private Status.HardwareFailureType hardwareFailureFromString(String hardwareFailureString) {
+        switch (hardwareFailureString) {
+            case "mce" : return Status.HardwareFailureType.mce;
+            case "smart" : return Status.HardwareFailureType.smart;
+            case "kernel" : return Status.HardwareFailureType.kernel;
+            case "unknown" : return Status.HardwareFailureType.unknown;
+            default : throw new IllegalArgumentException("Unknown hardware failure '" + hardwareFailureString + "'");
+        }
+    }
+    private String toString(Status.HardwareFailureType type) {
+        switch (type) {
+            case mce: return "mce";
+            case smart: return "smart";
+            case kernel: return "kernel";
+            case unknown: return "unknown";
+            default : throw new IllegalArgumentException("Serialized form of '" + type + " not defined");
+        }
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 5d1b8a65b3c..d233af58e73 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -71,8 +72,10 @@ public class NodePatcher {
                 return node.setStatus(node.status().setFailCount(asLong(value).intValue()));
             case "flavor" :
                 return node.setConfiguration(node.configuration().setFlavor(nodeFlavors.getFlavorOrThrow(asString(value))));
-            case "hardwareFailure" :
-                return node.setStatus(node.status().setHardwareFailure(asBoolean(value)));
+            case "hardwareFailure" : // TODO (Aug 2016): Remove support for this when mpolden says ok
+                return node.setStatus(node.status().setHardwareFailure(toHardwareFailureType(asBoolean(value))));
+            case "hardwareFailureType" :
+                return node.setStatus(node.status().setHardwareFailure(toHardwareFailureType(asString(value))));
             case "parentHostname" :
                 return node.setParentHostname(asString(value));
             default :
@@ -106,4 +109,18 @@ public class NodePatcher {
         return field.asBool();
     }
 
+    private Optional<Status.HardwareFailureType> toHardwareFailureType(boolean failure) {
+        return failure ? Optional.of(Status.HardwareFailureType.unknown) : Optional.empty();
+    }
+
+    private Optional<Status.HardwareFailureType> toHardwareFailureType(String failureType) {
+        switch (failureType) {
+            case "mce" : return Optional.of(Status.HardwareFailureType.mce);
+            case "smart" : return Optional.of(Status.HardwareFailureType.smart);
+            case "kernel" : return Optional.of(Status.HardwareFailureType.kernel);
+            case "unknown" : throw new IllegalArgumentException("An actual hardware failure type must be provided, not 'unknown'");
+            default : throw new IllegalArgumentException("Unknown hardware failure '" + failureType + "'");
+        }
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 87a56d39183..9983f2706bd 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
 import com.yahoo.vespa.hosted.provision.restapi.NodeStateSerializer;
 
@@ -167,8 +168,7 @@ class NodesResponse extends HttpResponse {
             toSlime(allocation.get().membership(), object.setObject("membership"));
             object.setLong("restartGeneration", allocation.get().restartGeneration().wanted());
             object.setLong("currentRestartGeneration", allocation.get().restartGeneration().current());
-            allocation.get().membership().cluster().dockerImage().ifPresent(
-                    image -> object.setString("wantedDockerImage", image));
+            allocation.get().membership().cluster().dockerImage().ifPresent(image -> object.setString("wantedDockerImage", image));
         }
         object.setLong("rebootGeneration", node.status().reboot().wanted());
         object.setLong("currentRebootGeneration", node.status().reboot().current());
@@ -179,7 +179,8 @@ class NodesResponse extends HttpResponse {
         node.status().dockerImage().ifPresent(image -> object.setString("currentDockerImage", image));
         node.status().stateVersion().ifPresent(version -> object.setString("convergedStateVersion", version));
         object.setLong("failCount", node.status().failCount());
-        object.setBool("hardwareFailure", node.status().hardwareFailure());
+        object.setBool("hardwareFailure", node.status().hardwareFailure().isPresent());
+        node.status().hardwareFailure().ifPresent(failure -> object.setString("hardwareFailureType", toString(failure)));
         toSlime(node.history(), object.setArray("history"));
     }
 
@@ -227,4 +228,14 @@ class NodesResponse extends HttpResponse {
                 .orElseThrow(() -> new RuntimeException("Node state '" + stateString + "' is not known"));
     }
 
+    private String toString(Status.HardwareFailureType type) {
+        switch (type) {
+            case mce: return "mce";
+            case smart: return "smart";
+            case kernel: return "kernel";
+            case unknown: return "unknown";
+            default : throw new IllegalArgumentException("Serialized form of '" + type + " not defined");
+        }
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index a1d9268ee33..4be148ccf97 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Configuration;
 import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
 import org.junit.Test;
@@ -87,7 +88,7 @@ public class FailedExpirerTest {
 
         // Set node2 to have a detected hardware failure
         Node node2 = nodeRepository.getNode("node2").get();
-        node2 = node2.setStatus(node2.status().setHardwareFailure(true));
+        node2 = node2.setStatus(node2.status().setHardwareFailure(Optional.of(Status.HardwareFailureType.mce)));
         nodeRepository.write(node2);
 
         // Allocate the nodes
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java
index c568b9db2a7..8c2e4d3c8f7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java
@@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Configuration;
 import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
 import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException;
@@ -148,8 +149,8 @@ public class NodeFailerTest {
         // Failures are detected on two ready nodes, which are then failed
         Node readyFail1 = nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).get(2);
         Node readyFail2 = nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).get(3);
-        nodeRepository.write(readyFail1.setStatus(readyFail1.status().setHardwareFailure(true)));
-        nodeRepository.write(readyFail2.setStatus(readyFail2.status().setHardwareFailure(true)));
+        nodeRepository.write(readyFail1.setStatus(readyFail1.status().setHardwareFailure(Optional.of(Status.HardwareFailureType.mce))));
+        nodeRepository.write(readyFail2.setStatus(readyFail2.status().setHardwareFailure(Optional.of(Status.HardwareFailureType.smart))));
         assertEquals(4, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size());
         failer.run();
         assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java
index 7f80a83bef7..3873ff1de98 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java
@@ -24,18 +24,6 @@ public class CuratorDatabaseClientTest {
     private CuratorDatabaseClient zkClient = new CuratorDatabaseClient(FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC());
 
     @Test
-    public void ensure_can_read_stored_host_with_instance_information_no_type() throws Exception {
-        String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159,\"allocated\":1421057746687,\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"music\",\"instanceId\":\"default\",\"serviceId\":\"container/default/0/0\"}}";
-
-        curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/allocated/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes());
-
-        List<Node> allocatedNodes = zkClient.getNodes(Node.State.active);
-        assertEquals(1, allocatedNodes.size());
-        assertEquals("container/default/0/0", allocatedNodes.get(0).allocation().get().membership().stringValue());
-        assertEquals(Node.Type.tenant, allocatedNodes.get(0).type());
-    }
-
-    @Test
     public void ensure_can_read_stored_host_information() throws Exception {
         String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159, \"type\":\"host\"}";
         curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 9ba29cec588..be45dfa152e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.node.Configuration;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.node.NodeFlavors;
+import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder;
 import org.junit.Test;
 
@@ -64,7 +65,7 @@ public class SerializationTest {
         node = node.setFlavor(FlavorConfigBuilder.createDummies("large").getFlavorOrThrow("large"));
         node = node.setStatus(node.status().setVespaVersion(Version.fromString("1.2.3")));
         node = node.setStatus(node.status().increaseFailCount().increaseFailCount());
-        node = node.setStatus(node.status().setHardwareFailure(true));
+        node = node.setStatus(node.status().setHardwareFailure(Optional.of(Status.HardwareFailureType.mce)));
         node = node.setType(Node.Type.tenant);
         Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
 
@@ -78,7 +79,7 @@ public class SerializationTest {
         assertEquals("large", copy.configuration().flavor().name());
         assertEquals("1.2.3", copy.status().vespaVersion().get().toString());
         assertEquals(2, copy.status().failCount());
-        assertEquals(true, copy.status().hardwareFailure());
+        assertEquals(Status.HardwareFailureType.mce, copy.status().hardwareFailure().get());
         assertEquals(node.allocation().get().owner(), copy.allocation().get().owner());
         assertEquals(node.allocation().get().membership(), copy.allocation().get().membership());
         assertEquals(node.allocation().get().removable(), copy.allocation().get().removable());
@@ -100,7 +101,9 @@ public class SerializationTest {
 
     @Test
     public void testRebootAndRestartandTypeNoCurrentValuesSerialization() {
-        String nodeData = "{\n" +
+        String nodeData = 
+                "{\n" +
+                "   \"type\" : \"tenant\",\n" +
                 "   \"rebootGeneration\" : 0,\n" +
                 "   \"configuration\" : {\n" +
                 "      \"flavor\" : \"default\"\n" +
@@ -132,6 +135,38 @@ public class SerializationTest {
         assertEquals(Node.Type.tenant, node.type());
     }
 
+    // TODO: Remove when 6.28 is deployed everywhere
+    @Test
+    public void testLegacyHardwareFailureBooleanDeserialization() {
+        String nodeData =
+                "{\n" +
+                "   \"type\" : \"tenant\",\n" +
+                "   \"rebootGeneration\" : 0,\n" +
+                "   \"configuration\" : {\n" +
+                "      \"flavor\" : \"default\"\n" +
+                "   },\n" +
+                "   \"history\" : [\n" +
+                "      {\n" +
+                "         \"type\" : \"reserved\",\n" +
+                "         \"at\" : 1444391402611\n" +
+                "      }\n" +
+                "   ],\n" +
+                "   \"instance\" : {\n" +
+                "      \"applicationId\" : \"myApplication\",\n" +
+                "      \"tenantId\" : \"myTenant\",\n" +
+                "      \"instanceId\" : \"myInstance\",\n" +
+                "      \"serviceId\" : \"content/myId/0\",\n" +
+                "      \"restartGeneration\" : 0,\n" +
+                "      \"removable\" : false\n" +
+                "   },\n" +
+                "   \"openStackId\" : \"myId\",\n" +
+                "   \"hostname\" : \"myHostname\",\n" +
+                "   \"hardwareFailure\" : true\n" +
+                "}";
+
+        Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData));
+        assertEquals(Status.HardwareFailureType.unknown, node.status().hardwareFailure().get());
+    }
     @Test
     public void testRetiredNodeSerialization() {
         Node node = createNode();
@@ -160,7 +195,7 @@ public class SerializationTest {
 
     @Test
     public void testAssimilatedDeserialization() {
-        Node node = nodeSerializer.fromJson(Node.State.active, "{\"hostname\":\"assimilate2.vespahosted.corp.bf1.yahoo.com\",\"openStackId\":\"\",\"configuration\":{\"flavor\":\"ugccloud-container\"},\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes());
+        Node node = nodeSerializer.fromJson(Node.State.active, "{\"type\":\"tenant\",\"hostname\":\"assimilate2.vespahosted.corp.bf1.yahoo.com\",\"openStackId\":\"\",\"configuration\":{\"flavor\":\"ugccloud-container\"},\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes());
         assertEquals(0, node.history().events().size());
         assertTrue(node.allocation().isPresent());
         assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value());
@@ -214,39 +249,6 @@ public class SerializationTest {
         assertEquals(parentHostname, deserializedNode.parentHostname().get());
     }
 
-    // TODO: Remove when 5.120 is released everywhere
-    @Test
-    public void serialize_parentHostname_from_dockerHostHostName() {
-        final String parentHostname = "parent.yahoo.com";
-        String nodeData = "{\n" +
-                "   \"rebootGeneration\" : 0,\n" +
-                "   \"configuration\" : {\n" +
-                "      \"flavor\" : \"default\"\n" +
-                "   },\n" +
-                "   \"history\" : [\n" +
-                "      {\n" +
-                "         \"type\" : \"reserved\",\n" +
-                "         \"at\" : 1444391402611\n" +
-                "      }\n" +
-                "   ],\n" +
-                "   \"instance\" : {\n" +
-                "      \"applicationId\" : \"myApplication\",\n" +
-                "      \"tenantId\" : \"myTenant\",\n" +
-                "      \"instanceId\" : \"myInstance\",\n" +
-                "      \"serviceId\" : \"content/myId/0\",\n" +
-                "      \"restartGeneration\" : 0,\n" +
-                "      \"removable\" : false\n" +
-                "   },\n" +
-                "   \"openStackId\" : \"fooId\",\n" +
-                "   \"hostname\" : \"fooHost\",\n" +
-                "   \"dockerHostHostName\" : \"" + parentHostname + "\"\n" +
-                "}";
-        // No parent hostname, but dockerHostHostName is set, so parent hostname should be set after deserialization
-
-        Node deserializedNode2 = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData));
-        assertEquals(parentHostname, deserializedNode2.parentHostname().get());
-    }
-
     private Node createNode() {
         return Node.create("myId", "myHostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host);
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index cdb5b246862..d519080a79e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -184,7 +184,7 @@ public class RestApiTest {
                 "{\"message\":\"Moved host12.yahoo.com to failed\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host12.yahoo.com",
                 new byte[0], Request.Method.PUT), 400,
-                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not deallocate host12.yahoo.com: Hardware failure flag is set\"}");
+                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not deallocate host12.yahoo.com: It has a hardware failure\"}");
     }
 
     @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index 80b06a8056e..a08c1aeaafb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -36,6 +36,7 @@
   "convergedStateVersion": "5.104.142-2.1.2408",
   "failCount": 0,
   "hardwareFailure": true,
+  "hardwareFailureType":"unknown",
   "history": [
     {
       "event": "readied",