Implement HostEncrypter

11 files changed, 305 insertions, 8 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
index 5460981a1b0..624e4c61662 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
@@ -50,6 +50,7 @@ public class NodeHistory {
         RebuildingOsUpgrader,
         SpareCapacityMaintainer,
         SwitchRebalancer,
+        HostEncrypter,
     }
 
 }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 4f1002b91a7..ff5aa450bb1 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -252,6 +252,12 @@ public class Flags {
             "Takes effect on next deployment through controller",
             APPLICATION_ID);
 
+    public static final UnboundIntFlag MAX_ENCRYPTING_HOSTS = defineIntFlag(
+            "max-encrypting-hosts", 0,
+            List.of("mpolden", "hakonhall"), "2021-05-27", "2021-10-01",
+            "The maximum number of hosts allowed to encrypt their disk concurrently",
+            "Takes effect immediately, but any currently encrypting hosts will not be cancelled when reducing the limit");
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index d74817fb8cd..594f2b6e619 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.node.ClusterId;
+import com.yahoo.vespa.hosted.provision.node.Report;
 
 import java.util.Comparator;
 import java.util.EnumSet;
@@ -214,6 +215,18 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> {
                              n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index))));
     }
 
+    // TODO(mpolden): Remove these when HostEncrypter is removed
+    /** Returns the subset of nodes which are being encrypted */
+    public NodeList encrypting() {
+        return matching(node -> node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() &&
+                                node.reports().getReport(Report.DISK_ENCRYPTED_ID).isEmpty());
+    }
+
+    /** Returns the subset of nodes which are encrypted */
+    public NodeList encrypted() {
+        return matching(node -> node.reports().getReport(Report.DISK_ENCRYPTED_ID).isPresent());
+    }
+
     /** Returns the parent node of the given child node */
     public Optional<Node> parentOf(Node child) {
         return child.parentHostname()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
new file mode 100644
index 00000000000..9174222b89b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
@@ -0,0 +1,100 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.IntFlag;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.ClusterId;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * This maintainer triggers encryption of hosts that have unencrypted disk.
+ *
+ * A host to be encrypted is retired and marked as want-to-encrypt by storing a report.
+ *
+ * This uses the same host selection criteria as {@link com.yahoo.vespa.hosted.provision.os.RebuildingOsUpgrader}.
+ *
+ * @author mpolden
+ */
+// TODO(mpolden): This can be removed once all hosts are encrypted
+public class HostEncrypter extends NodeRepositoryMaintainer {
+
+    private static final Logger LOG = Logger.getLogger(HostEncrypter.class.getName());
+
+    private final IntFlag maxEncryptingHosts;
+
+    public HostEncrypter(NodeRepository nodeRepository, Duration interval, Metric metric) {
+        super(nodeRepository, interval, metric);
+        this.maxEncryptingHosts = Flags.MAX_ENCRYPTING_HOSTS.bindTo(nodeRepository.flagSource());
+    }
+
+    @Override
+    protected boolean maintain() {
+        Instant now = nodeRepository().clock().instant();
+        NodeList allNodes = nodeRepository().nodes().list();
+        for (var nodeType : NodeType.values()) {
+            if (!nodeType.isHost()) continue;
+            unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now));
+        }
+        return true;
+    }
+
+    /** Returns unencrypted hosts of given type that can be encrypted */
+    private List<Node> unencryptedHosts(NodeList allNodes, NodeType hostType) {
+        if (!hostType.isHost()) throw new IllegalArgumentException("Expected host type, got " + hostType);
+        NodeList hostsOfTargetType =  allNodes.nodeType(hostType);
+        int hostLimit = hostLimit(hostsOfTargetType);
+
+        // Find stateful clusters with retiring nodes
+        NodeList activeNodes = allNodes.state(Node.State.active);
+        Set<ClusterId> retiringClusters = new HashSet<>(activeNodes.nodeType(hostType.childNodeType())
+                                                                   .retiring()
+                                                                   .statefulClusters());
+
+        // Encrypt hosts not containing stateful clusters with retiring nodes, up to limit
+        List<Node> hostsToEncrypt = new ArrayList<>(hostLimit);
+        NodeList candidates = hostsOfTargetType.state(Node.State.active)
+                                               .not().encrypted()
+                                               .not().encrypting()
+                                               // Require an OS version supporting encryption
+                                               .matching(node -> node.status().osVersion().current()
+                                                                     .orElse(Version.emptyVersion).getMajor() >= 8);
+
+        for (Node host : candidates) {
+            if (hostsToEncrypt.size() == hostLimit) break;
+            Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters();
+            boolean canEncrypt = Collections.disjoint(retiringClusters, clustersOnHost);
+            if (canEncrypt) {
+                hostsToEncrypt.add(host);
+                retiringClusters.addAll(clustersOnHost);
+            }
+        }
+        return Collections.unmodifiableList(hostsToEncrypt);
+
+    }
+
+    /** Returns the number of hosts that can encrypt concurrently */
+    private int hostLimit(NodeList hosts) {
+        return Math.max(0, maxEncryptingHosts.value() - hosts.encrypting().size());
+    }
+
+    private void encrypt(Node host, Instant now) {
+        LOG.info("Retiring and encrypting " + host);
+        nodeRepository().nodes().encrypt(host.hostname(), Agent.HostEncrypter, now);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index e9f107dd5f7..44ee8b5a8b3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -66,6 +66,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new AutoscalingMaintainer(nodeRepository, deployer, metric, defaults.autoscalingInterval));
         maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric));
         maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer));
+        maintainers.add(new HostEncrypter(nodeRepository, defaults.hostEncrypterInterval, metric));
 
         provisionServiceProvider.getLoadBalancerService(nodeRepository)
                                 .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric))
@@ -117,6 +118,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration autoscalingInterval;
         private final Duration scalingSuggestionsInterval;
         private final Duration switchRebalancerInterval;
+        private final Duration hostEncrypterInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -143,6 +145,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             scalingSuggestionsInterval = Duration.ofMinutes(31);
             spareCapacityMaintenanceInterval = Duration.ofMinutes(30);
             switchRebalancerInterval = Duration.ofHours(1);
+            hostEncrypterInterval = Duration.ofMinutes(5);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
             inactiveConfigServerExpiry = Duration.ofMinutes(5);
             inactiveControllerExpiry = Duration.ofMinutes(5);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
index 8a943400b37..d1c3f00ddca 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
@@ -26,5 +26,6 @@ public enum Agent {
     RebuildingOsUpgrader,
     SpareCapacityMaintainer,
     SwitchRebalancer,
+    HostEncrypter,
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
index 2ef68566a87..04f7111af2a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
@@ -611,29 +611,39 @@ public class Nodes {
 
     /** Retire and deprovision given host and all of its children */
     public List<Node> deprovision(String hostname, Agent agent, Instant instant) {
-        return decomission(hostname, DecommisionOperation.deprovision, agent, instant);
+        return decommission(hostname, DecommissionOperation.deprovision, agent, instant);
     }
 
     /** Retire and rebuild given host and all of its children */
     public List<Node> rebuild(String hostname, Agent agent, Instant instant) {
-        return decomission(hostname, DecommisionOperation.rebuild, agent, instant);
+        return decommission(hostname, DecommissionOperation.rebuild, agent, instant);
     }
 
-    private List<Node> decomission(String hostname, DecommisionOperation op, Agent agent, Instant instant) {
+    /** Retire and encrypt given host and all of its children */
+    public List<Node> encrypt(String hostname, Agent agent, Instant instant) {
+        return decommission(hostname, DecommissionOperation.encrypt, agent, instant);
+    }
+
+    private List<Node> decommission(String hostname, DecommissionOperation op, Agent agent, Instant instant) {
         Optional<NodeMutex> nodeMutex = lockAndGet(hostname);
         if (nodeMutex.isEmpty()) return List.of();
         Node host = nodeMutex.get().node();
         if (!host.type().isHost()) throw new IllegalArgumentException("Cannot " + op + " non-host " + host);
         List<Node> result;
-        boolean wantToDeprovision = op == DecommisionOperation.deprovision;
-        boolean wantToRebuild = op == DecommisionOperation.rebuild;
+        boolean wantToDeprovision = op == DecommissionOperation.deprovision;
+        boolean wantToRebuild = op == DecommissionOperation.rebuild;
         try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) {
             // This takes allocationLock to prevent any further allocation of nodes on this host
             host = lock.node();
             result = performOn(list(allocationLock).childrenOf(host),
                                (node, nodeLock) -> write(node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant),
                                                          nodeLock));
-            result.add(write(host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), lock));
+            Node newHost = host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant);
+            if (op == DecommissionOperation.encrypt) {
+                Report report = Report.basicReport(Report.WANT_TO_ENCRYPT_ID, Report.Type.UNSPECIFIED, instant, "Host should be encrypted");
+                newHost = newHost.with(newHost.reports().withReport(report));
+            }
+            result.add(write(newHost, lock));
         }
         return result;
     }
@@ -806,10 +816,11 @@ public class Nodes {
                retirementRequestedByOperator;
     }
 
-    /** The different ways a host can be decomissioned */
-    private enum DecommisionOperation {
+    /** The different ways a host can be decommissioned */
+    private enum DecommissionOperation {
         deprovision,
         rebuild,
+        encrypt,
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
index 37141d8f25b..4eb76828131 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
@@ -27,6 +27,10 @@ public class Report {
     /** The description of the report. */
     public static final String DESCRIPTION_FIELD = "description";
 
+    /** Known report IDs */
+    public static final String WANT_TO_ENCRYPT_ID = "wantToEncrypt";
+    public static final String DISK_ENCRYPTED_ID = "diskEncrypted";
+
     private final String reportId;
     private final Type type;
     private final Instant createdTime;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 97b9393bdd4..d83f21e5fec 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -481,6 +481,7 @@ public class NodeSerializer {
             case "RebuildingOsUpgrader" : return Agent.RebuildingOsUpgrader;
             case "SpareCapacityMaintainer": return Agent.SpareCapacityMaintainer;
             case "SwitchRebalancer": return Agent.SwitchRebalancer;
+            case "HostEncrypter": return Agent.HostEncrypter;
         }
         throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'");
     }
@@ -502,6 +503,7 @@ public class NodeSerializer {
             case RebuildingOsUpgrader: return "RebuildingOsUpgrader";
             case SpareCapacityMaintainer: return "SpareCapacityMaintainer";
             case SwitchRebalancer: return "SwitchRebalancer";
+            case HostEncrypter: return "HostEncrypter";
         }
         throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java
new file mode 100644
index 00000000000..b3c78aa4627
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java
@@ -0,0 +1,153 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.test.MockMetric;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Report;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author mpolden
+ */
+public class HostEncrypterTest {
+
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
+
+    @Test
+    public void no_hosts_encrypted_with_default_flag_value() {
+        provisionHosts(1);
+        HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric());
+        encrypter.maintain();
+        assertEquals(0, tester.nodeRepository().nodes().list().encrypting().size());
+    }
+
+    @Test
+    public void encrypt_hosts() {
+        tester.flagSource().withIntFlag(Flags.MAX_ENCRYPTING_HOSTS.id(), 3);
+        Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host);
+        HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric());
+
+        // Provision hosts and deploy applications
+        int hostCount = 5;
+        ApplicationId app1 = ApplicationId.from("t1", "a1", "i1");
+        ApplicationId app2 = ApplicationId.from("t2", "a2", "i2");
+        provisionHosts(hostCount);
+        deployApplication(app1);
+        deployApplication(app2);
+
+        // Encrypts 1 host per stateful cluster and 1 empty host
+        encrypter.maintain();
+        NodeList allNodes = tester.nodeRepository().nodes().list();
+        List<Node> hostsEncrypting = allNodes.nodeType(NodeType.host)
+                                             .encrypting()
+                                             .sortedBy(Comparator.comparing(Node::hostname))
+                                             .asList();
+        List<Optional<ApplicationId>> owners = List.of(Optional.of(app1), Optional.of(app2), Optional.empty());
+        assertEquals(owners.size(), hostsEncrypting.size());
+        for (int i = 0; i < hostsEncrypting.size(); i++) {
+            Optional<ApplicationId> owner = owners.get(i);
+            List<Node> retiringChildren = allNodes.childrenOf(hostsEncrypting.get(i)).retiring().asList();
+            assertEquals(owner.isPresent() ? 1 : 0, retiringChildren.size());
+            assertEquals("Encrypting host of " + owner.map(ApplicationId::toString)
+                                                      .orElse("no application"),
+                         owner,
+                         retiringChildren.stream()
+                                         .findFirst()
+                                         .flatMap(Node::allocation)
+                                         .map(Allocation::owner));
+        }
+
+        // Replace any retired nodes
+        replaceNodes(app1);
+        replaceNodes(app2);
+
+        // Complete encryption
+        completeEncryptionOf(hostsEncrypting);
+        assertEquals(3, hosts.get().encrypted().size());
+
+        // Both applications have moved their nodes to the remaining unencrypted hosts
+        allNodes = tester.nodeRepository().nodes().list();
+        NodeList unencryptedHosts = allNodes.nodeType(NodeType.host).not().encrypted();
+        assertEquals(2, unencryptedHosts.size());
+        for (var host : unencryptedHosts) {
+            assertEquals(1, allNodes.childrenOf(host).owner(app1).size());
+            assertEquals(1, allNodes.childrenOf(host).owner(app2).size());
+        }
+
+        // Since both applications now occupy all remaining hosts, we can only upgrade 1 at a time
+        for (int i = 0; i < unencryptedHosts.size(); i++) {
+            encrypter.maintain();
+            hostsEncrypting = hosts.get().encrypting().asList();
+            assertEquals(1, hostsEncrypting.size());
+            replaceNodes(app1);
+            replaceNodes(app2);
+            completeEncryptionOf(hostsEncrypting);
+        }
+
+        // Resuming encryption has no effect as all hosts are now encrypted
+        encrypter.maintain();
+        NodeList allHosts = hosts.get();
+        assertEquals(0, allHosts.encrypting().size());
+        assertEquals(allHosts.size(), allHosts.encrypted().size());
+    }
+
+    private void provisionHosts(int hostCount) {
+        List<Node> provisionedHosts = tester.makeReadyNodes(hostCount, new NodeResources(48, 128, 2000, 10), NodeType.host, 10);
+        // Set OS version supporting encryption
+        tester.patchNodes(provisionedHosts, (host) -> host.with(host.status().withOsVersion(host.status().osVersion().withCurrent(Optional.of(Version.fromString("8.0"))))));
+        tester.activateTenantHosts();
+    }
+
+    private void completeEncryptionOf(List<Node> nodes) {
+        Instant now = tester.clock().instant();
+        tester.patchNodes(nodes, (node) -> {
+            if (node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isEmpty()) throw new IllegalArgumentException(node + " is not requested to encrypt");
+            return node.with(node.reports().withReport(Report.basicReport(Report.DISK_ENCRYPTED_ID,
+                                                                          Report.Type.UNSPECIFIED,
+                                                                          now,
+                                                                          "Host is encrypted")))
+                       .withWantToRetire(false, Agent.system, now);
+        });
+    }
+
+    private void deployApplication(ApplicationId application) {
+        ClusterSpec contentSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7").build();
+        List<HostSpec> hostSpecs = tester.prepare(application, contentSpec, 2, 1, new NodeResources(4, 8, 100, 0.3));
+        tester.activate(application, hostSpecs);
+    }
+
+    private void replaceNodes(ApplicationId application) {
+        // Deploy to retire nodes
+        deployApplication(application);
+        List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList();
+        assertFalse("At least one node is retired", retired.isEmpty());
+        tester.nodeRepository().nodes().setRemovable(application, retired);
+
+        // Redeploy to deactivate removable nodes and allocate new ones
+        deployApplication(application);
+        tester.nodeRepository().nodes().list(Node.State.inactive).owner(application)
+              .forEach(node -> tester.nodeRepository().nodes().removeRecursively(node, true));
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
index b31c597e2b0..2dcf2d0b838 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
@@ -10,6 +10,9 @@
       "name": "FailedExpirer"
     },
     {
+      "name": "HostEncrypter"
+    },
+    {
       "name": "InactiveExpirer"
     },
     {