Remove HostEncrypter

5 files changed, 0 insertions, 345 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index dd3c26fd653..9a110427223 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -215,18 +215,6 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> {
                              n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index))));
     }
 
-    // TODO(mpolden): Remove these when HostEncrypter is removed
-    /** Returns the subset of nodes which are being encrypted */
-    public NodeList encrypting() {
-        return matching(node -> node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() &&
-                                node.reports().getReport(Report.DISK_ENCRYPTED_ID).isEmpty());
-    }
-
-    /** Returns the subset of nodes which are encrypted */
-    public NodeList encrypted() {
-        return matching(node -> node.reports().getReport(Report.DISK_ENCRYPTED_ID).isPresent());
-    }
-
     /** Returns the parent node of the given child node */
     public Optional<Node> parentOf(Node child) {
         return child.parentHostname()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
deleted file mode 100644
index 017e30c2d46..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.IntFlag;
-import com.yahoo.vespa.flags.ListFlag;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.node.Agent;
-import com.yahoo.vespa.hosted.provision.node.ClusterId;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * This maintainer triggers encryption of hosts that have unencrypted disk.
- *
- * A host to be encrypted is retired and marked as want-to-encrypt by storing a report.
- *
- * This uses the same host selection criteria as {@link com.yahoo.vespa.hosted.provision.os.RebuildingOsUpgrader}.
- *
- * @author mpolden
- */
-// TODO(mpolden): This can be removed once all hosts are encrypted
-public class HostEncrypter extends NodeRepositoryMaintainer {
-
-    private static final Logger LOG = Logger.getLogger(HostEncrypter.class.getName());
-
-    private final IntFlag maxEncryptingHosts;
-    private final ListFlag<String> deferApplicationEncryption;
-
-    public HostEncrypter(NodeRepository nodeRepository, Duration interval, Metric metric) {
-        super(nodeRepository, interval, metric);
-        this.maxEncryptingHosts = Flags.MAX_ENCRYPTING_HOSTS.bindTo(nodeRepository.flagSource());
-        this.deferApplicationEncryption = Flags.DEFER_APPLICATION_ENCRYPTION.bindTo(nodeRepository.flagSource());
-    }
-
-    @Override
-    protected double maintain() {
-        Instant now = nodeRepository().clock().instant();
-        NodeList allNodes = nodeRepository().nodes().list();
-        for (var nodeType : NodeType.values()) {
-            if (!nodeType.isHost()) continue;
-            if (upgradingVespa(allNodes, nodeType)) continue;
-            unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now));
-        }
-        return 1.0;
-    }
-
-    /** Returns whether any node of given type is currently upgrading its Vespa version */
-    private boolean upgradingVespa(NodeList allNodes, NodeType hostType) {
-        return allNodes.state(Node.State.ready, Node.State.active)
-                       .nodeType(hostType)
-                       .changingVersion()
-                       .size() > 0;
-    }
-
-    /** Returns unencrypted hosts of given type that can be encrypted */
-    private List<Node> unencryptedHosts(NodeList allNodes, NodeType hostType) {
-        if (!hostType.isHost()) throw new IllegalArgumentException("Expected host type, got " + hostType);
-        NodeList hostsOfTargetType =  allNodes.nodeType(hostType);
-        int hostLimit = hostLimit(hostsOfTargetType, hostType);
-
-        // Find stateful clusters with retiring nodes
-        NodeList activeNodes = allNodes.state(Node.State.active);
-        Set<ClusterId> retiringClusters = new HashSet<>(activeNodes.nodeType(hostType.childNodeType())
-                                                                   .retiring()
-                                                                   .statefulClusters());
-
-        // Encrypt hosts not containing stateful clusters with retiring nodes, up to limit
-        List<Node> hostsToEncrypt = new ArrayList<>(hostLimit);
-
-        Set<ApplicationId> deferredApplications = deferApplicationEncryption.value().stream()
-                                                                            .map(ApplicationId::fromSerializedForm)
-                                                                            .collect(Collectors.toSet());
-        NodeList candidates = hostsOfTargetType.state(Node.State.active)
-                                               .not().encrypted()
-                                               .not().encrypting()
-                                               .matching(host -> encryptHost(host, allNodes, deferredApplications))
-                                               // Require an OS version supporting encryption
-                                               .matching(node -> node.status().osVersion().current()
-                                                                     .orElse(Version.emptyVersion)
-                                                                     .getMajor() >= 8);
-
-        for (Node host : candidates) {
-            if (hostsToEncrypt.size() == hostLimit) break;
-            Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters();
-            boolean canEncrypt = Collections.disjoint(retiringClusters, clustersOnHost);
-            if (canEncrypt) {
-                hostsToEncrypt.add(host);
-                retiringClusters.addAll(clustersOnHost);
-            }
-        }
-        return Collections.unmodifiableList(hostsToEncrypt);
-
-    }
-
-    /** Returns the number of hosts that can encrypt concurrently */
-    private int hostLimit(NodeList hosts, NodeType hostType) {
-        if (hosts.stream().anyMatch(host -> host.type() != hostType)) throw new IllegalArgumentException("All hosts must be a " + hostType);
-        if (maxEncryptingHosts.value() < 1) return 0; // 0 or negative value effectively stops encryption of all hosts
-        int limit = hostType == NodeType.host ? maxEncryptingHosts.value() : 1;
-        return Math.max(0, limit - hosts.encrypting().size());
-    }
-
-    private boolean encryptHost(Node host, NodeList allNodes, Set<ApplicationId> deferredApplications) {
-        Set<ApplicationId> applicationsOnHost = allNodes.childrenOf(host).stream()
-                                                        .filter(node -> node.allocation().isPresent())
-                                                        .map(node -> node.allocation().get().owner())
-                                                        .collect(Collectors.toSet());
-        return Collections.disjoint(applicationsOnHost, deferredApplications);
-    }
-
-    private void encrypt(Node host, Instant now) {
-        LOG.info("Retiring and encrypting " + host);
-        nodeRepository().nodes().encrypt(host.hostname(), Agent.HostEncrypter, now);
-    }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 873d8ceb7b9..2313dfbde0b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -66,7 +66,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new AutoscalingMaintainer(nodeRepository, deployer, metric, defaults.autoscalingInterval));
         maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric));
         maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer));
-        maintainers.add(new HostEncrypter(nodeRepository, defaults.hostEncrypterInterval, metric));
 
         provisionServiceProvider.getLoadBalancerService(nodeRepository)
                                 .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric))
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java
deleted file mode 100644
index 2149badda70..00000000000
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.NodeResources;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.jdisc.test.MockMetric;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.node.Agent;
-import com.yahoo.vespa.hosted.provision.node.Allocation;
-import com.yahoo.vespa.hosted.provision.node.Report;
-import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
-import org.junit.Test;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author mpolden
- */
-public class HostEncrypterTest {
-
-    private final ApplicationId infraApplication = ApplicationId.from("hosted-vespa", "infra", "default");
-    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
-    private final HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric());
-
-    @Test
-    public void no_hosts_encrypted_with_default_flag_value() {
-        provisionHosts(1);
-        encrypter.maintain();
-        assertEquals(0, tester.nodeRepository().nodes().list().encrypting().size());
-    }
-
-    @Test
-    public void deferred_hosts_are_not_encrypted() {
-        int hostCount = 4;
-        int proxyHostCount = 1;
-        ApplicationId app1 = ApplicationId.from("t1", "a1", "i1");
-        ApplicationId app2 = ApplicationId.from("t2", "a2", "i2");
-        provisionHosts(hostCount);
-        deployApplication(app1);
-        deployApplication(app2);
-
-        ApplicationId proxyHostApp = ApplicationId.from("hosted-vespa", "proxy-host", "default");
-        List<Node> proxyHosts = tester.makeReadyNodes(proxyHostCount, "default", NodeType.proxyhost, 10);
-        tester.patchNodes(proxyHosts, (host) -> host.with(host.status().withOsVersion(host.status().osVersion().withCurrent(Optional.of(Version.fromString("8.0"))))));
-        tester.prepareAndActivateInfraApplication(proxyHostApp, NodeType.proxyhost);
-
-        tester.flagSource()
-              .withIntFlag(Flags.MAX_ENCRYPTING_HOSTS.id(), hostCount + proxyHostCount)
-              .withListFlag(Flags.DEFER_APPLICATION_ENCRYPTION.id(), List.of(app2.serializedForm()), String.class);
-        encrypter.maintain();
-        NodeList allNodes = tester.nodeRepository().nodes().list();
-        NodeList encryptingHosts = allNodes.encrypting().parents();
-
-        assertEquals(1, encryptingHosts.nodeType(NodeType.proxyhost).size());
-
-        assertEquals(1, encryptingHosts.nodeType(NodeType.host).size());
-        assertEquals("Host of included application is encrypted", Set.of(app1),
-                     allNodes.childrenOf(encryptingHosts.nodeType(NodeType.host).asList().get(0)).stream()
-                             .map(node -> node.allocation().get().owner())
-                             .collect(Collectors.toSet()));
-    }
-
-    @Test
-    public void encrypt_hosts() {
-        tester.flagSource().withIntFlag(Flags.MAX_ENCRYPTING_HOSTS.id(), 3);
-        Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host);
-
-        // Provision hosts and deploy applications
-        int hostCount = 5;
-        ApplicationId app1 = ApplicationId.from("t1", "a1", "i1");
-        ApplicationId app2 = ApplicationId.from("t2", "a2", "i2");
-        provisionHosts(hostCount);
-        deployApplication(app1);
-        deployApplication(app2);
-
-        // Encrypts 1 host per stateful cluster and 1 empty host
-        encrypter.maintain();
-        NodeList allNodes = tester.nodeRepository().nodes().list();
-        List<Node> hostsEncrypting = allNodes.nodeType(NodeType.host)
-                                             .encrypting()
-                                             .sortedBy(Comparator.comparing(Node::hostname))
-                                             .asList();
-        List<Optional<ApplicationId>> owners = List.of(Optional.of(app1), Optional.of(app2), Optional.empty());
-        assertEquals(owners.size(), hostsEncrypting.size());
-        for (int i = 0; i < hostsEncrypting.size(); i++) {
-            Optional<ApplicationId> owner = owners.get(i);
-            List<Node> retiringChildren = allNodes.childrenOf(hostsEncrypting.get(i)).retiring().encrypting().asList();
-            assertEquals(owner.isPresent() ? 1 : 0, retiringChildren.size());
-            assertEquals("Encrypting host of " + owner.map(ApplicationId::toString)
-                                                      .orElse("no application"),
-                         owner,
-                         retiringChildren.stream()
-                                         .findFirst()
-                                         .flatMap(Node::allocation)
-                                         .map(Allocation::owner));
-        }
-
-        // Replace any retired nodes
-        replaceNodes(app1);
-        replaceNodes(app2);
-
-        // Complete encryption
-        completeEncryptionOf(hostsEncrypting);
-        assertEquals(3, hosts.get().encrypted().size());
-
-        // Both applications have moved their nodes to the remaining unencrypted hosts
-        allNodes = tester.nodeRepository().nodes().list();
-        NodeList unencryptedHosts = allNodes.nodeType(NodeType.host).not().encrypted();
-        assertEquals(2, unencryptedHosts.size());
-        for (var host : unencryptedHosts) {
-            assertEquals(1, allNodes.childrenOf(host).owner(app1).size());
-            assertEquals(1, allNodes.childrenOf(host).owner(app2).size());
-        }
-
-        // Since both applications now occupy all remaining hosts, we can only upgrade 1 at a time
-        for (int i = 0; i < unencryptedHosts.size(); i++) {
-            encrypter.maintain();
-            hostsEncrypting = hosts.get().encrypting().asList();
-            assertEquals(1, hostsEncrypting.size());
-            replaceNodes(app1);
-            replaceNodes(app2);
-            completeEncryptionOf(hostsEncrypting);
-        }
-
-        // Resuming encryption has no effect as all hosts are now encrypted
-        encrypter.maintain();
-        NodeList allHosts = hosts.get();
-        assertEquals(0, allHosts.encrypting().size());
-        assertEquals(allHosts.size(), allHosts.encrypted().size());
-    }
-
-    private void provisionHosts(int hostCount) {
-        List<Node> provisionedHosts = tester.makeReadyNodes(hostCount, new NodeResources(48, 128, 2000, 10), NodeType.host, 10);
-        // Set OS version supporting encryption
-        tester.patchNodes(provisionedHosts, (host) -> host.with(host.status().withOsVersion(host.status().osVersion().withCurrent(Optional.of(Version.fromString("8.0"))))));
-        tester.prepareAndActivateInfraApplication(infraApplication, NodeType.host);
-    }
-
-    private void completeEncryptionOf(List<Node> nodes) {
-        Instant now = tester.clock().instant();
-        // Redeploy to park retired hosts
-        replaceNodes(infraApplication, (application) -> tester.prepareAndActivateInfraApplication(application, NodeType.host));
-        List<Node> patchedNodes = tester.patchNodes(nodes, (node) -> {
-            assertSame(Node.State.parked, node.state());
-            assertTrue(node + " wants to encrypt", node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent());
-            return node.with(node.reports().withReport(Report.basicReport(Report.DISK_ENCRYPTED_ID,
-                                                                          Report.Type.UNSPECIFIED,
-                                                                          now,
-                                                                          "Host is encrypted")));
-        });
-        patchedNodes = tester.nodeRepository().nodes().deallocate(patchedNodes, Agent.system, getClass().getSimpleName());
-        tester.nodeRepository().nodes().setReady(patchedNodes, Agent.system, getClass().getSimpleName());
-        tester.activateTenantHosts();
-    }
-
-    private void deployApplication(ApplicationId application) {
-        ClusterSpec contentSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7").build();
-        List<HostSpec> hostSpecs = tester.prepare(application, contentSpec, 2, 1, new NodeResources(4, 8, 100, 0.3));
-        tester.activate(application, hostSpecs);
-    }
-
-    private void replaceNodes(ApplicationId application) {
-        replaceNodes(application, this::deployApplication);
-    }
-
-    private void replaceNodes(ApplicationId application, Consumer<ApplicationId> deployer) {
-        // Deploy to retire nodes
-        deployer.accept(application);
-        List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList();
-        assertFalse("At least one node is retired", retired.isEmpty());
-        tester.nodeRepository().nodes().setRemovable(application, retired);
-
-        // Redeploy to deactivate removable nodes and allocate new ones
-        deployer.accept(application);
-        tester.nodeRepository().nodes().list(Node.State.inactive).owner(application)
-              .forEach(node -> tester.nodeRepository().nodes().removeRecursively(node, true));
-    }
-
-}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
index 72224ef3cba..4c8c5d80018 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json
@@ -13,9 +13,6 @@
       "name": "FailedExpirer"
     },
     {
-      "name": "HostEncrypter"
-    },
-    {
       "name": "InactiveExpirer"
     },
     {