Merge pull request #19576 from vespa-engine/mpolden/remove-host-encrypter

Remove HostEncrypter

5 files changed, 1 insertions, 167 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index dd3c26fd653..9a110427223 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -215,18 +215,6 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> {
                              n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index))));
     }
 
-    // TODO(mpolden): Remove these when HostEncrypter is removed
-    /** Returns the subset of nodes which are being encrypted */
-    public NodeList encrypting() {
-        return matching(node -> node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() &&
-                                node.reports().getReport(Report.DISK_ENCRYPTED_ID).isEmpty());
-    }
-
-    /** Returns the subset of nodes which are encrypted */
-    public NodeList encrypted() {
-        return matching(node -> node.reports().getReport(Report.DISK_ENCRYPTED_ID).isPresent());
-    }
-
     /** Returns the parent node of the given child node */
     public Optional<Node> parentOf(Node child) {
         return child.parentHostname()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
deleted file mode 100644
index 017e30c2d46..00000000000
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.maintenance;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.IntFlag;
-import com.yahoo.vespa.flags.ListFlag;
-import com.yahoo.vespa.hosted.provision.Node;
-import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.node.Agent;
-import com.yahoo.vespa.hosted.provision.node.ClusterId;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * This maintainer triggers encryption of hosts that have unencrypted disk.
- *
- * A host to be encrypted is retired and marked as want-to-encrypt by storing a report.
- *
- * This uses the same host selection criteria as {@link com.yahoo.vespa.hosted.provision.os.RebuildingOsUpgrader}.
- *
- * @author mpolden
- */
-// TODO(mpolden): This can be removed once all hosts are encrypted
-public class HostEncrypter extends NodeRepositoryMaintainer {
-
-    private static final Logger LOG = Logger.getLogger(HostEncrypter.class.getName());
-
-    private final IntFlag maxEncryptingHosts;
-    private final ListFlag<String> deferApplicationEncryption;
-
-    public HostEncrypter(NodeRepository nodeRepository, Duration interval, Metric metric) {
-        super(nodeRepository, interval, metric);
-        this.maxEncryptingHosts = Flags.MAX_ENCRYPTING_HOSTS.bindTo(nodeRepository.flagSource());
-        this.deferApplicationEncryption = Flags.DEFER_APPLICATION_ENCRYPTION.bindTo(nodeRepository.flagSource());
-    }
-
-    @Override
-    protected double maintain() {
-        Instant now = nodeRepository().clock().instant();
-        NodeList allNodes = nodeRepository().nodes().list();
-        for (var nodeType : NodeType.values()) {
-            if (!nodeType.isHost()) continue;
-            if (upgradingVespa(allNodes, nodeType)) continue;
-            unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now));
-        }
-        return 1.0;
-    }
-
-    /** Returns whether any node of given type is currently upgrading its Vespa version */
-    private boolean upgradingVespa(NodeList allNodes, NodeType hostType) {
-        return allNodes.state(Node.State.ready, Node.State.active)
-                       .nodeType(hostType)
-                       .changingVersion()
-                       .size() > 0;
-    }
-
-    /** Returns unencrypted hosts of given type that can be encrypted */
-    private List<Node> unencryptedHosts(NodeList allNodes, NodeType hostType) {
-        if (!hostType.isHost()) throw new IllegalArgumentException("Expected host type, got " + hostType);
-        NodeList hostsOfTargetType =  allNodes.nodeType(hostType);
-        int hostLimit = hostLimit(hostsOfTargetType, hostType);
-
-        // Find stateful clusters with retiring nodes
-        NodeList activeNodes = allNodes.state(Node.State.active);
-        Set<ClusterId> retiringClusters = new HashSet<>(activeNodes.nodeType(hostType.childNodeType())
-                                                                   .retiring()
-                                                                   .statefulClusters());
-
-        // Encrypt hosts not containing stateful clusters with retiring nodes, up to limit
-        List<Node> hostsToEncrypt = new ArrayList<>(hostLimit);
-
-        Set<ApplicationId> deferredApplications = deferApplicationEncryption.value().stream()
-                                                                            .map(ApplicationId::fromSerializedForm)
-                                                                            .collect(Collectors.toSet());
-        NodeList candidates = hostsOfTargetType.state(Node.State.active)
-                                               .not().encrypted()
-                                               .not().encrypting()
-                                               .matching(host -> encryptHost(host, allNodes, deferredApplications))
-                                               // Require an OS version supporting encryption
-                                               .matching(node -> node.status().osVersion().current()
-                                                                     .orElse(Version.emptyVersion)
-                                                                     .getMajor() >= 8);
-
-        for (Node host : candidates) {
-            if (hostsToEncrypt.size() == hostLimit) break;
-            Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters();
-            boolean canEncrypt = Collections.disjoint(retiringClusters, clustersOnHost);
-            if (canEncrypt) {
-                hostsToEncrypt.add(host);
-                retiringClusters.addAll(clustersOnHost);
-            }
-        }
-        return Collections.unmodifiableList(hostsToEncrypt);
-
-    }
-
-    /** Returns the number of hosts that can encrypt concurrently */
-    private int hostLimit(NodeList hosts, NodeType hostType) {
-        if (hosts.stream().anyMatch(host -> host.type() != hostType)) throw new IllegalArgumentException("All hosts must be a " + hostType);
-        if (maxEncryptingHosts.value() < 1) return 0; // 0 or negative value effectively stops encryption of all hosts
-        int limit = hostType == NodeType.host ? maxEncryptingHosts.value() : 1;
-        return Math.max(0, limit - hosts.encrypting().size());
-    }
-
-    private boolean encryptHost(Node host, NodeList allNodes, Set<ApplicationId> deferredApplications) {
-        Set<ApplicationId> applicationsOnHost = allNodes.childrenOf(host).stream()
-                                                        .filter(node -> node.allocation().isPresent())
-                                                        .map(node -> node.allocation().get().owner())
-                                                        .collect(Collectors.toSet());
-        return Collections.disjoint(applicationsOnHost, deferredApplications);
-    }
-
-    private void encrypt(Node host, Instant now) {
-        LOG.info("Retiring and encrypting " + host);
-        nodeRepository().nodes().encrypt(host.hostname(), Agent.HostEncrypter, now);
-    }
-
-}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 873d8ceb7b9..2313dfbde0b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -66,7 +66,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new AutoscalingMaintainer(nodeRepository, deployer, metric, defaults.autoscalingInterval));
         maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric));
         maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer));
-        maintainers.add(new HostEncrypter(nodeRepository, defaults.hostEncrypterInterval, metric));
 
         provisionServiceProvider.getLoadBalancerService(nodeRepository)
                                 .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
index 9ba7f5684ee..480fd72967e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
@@ -316,8 +316,7 @@ public class Nodes {
 
     public Node deallocate(Node node, Agent agent, String reason, NestedTransaction transaction) {
         if (parkOnDeallocationOf(node, agent)) {
-            boolean keepAllocation = node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent();
-            return park(node.hostname(), keepAllocation, agent, reason, transaction);
+            return park(node.hostname(), false, agent, reason, transaction);
         } else {
             return db.writeTo(Node.State.dirty, List.of(node), agent, Optional.of(reason), transaction).get(0);
         }
@@ -641,11 +640,6 @@ public class Nodes {
         return decommission(hostname, DecommissionOperation.rebuild, agent, instant);
     }
 
-    /** Retire and encrypt given host and all of its children */
-    public List<Node> encrypt(String hostname, Agent agent, Instant instant) {
-        return decommission(hostname, DecommissionOperation.encrypt, agent, instant);
-    }
-
     private List<Node> decommission(String hostname, DecommissionOperation op, Agent agent, Instant instant) {
         Optional<NodeMutex> nodeMutex = lockAndGet(hostname);
         if (nodeMutex.isEmpty()) return List.of();
@@ -654,23 +648,14 @@ public class Nodes {
         List<Node> result;
         boolean wantToDeprovision = op == DecommissionOperation.deprovision;
         boolean wantToRebuild = op == DecommissionOperation.rebuild;
-        Optional<Report> wantToEncryptReport = op == DecommissionOperation.encrypt
-                ? Optional.of(Report.basicReport(Report.WANT_TO_ENCRYPT_ID, Report.Type.UNSPECIFIED, instant, ""))
-                : Optional.empty();
         try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) {
             // This takes allocationLock to prevent any further allocation of nodes on this host
             host = lock.node();
             result = performOn(list(allocationLock).childrenOf(host), (node, nodeLock) -> {
                 Node newNode = node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant);
-                if (wantToEncryptReport.isPresent()) {
-                    newNode = newNode.with(newNode.reports().withReport(wantToEncryptReport.get()));
-                }
                 return write(newNode, nodeLock);
             });
             Node newHost = host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant);
-            if (wantToEncryptReport.isPresent()) {
-                newHost = newHost.with(newHost.reports().withReport(wantToEncryptReport.get()));
-            }
             result.add(write(newHost, lock));
         }
         return result;
@@ -841,7 +826,6 @@ public class Nodes {
                                                     .orElse(false);
         return node.status().wantToDeprovision() ||
                node.status().wantToRebuild() ||
-               node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() ||
                retirementRequestedByOperator;
     }
 
@@ -849,7 +833,6 @@ public class Nodes {
     private enum DecommissionOperation {
         deprovision,
         rebuild,
-        encrypt,
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
index f5c5b9f2857..37141d8f25b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
@@ -27,11 +27,6 @@ public class Report {
     /** The description of the report. */
     public static final String DESCRIPTION_FIELD = "description";
 
-    /** Known report IDs */
-    // TODO(mpolden): Remove together with HostEncrypter
-    public static final String WANT_TO_ENCRYPT_ID = "wantToEncrypt";
-    public static final String DISK_ENCRYPTED_ID = "diskEncrypted";
-
     private final String reportId;
     private final Type type;
     private final Instant createdTime;