diff options
author | Martin Polden <mpolden@mpolden.no> | 2021-05-27 14:44:50 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-27 14:44:50 +0200 |
commit | 2cfd9a73adee4d2b6d3d446f19e61167e31bd043 (patch) | |
tree | 134d5a2df75e574d9efacd5e9df11861b730a9f4 | |
parent | 469c7245b62be6f5ee114827cd0c69971e54609f (diff) | |
parent | 4b8f28b9978df1393c5b7b4ffc0912dcd7fabf7c (diff) |
Merge pull request #18009 from vespa-engine/mpolden/host-encrypter
Implement HostEncrypter
12 files changed, 322 insertions, 24 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java index 5460981a1b0..624e4c61662 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -50,6 +50,7 @@ public class NodeHistory { RebuildingOsUpgrader, SpareCapacityMaintainer, SwitchRebalancer, + HostEncrypter, } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5693159dfd5..aeec579183d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -259,6 +259,12 @@ public class Flags { "Takes effect on next deployment through controller", APPLICATION_ID); + public static final UnboundIntFlag MAX_ENCRYPTING_HOSTS = defineIntFlag( + "max-encrypting-hosts", 0, + List.of("mpolden", "hakonhall"), "2021-05-27", "2021-10-01", + "The maximum number of hosts allowed to encrypt their disk concurrently", + "Takes effect on next run of HostEncrypter, but any currently encrypting hosts will not be cancelled when reducing the limit"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 4e9468925b6..594f2b6e619 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.node.ClusterId; +import com.yahoo.vespa.hosted.provision.node.Report; import java.util.Comparator; import java.util.EnumSet; @@ -213,6 +215,18 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index)))); } + // TODO(mpolden): Remove these when HostEncrypter is removed + /** Returns the subset of nodes which are being encrypted */ + public NodeList encrypting() { + return matching(node -> node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isPresent() && + node.reports().getReport(Report.DISK_ENCRYPTED_ID).isEmpty()); + } + + /** Returns the subset of nodes which are encrypted */ + public NodeList encrypted() { + return matching(node -> node.reports().getReport(Report.DISK_ENCRYPTED_ID).isPresent()); + } + /** Returns the parent node of the given child node */ public Optional<Node> parentOf(Node child) { return child.parentHostname() @@ -225,6 +239,16 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { return stream().map(Node::hostname).collect(Collectors.toUnmodifiableSet()); } + /** Returns the stateful clusters on nodes in this */ + public Set<ClusterId> statefulClusters() { + return stream().filter(node -> node.allocation().isPresent() && + node.allocation().get().membership().cluster().isStateful()) + .map(node -> new ClusterId(node.allocation().get().owner(), + node.allocation().get().membership().cluster().id())) + .collect(Collectors.toUnmodifiableSet()); + + } + /** * Returns the cluster spec of the nodes in this, without any group designation * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java new file mode 100644 index 00000000000..50f77a7df7b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java @@ -0,0 +1,103 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.IntFlag; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.ClusterId; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +/** + * This maintainer triggers encryption of hosts that have unencrypted disk. + * + * A host to be encrypted is retired and marked as want-to-encrypt by storing a report. + * + * This uses the same host selection criteria as {@link com.yahoo.vespa.hosted.provision.os.RebuildingOsUpgrader}. + * + * @author mpolden + */ +// TODO(mpolden): This can be removed once all hosts are encrypted +public class HostEncrypter extends NodeRepositoryMaintainer { + + private static final Logger LOG = Logger.getLogger(HostEncrypter.class.getName()); + + private final IntFlag maxEncryptingHosts; + + public HostEncrypter(NodeRepository nodeRepository, Duration interval, Metric metric) { + super(nodeRepository, interval, metric); + this.maxEncryptingHosts = Flags.MAX_ENCRYPTING_HOSTS.bindTo(nodeRepository.flagSource()); + } + + @Override + protected boolean maintain() { + Instant now = nodeRepository().clock().instant(); + NodeList allNodes = nodeRepository().nodes().list(); + for (var nodeType : NodeType.values()) { + if (!nodeType.isHost()) continue; + unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now)); + } + return true; + } + + /** Returns unencrypted hosts of given type that can be encrypted */ + private List<Node> unencryptedHosts(NodeList allNodes, NodeType hostType) { + if (!hostType.isHost()) throw new IllegalArgumentException("Expected host type, got " + hostType); + NodeList hostsOfTargetType = allNodes.nodeType(hostType); + int hostLimit = hostLimit(hostsOfTargetType, hostType); + + // Find stateful clusters with retiring nodes + NodeList activeNodes = allNodes.state(Node.State.active); + Set<ClusterId> retiringClusters = new HashSet<>(activeNodes.nodeType(hostType.childNodeType()) + .retiring() + .statefulClusters()); + + // Encrypt hosts not containing stateful clusters with retiring nodes, up to limit + List<Node> hostsToEncrypt = new ArrayList<>(hostLimit); + NodeList candidates = hostsOfTargetType.state(Node.State.active) + .not().encrypted() + .not().encrypting() + // Require an OS version supporting encryption + .matching(node -> node.status().osVersion().current() + .orElse(Version.emptyVersion).getMajor() >= 8); + + for (Node host : candidates) { + if (hostsToEncrypt.size() == hostLimit) break; + Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters(); + boolean canEncrypt = Collections.disjoint(retiringClusters, clustersOnHost); + if (canEncrypt) { + hostsToEncrypt.add(host); + retiringClusters.addAll(clustersOnHost); + } + } + return Collections.unmodifiableList(hostsToEncrypt); + + } + + /** Returns the number of hosts that can encrypt concurrently */ + private int hostLimit(NodeList hosts, NodeType hostType) { + if (hosts.stream().anyMatch(host -> host.type() != hostType)) throw new IllegalArgumentException("All hosts must be a " + hostType); + if (maxEncryptingHosts.value() < 1) return 0; // 0 or negative value effectively stops encryption of all hosts + int limit = hostType == NodeType.host ? maxEncryptingHosts.value() : 1; + return Math.max(0, limit - hosts.encrypting().size()); + } + + private void encrypt(Node host, Instant now) { + LOG.info("Retiring and encrypting " + host); + nodeRepository().nodes().encrypt(host.hostname(), Agent.HostEncrypter, now); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index e9f107dd5f7..44ee8b5a8b3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -66,6 +66,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new AutoscalingMaintainer(nodeRepository, deployer, metric, defaults.autoscalingInterval)); maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric)); maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer)); + maintainers.add(new HostEncrypter(nodeRepository, defaults.hostEncrypterInterval, metric)); provisionServiceProvider.getLoadBalancerService(nodeRepository) .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)) @@ -117,6 +118,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration autoscalingInterval; private final Duration scalingSuggestionsInterval; private final Duration switchRebalancerInterval; + private final Duration hostEncrypterInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -143,6 +145,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { scalingSuggestionsInterval = Duration.ofMinutes(31); spareCapacityMaintenanceInterval = Duration.ofMinutes(30); switchRebalancerInterval = Duration.ofHours(1); + hostEncrypterInterval = Duration.ofMinutes(5); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; inactiveConfigServerExpiry = Duration.ofMinutes(5); inactiveControllerExpiry = Duration.ofMinutes(5); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index 8a943400b37..d1c3f00ddca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -26,5 +26,6 @@ public enum Agent { RebuildingOsUpgrader, SpareCapacityMaintainer, SwitchRebalancer, + HostEncrypter, } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 2ef68566a87..59cee0a469e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -611,29 +611,39 @@ public class Nodes { /** Retire and deprovision given host and all of its children */ public List<Node> deprovision(String hostname, Agent agent, Instant instant) { - return decomission(hostname, DecommisionOperation.deprovision, agent, instant); + return decommission(hostname, DecommissionOperation.deprovision, agent, instant); } /** Retire and rebuild given host and all of its children */ public List<Node> rebuild(String hostname, Agent agent, Instant instant) { - return decomission(hostname, DecommisionOperation.rebuild, agent, instant); + return decommission(hostname, DecommissionOperation.rebuild, agent, instant); } - private List<Node> decomission(String hostname, DecommisionOperation op, Agent agent, Instant instant) { + /** Retire and encrypt given host and all of its children */ + public List<Node> encrypt(String hostname, Agent agent, Instant instant) { + return decommission(hostname, DecommissionOperation.encrypt, agent, instant); + } + + private List<Node> decommission(String hostname, DecommissionOperation op, Agent agent, Instant instant) { Optional<NodeMutex> nodeMutex = lockAndGet(hostname); if (nodeMutex.isEmpty()) return List.of(); Node host = nodeMutex.get().node(); if (!host.type().isHost()) throw new IllegalArgumentException("Cannot " + op + " non-host " + host); List<Node> result; - boolean wantToDeprovision = op == DecommisionOperation.deprovision; - boolean wantToRebuild = op == DecommisionOperation.rebuild; + boolean wantToDeprovision = op == DecommissionOperation.deprovision; + boolean wantToRebuild = op == DecommissionOperation.rebuild; try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) { // This takes allocationLock to prevent any further allocation of nodes on this host host = lock.node(); result = performOn(list(allocationLock).childrenOf(host), (node, nodeLock) -> write(node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), nodeLock)); - result.add(write(host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), lock)); + Node newHost = host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant); + if (op == DecommissionOperation.encrypt) { + Report report = Report.basicReport(Report.WANT_TO_ENCRYPT_ID, Report.Type.UNSPECIFIED, instant, ""); + newHost = newHost.with(newHost.reports().withReport(report)); + } + result.add(write(newHost, lock)); } return result; } @@ -806,10 +816,11 @@ public class Nodes { retirementRequestedByOperator; } - /** The different ways a host can be decomissioned */ - private enum DecommisionOperation { + /** The different ways a host can be decommissioned */ + private enum DecommissionOperation { deprovision, rebuild, + encrypt, } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index 37141d8f25b..4eb76828131 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -27,6 +27,10 @@ public class Report { /** The description of the report. */ public static final String DESCRIPTION_FIELD = "description"; + /** Known report IDs */ + public static final String WANT_TO_ENCRYPT_ID = "wantToEncrypt"; + public static final String DISK_ENCRYPTED_ID = "diskEncrypted"; + private final String reportId; private final Type type; private final Instant createdTime; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index 20536b9dd9f..e373f5edbfb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.ClusterId; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; @@ -69,8 +68,8 @@ public class RebuildingOsUpgrader implements OsUpgrader { // Find stateful clusters with retiring nodes NodeList activeNodes = allNodes.state(Node.State.active); - Set<ClusterId> retiringClusters = statefulClustersOf(activeNodes.nodeType(target.nodeType().childNodeType()) - .retiring()); + Set<ClusterId> retiringClusters = new HashSet<>(activeNodes.nodeType(target.nodeType().childNodeType()) + .retiring().statefulClusters()); // Rebuild hosts not containing stateful clusters with retiring nodes, up to rebuild limit List<Node> hostsToRebuild = new ArrayList<>(rebuildLimit); @@ -80,7 +79,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; - Set<ClusterId> clustersOnHost = statefulClustersOf(activeNodes.childrenOf(host)); + Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters(); boolean canRebuild = Collections.disjoint(retiringClusters, clustersOnHost); if (canRebuild) { hostsToRebuild.add(host); @@ -98,18 +97,6 @@ public class RebuildingOsUpgrader implements OsUpgrader { nodeRepository.nodes().upgradeOs(NodeListFilter.from(host), Optional.of(target)); } - private static Set<ClusterId> statefulClustersOf(NodeList nodes) { - Set<ClusterId> clusters = new HashSet<>(); - for (Node node : nodes) { - if (node.type().isHost()) illegal("All nodes must be children, got host " + node); - if (node.allocation().isEmpty()) continue; - Allocation allocation = node.allocation().get(); - if (!allocation.membership().cluster().isStateful()) continue; - clusters.add(new ClusterId(allocation.owner(), allocation.membership().cluster().id())); - } - return clusters; - } - private static void illegal(String msg) { throw new IllegalArgumentException(msg); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 97b9393bdd4..d83f21e5fec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -481,6 +481,7 @@ public class NodeSerializer { case "RebuildingOsUpgrader" : return Agent.RebuildingOsUpgrader; case "SpareCapacityMaintainer": return Agent.SpareCapacityMaintainer; case "SwitchRebalancer": return Agent.SwitchRebalancer; + case "HostEncrypter": return Agent.HostEncrypter; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } @@ -502,6 +503,7 @@ public class NodeSerializer { case RebuildingOsUpgrader: return "RebuildingOsUpgrader"; case SpareCapacityMaintainer: return "SpareCapacityMaintainer"; case SwitchRebalancer: return "SwitchRebalancer"; + case HostEncrypter: return "HostEncrypter"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java new file mode 100644 index 00000000000..b3c78aa4627 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypterTest.java @@ -0,0 +1,153 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Report; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author mpolden + */ +public class HostEncrypterTest { + + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + + @Test + public void no_hosts_encrypted_with_default_flag_value() { + provisionHosts(1); + HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric()); + encrypter.maintain(); + assertEquals(0, tester.nodeRepository().nodes().list().encrypting().size()); + } + + @Test + public void encrypt_hosts() { + tester.flagSource().withIntFlag(Flags.MAX_ENCRYPTING_HOSTS.id(), 3); + Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); + HostEncrypter encrypter = new HostEncrypter(tester.nodeRepository(), Duration.ofDays(1), new MockMetric()); + + // Provision hosts and deploy applications + int hostCount = 5; + ApplicationId app1 = ApplicationId.from("t1", "a1", "i1"); + ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); + provisionHosts(hostCount); + deployApplication(app1); + deployApplication(app2); + + // Encrypts 1 host per stateful cluster and 1 empty host + encrypter.maintain(); + NodeList allNodes = tester.nodeRepository().nodes().list(); + List<Node> hostsEncrypting = allNodes.nodeType(NodeType.host) + .encrypting() + .sortedBy(Comparator.comparing(Node::hostname)) + .asList(); + List<Optional<ApplicationId>> owners = List.of(Optional.of(app1), Optional.of(app2), Optional.empty()); + assertEquals(owners.size(), hostsEncrypting.size()); + for (int i = 0; i < hostsEncrypting.size(); i++) { + Optional<ApplicationId> owner = owners.get(i); + List<Node> retiringChildren = allNodes.childrenOf(hostsEncrypting.get(i)).retiring().asList(); + assertEquals(owner.isPresent() ? 1 : 0, retiringChildren.size()); + assertEquals("Encrypting host of " + owner.map(ApplicationId::toString) + .orElse("no application"), + owner, + retiringChildren.stream() + .findFirst() + .flatMap(Node::allocation) + .map(Allocation::owner)); + } + + // Replace any retired nodes + replaceNodes(app1); + replaceNodes(app2); + + // Complete encryption + completeEncryptionOf(hostsEncrypting); + assertEquals(3, hosts.get().encrypted().size()); + + // Both applications have moved their nodes to the remaining unencrypted hosts + allNodes = tester.nodeRepository().nodes().list(); + NodeList unencryptedHosts = allNodes.nodeType(NodeType.host).not().encrypted(); + assertEquals(2, unencryptedHosts.size()); + for (var host : unencryptedHosts) { + assertEquals(1, allNodes.childrenOf(host).owner(app1).size()); + assertEquals(1, allNodes.childrenOf(host).owner(app2).size()); + } + + // Since both applications now occupy all remaining hosts, we can only upgrade 1 at a time + for (int i = 0; i < unencryptedHosts.size(); i++) { + encrypter.maintain(); + hostsEncrypting = hosts.get().encrypting().asList(); + assertEquals(1, hostsEncrypting.size()); + replaceNodes(app1); + replaceNodes(app2); + completeEncryptionOf(hostsEncrypting); + } + + // Resuming encryption has no effect as all hosts are now encrypted + encrypter.maintain(); + NodeList allHosts = hosts.get(); + assertEquals(0, allHosts.encrypting().size()); + assertEquals(allHosts.size(), allHosts.encrypted().size()); + } + + private void provisionHosts(int hostCount) { + List<Node> provisionedHosts = tester.makeReadyNodes(hostCount, new NodeResources(48, 128, 2000, 10), NodeType.host, 10); + // Set OS version supporting encryption + tester.patchNodes(provisionedHosts, (host) -> host.with(host.status().withOsVersion(host.status().osVersion().withCurrent(Optional.of(Version.fromString("8.0")))))); + tester.activateTenantHosts(); + } + + private void completeEncryptionOf(List<Node> nodes) { + Instant now = tester.clock().instant(); + tester.patchNodes(nodes, (node) -> { + if (node.reports().getReport(Report.WANT_TO_ENCRYPT_ID).isEmpty()) throw new IllegalArgumentException(node + " is not requested to encrypt"); + return node.with(node.reports().withReport(Report.basicReport(Report.DISK_ENCRYPTED_ID, + Report.Type.UNSPECIFIED, + now, + "Host is encrypted"))) + .withWantToRetire(false, Agent.system, now); + }); + } + + private void deployApplication(ApplicationId application) { + ClusterSpec contentSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7").build(); + List<HostSpec> hostSpecs = tester.prepare(application, contentSpec, 2, 1, new NodeResources(4, 8, 100, 0.3)); + tester.activate(application, hostSpecs); + } + + private void replaceNodes(ApplicationId application) { + // Deploy to retire nodes + deployApplication(application); + List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList(); + assertFalse("At least one node is retired", retired.isEmpty()); + tester.nodeRepository().nodes().setRemovable(application, retired); + + // Redeploy to deactivate removable nodes and allocate new ones + deployApplication(application); + tester.nodeRepository().nodes().list(Node.State.inactive).owner(application) + .forEach(node -> tester.nodeRepository().nodes().removeRecursively(node, true)); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json index b31c597e2b0..2dcf2d0b838 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json @@ -10,6 +10,9 @@ "name": "FailedExpirer" }, { + "name": "HostEncrypter" + }, + { "name": "InactiveExpirer" }, { |