diff options
author | Martin Polden <mpolden@mpolden.no> | 2020-05-28 15:04:26 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-28 15:04:26 +0200 |
commit | 4552075772789e0db6d4ab0e21157b393274432b (patch) | |
tree | 1c33f2c1ff2897d0e3dadbc2bfa96b2385bdcaf8 /node-repository/src | |
parent | 13d1a3491b1daac7a6058300e83014200e30386c (diff) | |
parent | 8903332a7a6ce57c7777b2f6976c1da781d8b52e (diff) |
Merge pull request #13401 from vespa-engine/mpolden/provision-exact-capacity
Support provisioning exact capacity
Diffstat (limited to 'node-repository/src')
22 files changed, 395 insertions, 232 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 490fed681f9..cc485374340 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -102,6 +102,7 @@ public class NodeRepository extends AbstractComponent { private final DockerImages dockerImages; private final JobControl jobControl; private final Applications applications; + private final boolean canProvisionHostsWhenRequired; /** * Creates a node repository from a zookeeper provider. @@ -119,7 +120,8 @@ public class NodeRepository extends AbstractComponent { Clock.systemUTC(), zone, new DnsNameResolver(), - DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), + provisionServiceProvider.getHostProvisioner().isPresent()); } /** @@ -133,7 +135,8 @@ public class NodeRepository extends AbstractComponent { Zone zone, NameResolver nameResolver, DockerImage dockerImage, - boolean useCuratorClientCache) { + boolean useCuratorClientCache, + boolean canProvisionHostsWhenRequired) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; @@ -146,6 +149,7 @@ public class NodeRepository extends AbstractComponent { this.dockerImages = new DockerImages(db, dockerImage); this.jobControl = new JobControl(db); this.applications = new Applications(db); + this.canProvisionHostsWhenRequired = canProvisionHostsWhenRequired; // read and write all nodes to make sure they are stored in the latest version of the serialized format for (State state : State.values()) @@ -795,12 +799,17 @@ public class NodeRepository extends AbstractComponent { if (host.status().wantToRetire()) return false; if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false; - if ( zone.getCloud().dynamicProvisioning()) + if ( canProvisionHostsWhenRequired()) return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state()); else return host.state() == State.active; } + /** Returns whether this has the ability to conjure hosts when required */ + public boolean canProvisionHostsWhenRequired() { + return canProvisionHostsWhenRequired; + } + /** Returns the time keeper of this system */ public Clock clock() { return clock; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 297285addc6..44bfed90106 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -10,7 +10,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.Collection; +import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -36,6 +36,7 @@ import java.util.stream.IntStream; /** * @author freva + * @author mpolden */ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { @@ -43,7 +44,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; - private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; + private final ListFlag<HostCapacity> targetCapacityFlag; DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, @@ -51,27 +52,25 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; - this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.targetCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource); } @Override protected void maintain() { - if (! nodeRepository().zone().getCloud().dynamicProvisioning()) return; - try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); - - updateProvisioningNodes(nodes, lock); + resumeProvisioning(nodes, lock); convergeToCapacity(nodes); } } - void updateProvisioningNodes(NodeList nodes, Mutex lock) { + /** Resume provisioning of already provisioned hosts and their children */ + private void resumeProvisioning(NodeList nodes, Mutex lock) { Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() - .filter(node -> node.parentHostname().isPresent()) - .collect(Collectors.groupingBy( - node -> node.parentHostname().get(), - Collectors.toSet())); + .filter(node -> node.parentHostname().isPresent()) + .collect(Collectors.groupingBy( + node -> node.parentHostname().get(), + Collectors.toSet())); nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); @@ -80,10 +79,10 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { nodeRepository().write(updatedNodes, lock); } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Failed to provision " + host.hostname() + " with " + children.size() + " children: " + - Exceptions.toMessageString(e)); + Exceptions.toMessageString(e)); } catch (FatalProvisioningException e) { log.log(Level.SEVERE, "Failed to provision " + host.hostname() + " with " + children.size() + - " children, failing out the host recursively", e); + " children, failing out the host recursively", e); // Fail out as operator to force a quick redeployment nodeRepository().failRecursively( host.hostname(), Agent.operator, "Failed by HostProvisioner due to provisioning failure"); @@ -93,31 +92,49 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { }); } - void convergeToCapacity(NodeList nodes) { - Collection<Node> removableHosts = getRemovableHosts(nodes); - List<NodeResources> preProvisionCapacity = preprovisionCapacityFlag.value().stream() - .flatMap(cap -> { - NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(), cap.getDiskGb(), 1); - return IntStream.range(0, cap.getCount()).mapToObj(i -> resources); - }) - .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed()) - .collect(Collectors.toList()); + /** Converge zone to wanted capacity */ + private void convergeToCapacity(NodeList nodes) { + List<NodeResources> capacity = targetCapacity(); + List<Node> excessHosts = provision(capacity, nodes); + excessHosts.forEach(host -> { + try { + hostProvisioner.deprovision(host); + nodeRepository().removeRecursively(host, true); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); + } + }); + } - for (Iterator<NodeResources> it = preProvisionCapacity.iterator(); it.hasNext() && !removableHosts.isEmpty();) { + /** + * Provision the nodes necessary to satisfy given capacity. + * + * @return Excess hosts that can safely be deprovisioned, if any. + */ + private List<Node> provision(List<NodeResources> capacity, NodeList nodes) { + List<Node> existingHosts = availableHostsOf(nodes); + if (nodeRepository().zone().getCloud().dynamicProvisioning()) { + existingHosts = removableHostsOf(existingHosts, nodes); + } else if (capacity.isEmpty()) { + return List.of(); + } + List<Node> excessHosts = new ArrayList<>(existingHosts); + for (Iterator<NodeResources> it = capacity.iterator(); it.hasNext() && !excessHosts.isEmpty(); ) { NodeResources resources = it.next(); - removableHosts.stream() - .filter(nodeRepository()::canAllocateTenantNodeTo) - .filter(host -> nodeRepository().resourcesCalculator().advertisedResourcesOf(host.flavor()).satisfies(resources)) - .min(Comparator.comparingInt(n -> n.flavor().cost())) - .ifPresent(host -> { - removableHosts.remove(host); - it.remove(); - }); + excessHosts.stream() + .filter(nodeRepository()::canAllocateTenantNodeTo) + .filter(host -> nodeRepository().resourcesCalculator() + .advertisedResourcesOf(host.flavor()) + .satisfies(resources)) + .min(Comparator.comparingInt(n -> n.flavor().cost())) + .ifPresent(host -> { + excessHosts.remove(host); + it.remove(); + }); } - - // pre-provisioning is best effort, do one host at a time - preProvisionCapacity.forEach(resources -> { + // Pre-provisioning is best effort, do one host at a time + capacity.forEach(resources -> { try { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Node> hosts = hostProvisioner.provisionHosts(nodeRepository().database().getProvisionIndexes(1), @@ -127,35 +144,47 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); nodeRepository().addNodes(hosts, Agent.DynamicProvisioningMaintainer); } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) { - log.log(Level.WARNING, "Failed to pre-provision " + resources + ":" + e.getMessage()); + log.log(Level.WARNING, "Failed to pre-provision " + resources + ": " + e.getMessage()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to pre-provision " + resources + ", will retry in " + interval(), e); } }); + return removableHostsOf(excessHosts, nodes); + } - // Finally, deprovision excess hosts. - removableHosts.forEach(host -> { - try { - hostProvisioner.deprovision(host); - nodeRepository().removeRecursively(host, true); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); - } - }); + + /** Reads node resources declared by target capacity flag */ + private List<NodeResources> targetCapacity() { + return targetCapacityFlag.value().stream() + .flatMap(cap -> { + NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(), + cap.getDiskGb(), 1); + return IntStream.range(0, cap.getCount()).mapToObj(i -> resources); + }) + .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed()) + .collect(Collectors.toList()); } - private static Collection<Node> getRemovableHosts(NodeList nodes) { - Map<String, Node> hostsByHostname = nodes.nodeType(NodeType.host) - .asList().stream() - .filter(host -> host.state() != Node.State.parked || host.status().wantToDeprovision()) - .collect(Collectors.toMap(Node::hostname, Function.identity())); + /** Returns hosts that are considered available, i.e. not parked or flagged for deprovisioning */ + private static List<Node> availableHostsOf(NodeList nodes) { + return nodes.nodeType(NodeType.host) + .matching(host -> host.state() != Node.State.parked || host.status().wantToDeprovision()) + .asList(); + } + + /** Returns the subset of given hosts that have no containers and are thus removable */ + private static List<Node> removableHostsOf(List<Node> hosts, NodeList allNodes) { + Map<String, Node> hostsByHostname = hosts.stream() + .collect(Collectors.toMap(Node::hostname, + Function.identity())); - nodes.asList().stream() + allNodes.asList().stream() .filter(node -> node.allocation().isPresent()) .flatMap(node -> node.parentHostname().stream()) .distinct() .forEach(hostsByHostname::remove); - return hostsByHostname.values(); + return List.copyOf(hostsByHostname.values()); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index d388fb5a967..8a82c74dd17 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -90,7 +90,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); - rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); + rebalancer = new Rebalancer(deployer, nodeRepository, metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index f044eaf97f6..12990447eee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity; -import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import java.time.Clock; import java.time.Duration; @@ -22,29 +21,27 @@ import java.util.Optional; * @author bratseth */ public class Rebalancer extends NodeRepositoryMaintainer { + static final Duration waitTimeAfterPreviousDeployment = Duration.ofMinutes(10); private final Deployer deployer; - private final Optional<HostProvisioner> hostProvisioner; private final Metric metric; private final Clock clock; public Rebalancer(Deployer deployer, NodeRepository nodeRepository, - Optional<HostProvisioner> hostProvisioner, Metric metric, Clock clock, Duration interval) { super(nodeRepository, interval); this.deployer = deployer; - this.hostProvisioner = hostProvisioner; this.metric = metric; this.clock = clock; } @Override protected void maintain() { - if (hostProvisioner.isPresent()) return; // All nodes will be allocated on new hosts, so rebalancing makes no sense + if (nodeRepository().canProvisionHostsWhenRequired()) return; // All nodes will be allocated on new hosts, so rebalancing makes no sense if (nodeRepository().zone().environment().isTest()) return; // Test zones have short lived deployments, no need to rebalance // Work with an unlocked snapshot as this can take a long time and full consistency is not needed diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index 6b8d1b13975..e04c1aa208d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -52,6 +52,12 @@ public class FlavorConfigBuilder { flavorConfigBuilder.addFlavor(flavorName, 2., 40., 40., 0.5, Flavor.Type.DOCKER_CONTAINER); else if (flavorName.equals("host")) flavorConfigBuilder.addFlavor(flavorName, 7., 100., 120., 5, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host2")) + flavorConfigBuilder.addFlavor(flavorName, 16, 24, 100, 1, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host3")) + flavorConfigBuilder.addFlavor(flavorName, 24, 64, 100, 1, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host4")) + flavorConfigBuilder.addFlavor(flavorName, 48, 128, 1000, 1, Flavor.Type.BARE_METAL); else if (flavorName.equals("devhost")) flavorConfigBuilder.addFlavor(flavorName, 4., 80., 100, 10, Flavor.Type.BARE_METAL); else diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index a1d8ffb03d3..caecf8edf2f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -11,13 +11,12 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -31,14 +30,14 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; - private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; + private final ListFlag<HostCapacity> preprovisionCapacityFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; - this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.preprovisionCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource); } /** @@ -59,7 +58,7 @@ public class GroupPreparer { // active config model which is changed on activate public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, MutableInteger highestIndex, int spareCount, int wantedGroups) { - boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && nodeRepository.zone().getCloud().dynamicProvisioning(); + boolean dynamicProvisioningEnabled = nodeRepository.canProvisionHostsWhenRequired() && nodeRepository.zone().getCloud().dynamicProvisioning(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); try (Mutex lock = nodeRepository.lock(application)) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index c79c6b45247..8e5cb9d088c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.time.Clock; import java.time.Instant; @@ -65,7 +64,7 @@ public class MockNodeRepository extends NodeRepository { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 88804576310..52c68cd74b2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvid import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import java.time.Clock; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -41,7 +40,7 @@ public class NodeRepositoryTester { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 3a87b5967fb..864cfad2f41 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -339,9 +339,10 @@ public class AutoscalingTest { flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); flavors.add(new Flavor("aws-small", new NodeResources(3, 80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); - AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.defaultCloud() - .withDynamicProvisioning(true) - .withAllowHostSharing(false), + AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.builder() + .dynamicProvisioning(true) + .allowHostSharing(false) + .build(), SystemName.main, Environment.prod, RegionName.from("us-east")), flavors); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index b1f6eaea502..a6b2f6b15ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -62,7 +62,7 @@ public class CapacityCheckerTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); } private void updateCapacityChecker() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 6eba517fde2..ba859655ab7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; @@ -14,13 +15,12 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; @@ -30,205 +30,239 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import org.hamcrest.BaseMatcher; -import org.hamcrest.Description; -import org.junit.Before; import org.junit.Test; import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.createNode; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyHostApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantHostApp; +import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.MockHostProvisioner.Behaviour; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.mockito.hamcrest.MockitoHamcrest.argThat; /** * @author freva + * @author mpolden */ public class DynamicProvisioningMaintainerTest { - private final HostProvisionerTester tester = new HostProvisionerTester(); - private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); - private static final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource() - .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); - private final DynamicProvisioningMaintainer maintainer = - new DynamicProvisioningMaintainer(tester.nodeRepository, - Duration.ofDays(1), - hostProvisioner, - flagSource); - @Test public void delegates_to_host_provisioner_and_writes_back_result() { - addNodes(); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.hostProvisioner.with(Behaviour.failDeprovisioning); // To avoid deleting excess nodes + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - - Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); - when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); + assertTrue("No IP addresses assigned", + Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); - Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); - when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenReturn(List.of(host4new, host41new)); + Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0"))); + Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0"))); + Node host41new = host41.with(host41.ipConfig().with(Set.of("::4:1", "::4:2"))); - maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); - verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); - verify(hostProvisioner).provision(eq(host3), eq(Set.of())); - verifyNoMoreInteractions(hostProvisioner); - - assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); - assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); - assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); + tester.maintainer.maintain(); + assertEquals(host3new, tester.nodeRepository.getNode("host3").get()); + assertEquals(host4new, tester.nodeRepository.getNode("host4").get()); + assertEquals(host41new, tester.nodeRepository.getNode("host4-1").get()); } @Test public void correctly_fails_if_irrecoverable_failure() { - Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()); - Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)); - - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenThrow(new FatalProvisioningException("Fatal")); - - maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); + var tester = new DynamicProvisioningTester(); + tester.hostProvisioner.with(Behaviour.failProvisioning); + Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned); + Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, DynamicProvisioningTester.tenantApp); + assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + tester.maintainer.maintain(); assertEquals(Set.of("host4", "host4-1"), - tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet())); + tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet())); } @Test public void finds_nodes_that_need_deprovisioning_without_pre_provisioning() { - addNodes(); + var tester = new DynamicProvisioningTester().addInitialNodes(); + assertTrue(tester.nodeRepository.getNode("host2").isPresent()); + assertTrue(tester.nodeRepository.getNode("host3").isPresent()); - maintainer.convergeToCapacity(tester.nodeRepository.list()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host3"))); - verifyNoMoreInteractions(hostProvisioner); + tester.maintainer.maintain(); assertTrue(tester.nodeRepository.getNode("host2").isEmpty()); assertTrue(tester.nodeRepository.getNode("host3").isEmpty()); } @Test public void does_not_deprovision_when_preprovisioning_enabled() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1)), PreprovisionCapacity.class); - addNodes(); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); // host2 because it is failed - verifyNoMoreInteractions(hostProvisioner); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(1, 3, 2, 1)), HostCapacity.class); + Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); + assertTrue(failedHost.isPresent()); + + tester.maintainer.maintain(); + assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); + assertEquals(1, tester.hostProvisioner.deprovisionedHosts); } @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), - List.of(new PreprovisionCapacity(2, 4, 8, 1), - new PreprovisionCapacity(2, 3, 2, 2)), - PreprovisionCapacity.class); - addNodes(); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); - assertTrue(tester.nodeRepository.getNode("host2").isEmpty()); - assertTrue(tester.nodeRepository.getNode("host3").isPresent()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); - verify(hostProvisioner, times(2)).provisionHosts(argThatLambda(list -> list.size() == 1), eq(new NodeResources(2, 3, 2, 1)), any(), any()); - verifyNoMoreInteractions(hostProvisioner); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), + List.of(new HostCapacity(24, 64, 100, 2), + new HostCapacity(16, 24, 100, 1)), + HostCapacity.class); + assertTrue(tester.nodeRepository.getNode("host2").isPresent()); + assertEquals(0 ,tester.hostProvisioner.provisionedHosts.size()); + + // failed host2 is removed + Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); + assertTrue(failedHost.isPresent()); + tester.maintainer.maintain(); + assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); + assertTrue("Host with matching resources is kept", tester.nodeRepository.getNode("host3").isPresent()); + + // Two more hosts are provisioned with expected resources + NodeResources resources = new NodeResources(24, 64, 100, 1); + assertEquals(2, tester.provisionedHostsMatching(resources)); } @Test public void does_not_remove_if_host_provisioner_failed() { - Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)); - doThrow(new RuntimeException()).when(hostProvisioner).deprovision(eq(host2)); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); + var tester = new DynamicProvisioningTester(); + Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, DynamicProvisioningTester.tenantApp); + tester.hostProvisioner.with(Behaviour.failDeprovisioning); - assertEquals(1, tester.nodeRepository.getNodes().size()); - verify(hostProvisioner).deprovision(eq(host2)); - verifyNoMoreInteractions(hostProvisioner); + tester.maintainer.maintain(); + assertTrue(tester.nodeRepository.getNode(host2.hostname()).isPresent()); } - @Before - public void setup() { - doAnswer(invocation -> { - Flavor flavor = invocation.getArgument(0, Flavor.class); - if ("default".equals(flavor.name())) return new NodeResources(2, 4, 8, 1); - return invocation.getArguments()[1]; - }).when(hostResourcesCalculator).advertisedResourcesOf(any()); + @Test + public void provision_exact_capacity() { + var tester = new DynamicProvisioningTester(Cloud.builder().dynamicProvisioning(false).build()); + NodeResources resources1 = new NodeResources(24, 64, 100, 1); + NodeResources resources2 = new NodeResources(16, 24, 100, 1); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), + new HostCapacity(resources2.vcpu(), resources2.memoryGb(), resources2.diskGb(), 2)), + HostCapacity.class); + tester.maintainer.maintain(); + + // Hosts are provisioned + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(2, tester.provisionedHostsMatching(resources2)); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + + // Target capacity is changed + NodeResources resources3 = new NodeResources(48, 128, 1000, 1); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), + new HostCapacity(resources3.vcpu(), resources3.memoryGb(), resources3.diskGb(), 1)), + HostCapacity.class); + + // Excess hosts are deprovisioned + tester.maintainer.maintain(); + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(0, tester.provisionedHostsMatching(resources2)); + assertEquals(1, tester.provisionedHostsMatching(resources3)); + assertEquals(2, tester.nodeRepository.getNodes(Node.State.deprovisioned).size()); + + // Activate hosts + tester.maintainer.maintain(); // Resume provisioning of new hosts + List<Node> provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList(); + tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName()); + tester.provisioningTester.deployZoneApp(); + + // Allocating nodes to a host does not result in provisioning of additional capacity + ApplicationId application = tester.provisioningTester.makeApplicationId(); + tester.provisioningTester.deploy(application, + Capacity.from(new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)))); + assertEquals(2, tester.nodeRepository.list().owner(application).size()); + tester.assertNodesUnchanged(); + + // Clearing flag does nothing + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(), HostCapacity.class); + tester.assertNodesUnchanged(); + + // Capacity reduction does not remove host with children + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1)), + HostCapacity.class); + tester.assertNodesUnchanged(); } - public void addNodes() { - List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), - createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), - createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()), + private static class DynamicProvisioningTester { - createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), - createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + private static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default"); + private static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default"); + private static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default"); + private static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default"); + private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default", "docker", "host2", "host3", "host4"); - createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + private final InMemoryFlagSource flagSource = new InMemoryFlagSource().withListFlag(Flags.TARGET_CAPACITY.id(), + List.of(), + HostCapacity.class); - createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), - createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + private final NodeRepository nodeRepository; + private final MockHostProvisioner hostProvisioner; + private final DynamicProvisioningMaintainer maintainer; + private final ProvisioningTester provisioningTester; - createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()), + public DynamicProvisioningTester() { + this(Cloud.builder().dynamicProvisioning(true).build()); + } - createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)), - createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp))) - .forEach(node -> tester.nodeRepository.database().addNodesInState(List.of(node), node.state())); - } + public DynamicProvisioningTester(Cloud cloud) { + MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup(); + this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver); + this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(), + Environment.defaultEnvironment(), + RegionName.defaultName())) + .flavors(flavors.getFlavors()) + .nameResolver(nameResolver) + .hostProvisioner(hostProvisioner) + .build(); + this.nodeRepository = provisioningTester.nodeRepository(); + this.maintainer = new DynamicProvisioningMaintainer(nodeRepository, + Duration.ofDays(1), + hostProvisioner, + flagSource); + } - @SuppressWarnings("unchecked") - private static <T> T argThatLambda(Predicate<T> predicate) { - return argThat(new BaseMatcher<T>() { - @Override public boolean matches(Object item) { return predicate.test((T) item); } - @Override public void describeTo(Description description) { } - }); - } + private DynamicProvisioningTester addInitialNodes() { + List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), + createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), + createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()), + createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)), + createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp))) + .forEach(node -> nodeRepository.database().addNodesInState(List.of(node), node.state())); + return this; + } - static class HostProvisionerTester { - private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "docker"); - static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default"); - static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default"); - static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default"); - static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default"); - - private final ManualClock clock = new ManualClock(); - private final Zone zone = new Zone(Cloud.defaultCloud().withDynamicProvisioning(true), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); - private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - hostResourcesCalculator, - new MockCurator(), - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-image"), true); - - Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { - Node node = createNode(hostname, parentHostname, nodeType, state, application); + private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state) { + return addNode(hostname, parentHostname, nodeType, state, null); + } + + private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, ApplicationId application) { + Node node = createNode(hostname, parentHostname, nodeType, state, Optional.ofNullable(application)); return nodeRepository.database().addNodesInState(List.of(node), node.state()).get(0); } - static Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { - Flavor flavor = nodeFlavors.getFlavor(parentHostname.isPresent() ? "docker" : "default").orElseThrow(); + private Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { + Flavor flavor = nodeRepository.flavors().getFlavor(parentHostname.isPresent() ? "docker" : "host2").orElseThrow(); Optional<Allocation> allocation = application .map(app -> new Allocation( app, @@ -238,7 +272,101 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); } + + private long provisionedHostsMatching(NodeResources resources) { + return hostProvisioner.provisionedHosts.stream() + .filter(host -> host.nodeResources().equals(resources)) + .count(); + } + + private void assertNodesUnchanged() { + List<Node> nodes = nodeRepository.getNodes(); + maintainer.maintain(); + assertEquals("Nodes are unchanged after maintenance run", nodes, nodeRepository.getNodes()); + } + + } + + static class MockHostProvisioner implements HostProvisioner { + + private final List<ProvisionedHost> provisionedHosts = new ArrayList<>(); + private final NodeFlavors flavors; + private final MockNameResolver nameResolver; + + private int deprovisionedHosts = 0; + private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); + + public MockHostProvisioner(NodeFlavors flavors, MockNameResolver nameResolver) { + this.flavors = flavors; + this.nameResolver = nameResolver; + } + + @Override + public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) { + Flavor hostFlavor = flavors.getFlavors().stream() + .filter(f -> !f.isDocker()) + .filter(f -> f.resources().compatibleWith(resources)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No host flavor found satisfying " + resources)); + List<ProvisionedHost> hosts = new ArrayList<>(); + for (int index : provisionIndexes) { + hosts.add(new ProvisionedHost("host" + index, + "hostname" + index, + hostFlavor, + "nodename" + index, + resources, + osVersion)); + } + provisionedHosts.addAll(hosts); + return hosts; + } + + @Override + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { + if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)"); + assertSame(Node.State.provisioned, host.state()); + List<Node> result = new ArrayList<>(); + result.add(withIpAssigned(host)); + for (var child : children) { + assertSame(Node.State.reserved, child.state()); + result.add(withIpAssigned(child)); + } + return result; + } + + @Override + public void deprovision(Node host) { + if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node"); + provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname())); + deprovisionedHosts++; + } + + private MockHostProvisioner with(Behaviour first, Behaviour... rest) { + this.behaviours = EnumSet.of(first, rest); + return this; + } + + private Node withIpAssigned(Node node) { + if (node.parentHostname().isPresent()) return node; + int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); + Set<String> addresses = Set.of("::" + hostIndex + ":0"); + nameResolver.addRecord(node.hostname(), addresses.iterator().next()); + Set<String> pool = new HashSet<>(); + for (int i = 1; i <= 2; i++) { + String ip = "::" + hostIndex + ":" + i; + pool.add(ip); + nameResolver.addRecord(node.hostname() + "-" + i, ip); + } + return node.with(node.ipConfig().with(addresses).with(IP.Pool.of(pool))); + } + + enum Behaviour { + failProvisioning, + failDeprovisioning, + } + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 4fb1c3bcab2..24a0020df4f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -258,7 +258,7 @@ public class FailedExpirerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), - true); + true, false); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 11df6146b06..754870e798e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -3,21 +3,17 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.time.Instant; @@ -44,7 +40,7 @@ public class MaintenanceTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); public MaintenanceTester() { curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index e1cb57c4eb1..9fc2f666d27 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -84,7 +84,7 @@ public class MetricsReporterTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node), Agent.system); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); @@ -150,7 +150,7 @@ public class MetricsReporterTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 0a95845898b..9cf03c6f33b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -82,7 +82,7 @@ public class NodeFailTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index e57d57d4c4c..b9c57e013c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -61,7 +61,7 @@ public class OperatorChangeApplicationMaintainerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, this.fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 8a2a69bb437..b6ffe4ebe26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -67,7 +67,7 @@ public class PeriodicApplicationMaintainerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 19dc3699dd9..05e5b4829e9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -150,7 +150,7 @@ public class RebalancerTest { cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), memoryApp, new MockDeployer.ApplicationContext(memoryApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources)))); deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); - rebalancer = new Rebalancer(deployer, tester.nodeRepository(), Optional.empty(), metric, tester.clock(), Duration.ofMinutes(1)); + rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, tester.clock(), Duration.ofMinutes(1)); tester.makeReadyNodes(3, "flat", NodeType.host, 8); tester.deployZoneApp(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 59514cb3c95..2c0dac10f9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -52,7 +52,7 @@ public class ReservationExpirerTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index addbf717811..29451d7b690 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -71,7 +71,7 @@ public class RetiredExpirerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 087f8f83f0c..ea0d78e3015 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -33,7 +33,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.config.provision.NodeResources.StorageType.remote; @@ -52,7 +51,7 @@ import static org.mockito.Mockito.verify; public class DynamicDockerProvisionTest { private static final Zone zone = new Zone( - Cloud.defaultCloud().withDynamicProvisioning(true).withAllowHostSharing(false), + Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(), SystemName.main, Environment.prod, RegionName.from("us-east")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 92e64e752a7..9a02bc17b4b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -94,6 +94,7 @@ public class ProvisioningTester { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); + ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.nodeRepository = new NodeRepository(nodeFlavors, resourcesCalculator, curator, @@ -101,9 +102,9 @@ public class ProvisioningTester { zone, nameResolver, DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, + provisionServiceProvider.getHostProvisioner().isPresent()); this.orchestrator = orchestrator; - ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new NullProvisionLogger(); |