Merge pull request #13401 from vespa-engine/mpolden/provision-exact-capacity

Support provisioning exact capacity

22 files changed, 395 insertions, 232 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 490fed681f9..cc485374340 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -102,6 +102,7 @@ public class NodeRepository extends AbstractComponent {
     private final DockerImages dockerImages;
     private final JobControl jobControl;
     private final Applications applications;
+    private final boolean canProvisionHostsWhenRequired;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -119,7 +120,8 @@ public class NodeRepository extends AbstractComponent {
              Clock.systemUTC(),
              zone,
              new DnsNameResolver(),
-             DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache());
+             DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(),
+             provisionServiceProvider.getHostProvisioner().isPresent());
     }
 
     /**
@@ -133,7 +135,8 @@ public class NodeRepository extends AbstractComponent {
                           Zone zone,
                           NameResolver nameResolver,
                           DockerImage dockerImage,
-                          boolean useCuratorClientCache) {
+                          boolean useCuratorClientCache,
+                          boolean canProvisionHostsWhenRequired) {
         this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache);
         this.zone = zone;
         this.clock = clock;
@@ -146,6 +149,7 @@ public class NodeRepository extends AbstractComponent {
         this.dockerImages = new DockerImages(db, dockerImage);
         this.jobControl = new JobControl(db);
         this.applications = new Applications(db);
+        this.canProvisionHostsWhenRequired = canProvisionHostsWhenRequired;
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (State state : State.values())
@@ -795,12 +799,17 @@ public class NodeRepository extends AbstractComponent {
         if (host.status().wantToRetire()) return false;
         if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false;
 
-        if ( zone.getCloud().dynamicProvisioning())
+        if ( canProvisionHostsWhenRequired())
             return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state());
         else
             return host.state() == State.active;
     }
 
+    /** Returns whether this has the ability to conjure hosts when required */
+    public boolean canProvisionHostsWhenRequired() {
+        return canProvisionHostsWhenRequired;
+    }
+
     /** Returns the time keeper of this system */
     public Clock clock() { return clock; }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 297285addc6..44bfed90106 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -10,7 +10,7 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.ListFlag;
-import com.yahoo.vespa.flags.custom.PreprovisionCapacity;
+import com.yahoo.vespa.flags.custom.HostCapacity;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
@@ -36,6 +36,7 @@ import java.util.stream.IntStream;
 
 /**
  * @author freva
+ * @author mpolden
  */
 public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
 
@@ -43,7 +44,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
     private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision");
 
     private final HostProvisioner hostProvisioner;
-    private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
+    private final ListFlag<HostCapacity> targetCapacityFlag;
 
     DynamicProvisioningMaintainer(NodeRepository nodeRepository,
                                   Duration interval,
@@ -51,27 +52,25 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                                   FlagSource flagSource) {
         super(nodeRepository, interval);
         this.hostProvisioner = hostProvisioner;
-        this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
+        this.targetCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource);
     }
 
     @Override
     protected void maintain() {
-        if (! nodeRepository().zone().getCloud().dynamicProvisioning()) return;
-
         try (Mutex lock = nodeRepository().lockUnallocated()) {
             NodeList nodes = nodeRepository().list();
-
-            updateProvisioningNodes(nodes, lock);
+            resumeProvisioning(nodes, lock);
             convergeToCapacity(nodes);
         }
     }
 
-    void updateProvisioningNodes(NodeList nodes, Mutex lock) {
+    /** Resume provisioning of already provisioned hosts and their children */
+    private void resumeProvisioning(NodeList nodes, Mutex lock) {
         Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream()
-                .filter(node -> node.parentHostname().isPresent())
-                .collect(Collectors.groupingBy(
-                        node -> node.parentHostname().get(),
-                        Collectors.toSet()));
+                                                                       .filter(node -> node.parentHostname().isPresent())
+                                                                       .collect(Collectors.groupingBy(
+                                                                               node -> node.parentHostname().get(),
+                                                                               Collectors.toSet()));
 
         nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> {
             Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of());
@@ -80,10 +79,10 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                 nodeRepository().write(updatedNodes, lock);
             } catch (IllegalArgumentException | IllegalStateException e) {
                 log.log(Level.INFO, "Failed to provision " + host.hostname() + " with " + children.size() + " children: " +
-                        Exceptions.toMessageString(e));
+                                    Exceptions.toMessageString(e));
             } catch (FatalProvisioningException e) {
                 log.log(Level.SEVERE, "Failed to provision " + host.hostname() + " with " + children.size()  +
-                        " children, failing out the host recursively", e);
+                                      " children, failing out the host recursively", e);
                 // Fail out as operator to force a quick redeployment
                 nodeRepository().failRecursively(
                         host.hostname(), Agent.operator, "Failed by HostProvisioner due to provisioning failure");
@@ -93,31 +92,49 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
         });
     }
 
-    void convergeToCapacity(NodeList nodes) {
-        Collection<Node> removableHosts = getRemovableHosts(nodes);
-        List<NodeResources> preProvisionCapacity = preprovisionCapacityFlag.value().stream()
-                .flatMap(cap -> {
-                    NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(), cap.getDiskGb(), 1);
-                    return IntStream.range(0, cap.getCount()).mapToObj(i -> resources);
-                })
-                .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed())
-                .collect(Collectors.toList());
+    /** Converge zone to wanted capacity */
+    private void convergeToCapacity(NodeList nodes) {
+        List<NodeResources> capacity = targetCapacity();
+        List<Node> excessHosts = provision(capacity, nodes);
+        excessHosts.forEach(host -> {
+            try {
+                hostProvisioner.deprovision(host);
+                nodeRepository().removeRecursively(host, true);
+            } catch (RuntimeException e) {
+                log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e);
+            }
+        });
+    }
 
 
-        for (Iterator<NodeResources> it = preProvisionCapacity.iterator(); it.hasNext() && !removableHosts.isEmpty();) {
+    /**
+     * Provision the nodes necessary to satisfy given capacity.
+     *
+     * @return Excess hosts that can safely be deprovisioned, if any.
+     */
+    private List<Node> provision(List<NodeResources> capacity, NodeList nodes) {
+        List<Node> existingHosts = availableHostsOf(nodes);
+        if (nodeRepository().zone().getCloud().dynamicProvisioning()) {
+            existingHosts = removableHostsOf(existingHosts, nodes);
+        } else if (capacity.isEmpty()) {
+            return List.of();
+        }
+        List<Node> excessHosts = new ArrayList<>(existingHosts);
+        for (Iterator<NodeResources> it = capacity.iterator(); it.hasNext() && !excessHosts.isEmpty(); ) {
             NodeResources resources = it.next();
-            removableHosts.stream()
-                    .filter(nodeRepository()::canAllocateTenantNodeTo)
-                    .filter(host -> nodeRepository().resourcesCalculator().advertisedResourcesOf(host.flavor()).satisfies(resources))
-                    .min(Comparator.comparingInt(n -> n.flavor().cost()))
-                    .ifPresent(host -> {
-                        removableHosts.remove(host);
-                        it.remove();
-                    });
+            excessHosts.stream()
+                       .filter(nodeRepository()::canAllocateTenantNodeTo)
+                       .filter(host -> nodeRepository().resourcesCalculator()
+                                                       .advertisedResourcesOf(host.flavor())
+                                                       .satisfies(resources))
+                       .min(Comparator.comparingInt(n -> n.flavor().cost()))
+                       .ifPresent(host -> {
+                           excessHosts.remove(host);
+                           it.remove();
+                       });
         }
-
-        // pre-provisioning is best effort, do one host at a time
-        preProvisionCapacity.forEach(resources -> {
+        // Pre-provisioning is best effort, do one host at a time
+        capacity.forEach(resources -> {
             try {
                 Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
                 List<Node> hosts = hostProvisioner.provisionHosts(nodeRepository().database().getProvisionIndexes(1),
@@ -127,35 +144,47 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                                                   .collect(Collectors.toList());
                 nodeRepository().addNodes(hosts, Agent.DynamicProvisioningMaintainer);
             } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) {
-                log.log(Level.WARNING, "Failed to pre-provision " + resources + ":" + e.getMessage());
+                log.log(Level.WARNING, "Failed to pre-provision " + resources + ": " + e.getMessage());
             } catch (RuntimeException e) {
                 log.log(Level.WARNING, "Failed to pre-provision " + resources + ", will retry in " + interval(), e);
             }
         });
+        return removableHostsOf(excessHosts, nodes);
+    }
 
-        // Finally, deprovision excess hosts.
-        removableHosts.forEach(host -> {
-            try {
-                hostProvisioner.deprovision(host);
-                nodeRepository().removeRecursively(host, true);
-            } catch (RuntimeException e) {
-                log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e);
-            }
-        });
+
+    /** Reads node resources declared by target capacity flag */
+    private List<NodeResources> targetCapacity() {
+        return targetCapacityFlag.value().stream()
+                                 .flatMap(cap -> {
+                                     NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(),
+                                                                                 cap.getDiskGb(), 1);
+                                     return IntStream.range(0, cap.getCount()).mapToObj(i -> resources);
+                                 })
+                                 .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed())
+                                 .collect(Collectors.toList());
     }
 
-    private static Collection<Node> getRemovableHosts(NodeList nodes) {
-        Map<String, Node> hostsByHostname = nodes.nodeType(NodeType.host)
-                .asList().stream()
-                .filter(host -> host.state() != Node.State.parked || host.status().wantToDeprovision())
-                .collect(Collectors.toMap(Node::hostname, Function.identity()));
+    /** Returns hosts that are considered available, i.e. not parked or flagged for deprovisioning */
+    private static List<Node> availableHostsOf(NodeList nodes) {
+        return nodes.nodeType(NodeType.host)
+                    .matching(host -> host.state() != Node.State.parked || host.status().wantToDeprovision())
+                    .asList();
+    }
+
+    /** Returns the subset of given hosts that have no containers and are thus removable */
+    private static List<Node> removableHostsOf(List<Node> hosts, NodeList allNodes) {
+        Map<String, Node> hostsByHostname = hosts.stream()
+                                                 .collect(Collectors.toMap(Node::hostname,
+                                                                           Function.identity()));
 
-        nodes.asList().stream()
+        allNodes.asList().stream()
                 .filter(node -> node.allocation().isPresent())
                 .flatMap(node -> node.parentHostname().stream())
                 .distinct()
                 .forEach(hostsByHostname::remove);
 
-        return hostsByHostname.values();
+        return List.copyOf(hostsByHostname.values());
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index d388fb5a967..8a82c74dd17 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -90,7 +90,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                 new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource));
         capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval);
         osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval);
-        rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval);
+        rebalancer = new Rebalancer(deployer, nodeRepository, metric, clock, defaults.rebalancerInterval);
         nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval);
         autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, deployer, metric, defaults.autoscalingInterval);
         scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
index f044eaf97f6..12990447eee 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity;
-import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -22,29 +21,27 @@ import java.util.Optional;
  * @author bratseth
  */
 public class Rebalancer extends NodeRepositoryMaintainer {
+
     static final Duration waitTimeAfterPreviousDeployment = Duration.ofMinutes(10);
 
     private final Deployer deployer;
-    private final Optional<HostProvisioner> hostProvisioner;
     private final Metric metric;
     private final Clock clock;
 
     public Rebalancer(Deployer deployer,
                       NodeRepository nodeRepository,
-                      Optional<HostProvisioner> hostProvisioner,
                       Metric metric,
                       Clock clock,
                       Duration interval) {
         super(nodeRepository, interval);
         this.deployer = deployer;
-        this.hostProvisioner = hostProvisioner;
         this.metric = metric;
         this.clock = clock;
     }
 
     @Override
     protected void maintain() {
-        if (hostProvisioner.isPresent()) return; // All nodes will be allocated on new hosts, so rebalancing makes no sense
+        if (nodeRepository().canProvisionHostsWhenRequired()) return; // All nodes will be allocated on new hosts, so rebalancing makes no sense
         if (nodeRepository().zone().environment().isTest()) return; // Test zones have short lived deployments, no need to rebalance
 
         // Work with an unlocked snapshot as this can take a long time and full consistency is not needed
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
index 6b8d1b13975..e04c1aa208d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
@@ -52,6 +52,12 @@ public class FlavorConfigBuilder {
                 flavorConfigBuilder.addFlavor(flavorName, 2.,  40., 40., 0.5, Flavor.Type.DOCKER_CONTAINER);
             else if (flavorName.equals("host"))
                 flavorConfigBuilder.addFlavor(flavorName, 7., 100., 120., 5, Flavor.Type.BARE_METAL);
+            else if (flavorName.equals("host2"))
+                flavorConfigBuilder.addFlavor(flavorName, 16, 24, 100, 1, Flavor.Type.BARE_METAL);
+            else if (flavorName.equals("host3"))
+                flavorConfigBuilder.addFlavor(flavorName, 24, 64, 100, 1, Flavor.Type.BARE_METAL);
+            else if (flavorName.equals("host4"))
+                flavorConfigBuilder.addFlavor(flavorName, 48, 128, 1000, 1, Flavor.Type.BARE_METAL);
             else if (flavorName.equals("devhost"))
                 flavorConfigBuilder.addFlavor(flavorName, 4.,  80., 100, 10, Flavor.Type.BARE_METAL);
             else
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index a1d8ffb03d3..caecf8edf2f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -11,13 +11,12 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.ListFlag;
-import com.yahoo.vespa.flags.custom.PreprovisionCapacity;
+import com.yahoo.vespa.flags.custom.HostCapacity;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -31,14 +30,14 @@ public class GroupPreparer {
 
     private final NodeRepository nodeRepository;
     private final Optional<HostProvisioner> hostProvisioner;
-    private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
+    private final ListFlag<HostCapacity> preprovisionCapacityFlag;
 
     public GroupPreparer(NodeRepository nodeRepository,
                          Optional<HostProvisioner> hostProvisioner,
                          FlagSource flagSource) {
         this.nodeRepository = nodeRepository;
         this.hostProvisioner = hostProvisioner;
-        this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
+        this.preprovisionCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource);
     }
 
     /**
@@ -59,7 +58,7 @@ public class GroupPreparer {
     // active config model which is changed on activate
     public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
                               List<Node> surplusActiveNodes, MutableInteger highestIndex, int spareCount, int wantedGroups) {
-        boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && nodeRepository.zone().getCloud().dynamicProvisioning();
+        boolean dynamicProvisioningEnabled = nodeRepository.canProvisionHostsWhenRequired() && nodeRepository.zone().getCloud().dynamicProvisioning();
         boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
         try (Mutex lock = nodeRepository.lock(application)) {
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index c79c6b45247..8e5cb9d088c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
-import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
 
 import java.time.Clock;
 import java.time.Instant;
@@ -65,7 +64,7 @@ public class MockNodeRepository extends NodeRepository {
               Zone.defaultZone(),
               new MockNameResolver().mockAnyLookup(),
               DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-              true);
+              true, false);
         this.flavors = flavors;
 
         curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
index 88804576310..52c68cd74b2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvid
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 
-import java.time.Clock;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -41,7 +40,7 @@ public class NodeRepositoryTester {
                                             Zone.defaultZone(),
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true);
+                                            true, false);
     }
     
     public NodeRepository nodeRepository() { return nodeRepository; }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
index 3a87b5967fb..864cfad2f41 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -339,9 +339,10 @@ public class AutoscalingTest {
         flavors.add(new Flavor("aws-large",  new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
         flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
         flavors.add(new Flavor("aws-small",  new NodeResources(3,  80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
-        AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.defaultCloud()
-                                                                       .withDynamicProvisioning(true)
-                                                                       .withAllowHostSharing(false),
+        AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.builder()
+                                                                       .dynamicProvisioning(true)
+                                                                       .allowHostSharing(false)
+                                                                       .build(),
                                                                   SystemName.main,
                                                                   Environment.prod, RegionName.from("us-east")),
                                                          flavors);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index b1f6eaea502..a6b2f6b15ea 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -62,7 +62,7 @@ public class CapacityCheckerTester {
                                             zone,
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true);
+                                            true, false);
     }
 
     private void updateCapacityChecker() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
index 6eba517fde2..ba859655ab7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
@@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.Cloud;
 import com.yahoo.config.provision.ClusterMembership;
-import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
@@ -14,13 +15,12 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
-import com.yahoo.vespa.flags.custom.PreprovisionCapacity;
+import com.yahoo.vespa.flags.custom.HostCapacity;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.hosted.provision.node.History;
@@ -30,205 +30,239 @@ import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
-import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.createNode;
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyApp;
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyHostApp;
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantApp;
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantHostApp;
+import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.MockHostProvisioner.Behaviour;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-import static org.mockito.hamcrest.MockitoHamcrest.argThat;
 
 /**
  * @author freva
+ * @author mpolden
  */
 public class DynamicProvisioningMaintainerTest {
 
-    private final HostProvisionerTester tester = new HostProvisionerTester();
-    private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
-    private static final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class);
-    private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
-            .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class);
-    private final DynamicProvisioningMaintainer maintainer =
-            new DynamicProvisioningMaintainer(tester.nodeRepository,
-                                              Duration.ofDays(1),
-                                              hostProvisioner,
-                                              flagSource);
-
     @Test
     public void delegates_to_host_provisioner_and_writes_back_result() {
-        addNodes();
+        var tester = new DynamicProvisioningTester().addInitialNodes();
+        tester.hostProvisioner.with(Behaviour.failDeprovisioning); // To avoid deleting excess nodes
+
         Node host3 = tester.nodeRepository.getNode("host3").orElseThrow();
         Node host4 = tester.nodeRepository.getNode("host4").orElseThrow();
         Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow();
-        assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
-
-        Node host3new = host3.with(host3.ipConfig().with(Set.of("::5")));
-        when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new));
+        assertTrue("No IP addresses assigned",
+                   Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
 
-        Node host4new = host4.with(host4.ipConfig().with(Set.of("::2")));
-        Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1")));
-        when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenReturn(List.of(host4new, host41new));
+        Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0")));
+        Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0")));
+        Node host41new = host41.with(host41.ipConfig().with(Set.of("::4:1", "::4:2")));
 
-        maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {});
-        verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41)));
-        verify(hostProvisioner).provision(eq(host3), eq(Set.of()));
-        verifyNoMoreInteractions(hostProvisioner);
-
-        assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3"));
-        assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4"));
-        assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1"));
+        tester.maintainer.maintain();
+        assertEquals(host3new, tester.nodeRepository.getNode("host3").get());
+        assertEquals(host4new, tester.nodeRepository.getNode("host4").get());
+        assertEquals(host41new, tester.nodeRepository.getNode("host4-1").get());
     }
 
     @Test
     public void correctly_fails_if_irrecoverable_failure() {
-        Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty());
-        Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp));
-
-        assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
-        when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenThrow(new FatalProvisioningException("Fatal"));
-
-        maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {});
+        var tester = new DynamicProvisioningTester();
+        tester.hostProvisioner.with(Behaviour.failProvisioning);
+        Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned);
+        Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, DynamicProvisioningTester.tenantApp);
+        assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
 
+        tester.maintainer.maintain();
         assertEquals(Set.of("host4", "host4-1"),
-                tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet()));
+                     tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet()));
     }
 
     @Test
     public void finds_nodes_that_need_deprovisioning_without_pre_provisioning() {
-        addNodes();
+        var tester = new DynamicProvisioningTester().addInitialNodes();
+        assertTrue(tester.nodeRepository.getNode("host2").isPresent());
+        assertTrue(tester.nodeRepository.getNode("host3").isPresent());
 
-        maintainer.convergeToCapacity(tester.nodeRepository.list());
-        verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2")));
-        verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host3")));
-        verifyNoMoreInteractions(hostProvisioner);
+        tester.maintainer.maintain();
         assertTrue(tester.nodeRepository.getNode("host2").isEmpty());
         assertTrue(tester.nodeRepository.getNode("host3").isEmpty());
     }
 
     @Test
     public void does_not_deprovision_when_preprovisioning_enabled() {
-        flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1)), PreprovisionCapacity.class);
-        addNodes();
-
-        maintainer.convergeToCapacity(tester.nodeRepository.list());
-        verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); // host2 because it is failed
-        verifyNoMoreInteractions(hostProvisioner);
+        var tester = new DynamicProvisioningTester().addInitialNodes();
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(1, 3, 2, 1)), HostCapacity.class);
+        Optional<Node> failedHost = tester.nodeRepository.getNode("host2");
+        assertTrue(failedHost.isPresent());
+
+        tester.maintainer.maintain();
+        assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty());
+        assertEquals(1, tester.hostProvisioner.deprovisionedHosts);
     }
 
     @Test
     public void provision_deficit_and_deprovision_excess() {
-        flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(),
-                                List.of(new PreprovisionCapacity(2, 4, 8, 1),
-                                        new PreprovisionCapacity(2, 3, 2, 2)),
-                                PreprovisionCapacity.class);
-        addNodes();
-
-        maintainer.convergeToCapacity(tester.nodeRepository.list());
-        assertTrue(tester.nodeRepository.getNode("host2").isEmpty());
-        assertTrue(tester.nodeRepository.getNode("host3").isPresent());
-        verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2")));
-        verify(hostProvisioner, times(2)).provisionHosts(argThatLambda(list -> list.size() == 1), eq(new NodeResources(2, 3, 2, 1)), any(), any());
-        verifyNoMoreInteractions(hostProvisioner);
+        var tester = new DynamicProvisioningTester().addInitialNodes();
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(),
+                                       List.of(new HostCapacity(24, 64, 100, 2),
+                                               new HostCapacity(16, 24, 100, 1)),
+                                       HostCapacity.class);
+        assertTrue(tester.nodeRepository.getNode("host2").isPresent());
+        assertEquals(0 ,tester.hostProvisioner.provisionedHosts.size());
+
+        // failed host2 is removed
+        Optional<Node> failedHost = tester.nodeRepository.getNode("host2");
+        assertTrue(failedHost.isPresent());
+        tester.maintainer.maintain();
+        assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty());
+        assertTrue("Host with matching resources is kept", tester.nodeRepository.getNode("host3").isPresent());
+
+        // Two more hosts are provisioned with expected resources
+        NodeResources resources = new NodeResources(24, 64, 100, 1);
+        assertEquals(2, tester.provisionedHostsMatching(resources));
     }
 
     @Test
     public void does_not_remove_if_host_provisioner_failed() {
-        Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp));
-        doThrow(new RuntimeException()).when(hostProvisioner).deprovision(eq(host2));
-
-        maintainer.convergeToCapacity(tester.nodeRepository.list());
+        var tester = new DynamicProvisioningTester();
+        Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, DynamicProvisioningTester.tenantApp);
+        tester.hostProvisioner.with(Behaviour.failDeprovisioning);
 
-        assertEquals(1, tester.nodeRepository.getNodes().size());
-        verify(hostProvisioner).deprovision(eq(host2));
-        verifyNoMoreInteractions(hostProvisioner);
+        tester.maintainer.maintain();
+        assertTrue(tester.nodeRepository.getNode(host2.hostname()).isPresent());
     }
 
-    @Before
-    public void setup() {
-        doAnswer(invocation ->  {
-            Flavor flavor = invocation.getArgument(0, Flavor.class);
-            if ("default".equals(flavor.name())) return new NodeResources(2, 4, 8, 1);
-            return invocation.getArguments()[1];
-        }).when(hostResourcesCalculator).advertisedResourcesOf(any());
+    @Test
+    public void provision_exact_capacity() {
+        var tester = new DynamicProvisioningTester(Cloud.builder().dynamicProvisioning(false).build());
+        NodeResources resources1 = new NodeResources(24, 64, 100, 1);
+        NodeResources resources2 = new NodeResources(16, 24, 100, 1);
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1),
+                                                                           new HostCapacity(resources2.vcpu(), resources2.memoryGb(), resources2.diskGb(), 2)),
+                                       HostCapacity.class);
+        tester.maintainer.maintain();
+
+        // Hosts are provisioned
+        assertEquals(1, tester.provisionedHostsMatching(resources1));
+        assertEquals(2, tester.provisionedHostsMatching(resources2));
+
+        // Next maintenance run does nothing
+        tester.assertNodesUnchanged();
+
+        // Target capacity is changed
+        NodeResources resources3 = new NodeResources(48, 128, 1000, 1);
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1),
+                                                                           new HostCapacity(resources3.vcpu(), resources3.memoryGb(), resources3.diskGb(), 1)),
+                                       HostCapacity.class);
+
+        // Excess hosts are deprovisioned
+        tester.maintainer.maintain();
+        assertEquals(1, tester.provisionedHostsMatching(resources1));
+        assertEquals(0, tester.provisionedHostsMatching(resources2));
+        assertEquals(1, tester.provisionedHostsMatching(resources3));
+        assertEquals(2, tester.nodeRepository.getNodes(Node.State.deprovisioned).size());
+
+        // Activate hosts
+        tester.maintainer.maintain(); // Resume provisioning of new hosts
+        List<Node> provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList();
+        tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName());
+        tester.provisioningTester.deployZoneApp();
+
+        // Allocating nodes to a host does not result in provisioning of additional capacity
+        ApplicationId application = tester.provisioningTester.makeApplicationId();
+        tester.provisioningTester.deploy(application,
+                                         Capacity.from(new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1))));
+        assertEquals(2, tester.nodeRepository.list().owner(application).size());
+        tester.assertNodesUnchanged();
+
+        // Clearing flag does nothing
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(), HostCapacity.class);
+        tester.assertNodesUnchanged();
+
+        // Capacity reduction does not remove host with children
+        tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1)),
+                                       HostCapacity.class);
+        tester.assertNodesUnchanged();
     }
 
-    public void addNodes() {
-        List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)),
-                createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
-                createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()),
+    private static class DynamicProvisioningTester {
 
-                createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)),
-                createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()),
+        private static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default");
+        private static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default");
+        private static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default");
+        private static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default");
+        private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default", "docker", "host2", "host3", "host4");
 
-                createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
+        private final InMemoryFlagSource flagSource = new InMemoryFlagSource().withListFlag(Flags.TARGET_CAPACITY.id(),
+                                                                                            List.of(),
+                                                                                            HostCapacity.class);
 
-                createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
-                createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
+        private final NodeRepository nodeRepository;
+        private final MockHostProvisioner hostProvisioner;
+        private final DynamicProvisioningMaintainer maintainer;
+        private final ProvisioningTester provisioningTester;
 
-                createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()),
+        public DynamicProvisioningTester() {
+            this(Cloud.builder().dynamicProvisioning(true).build());
+        }
 
-                createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)),
-                createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp)))
-                .forEach(node -> tester.nodeRepository.database().addNodesInState(List.of(node), node.state()));
-    }
+        public DynamicProvisioningTester(Cloud cloud) {
+            MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup();
+            this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver);
+            this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(),
+                                                                                     Environment.defaultEnvironment(),
+                                                                                     RegionName.defaultName()))
+                                                                      .flavors(flavors.getFlavors())
+                                                                      .nameResolver(nameResolver)
+                                                                      .hostProvisioner(hostProvisioner)
+                                                                      .build();
+            this.nodeRepository = provisioningTester.nodeRepository();
+            this.maintainer = new DynamicProvisioningMaintainer(nodeRepository,
+                                                                Duration.ofDays(1),
+                                                                hostProvisioner,
+                                                                flagSource);
+        }
 
-    @SuppressWarnings("unchecked")
-    private static <T> T argThatLambda(Predicate<T> predicate) {
-        return argThat(new BaseMatcher<T>() {
-            @Override public boolean matches(Object item) { return predicate.test((T) item); }
-            @Override public void describeTo(Description description) { }
-        });
-    }
+        private DynamicProvisioningTester addInitialNodes() {
+            List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)),
+                    createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
+                    createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()),
+                    createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)),
+                    createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()),
+                    createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
+                    createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
+                    createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
+                    createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()),
+                    createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)),
+                    createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp)))
+                .forEach(node -> nodeRepository.database().addNodesInState(List.of(node), node.state()));
+            return this;
+        }
 
-    static class HostProvisionerTester {
-        private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "docker");
-        static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default");
-        static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default");
-        static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default");
-        static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default");
-
-        private final ManualClock clock = new ManualClock();
-        private final Zone zone = new Zone(Cloud.defaultCloud().withDynamicProvisioning(true), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName());
-        private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors,
-                                                                         hostResourcesCalculator,
-                                                                         new MockCurator(),
-                                                                         clock,
-                                                                         zone,
-                                                                         new MockNameResolver().mockAnyLookup(),
-                DockerImage.fromString("docker-image"), true);
-
-        Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) {
-            Node node = createNode(hostname, parentHostname, nodeType, state, application);
+        private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state) {
+            return addNode(hostname, parentHostname, nodeType, state, null);
+        }
+
+        private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, ApplicationId application) {
+            Node node = createNode(hostname, parentHostname, nodeType, state, Optional.ofNullable(application));
             return nodeRepository.database().addNodesInState(List.of(node), node.state()).get(0);
         }
 
-        static Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) {
-            Flavor flavor = nodeFlavors.getFlavor(parentHostname.isPresent() ? "docker" : "default").orElseThrow();
+        private Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) {
+            Flavor flavor = nodeRepository.flavors().getFlavor(parentHostname.isPresent() ? "docker" : "host2").orElseThrow();
             Optional<Allocation> allocation = application
                     .map(app -> new Allocation(
                             app,
@@ -238,7 +272,101 @@ public class DynamicProvisioningMaintainerTest {
                             false));
             var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of());
             return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(),
-                    state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty());
+                            state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty());
         }
+
+        private long provisionedHostsMatching(NodeResources resources) {
+            return hostProvisioner.provisionedHosts.stream()
+                                                   .filter(host -> host.nodeResources().equals(resources))
+                                                   .count();
+        }
+
+        private void assertNodesUnchanged() {
+            List<Node> nodes = nodeRepository.getNodes();
+            maintainer.maintain();
+            assertEquals("Nodes are unchanged after maintenance run", nodes, nodeRepository.getNodes());
+        }
+
+    }
+
+    static class MockHostProvisioner implements HostProvisioner {
+
+        private final List<ProvisionedHost> provisionedHosts = new ArrayList<>();
+        private final NodeFlavors flavors;
+        private final MockNameResolver nameResolver;
+
+        private int deprovisionedHosts = 0;
+        private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class);
+
+        public MockHostProvisioner(NodeFlavors flavors, MockNameResolver nameResolver) {
+            this.flavors = flavors;
+            this.nameResolver = nameResolver;
+        }
+
+        @Override
+        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) {
+            Flavor hostFlavor = flavors.getFlavors().stream()
+                                       .filter(f -> !f.isDocker())
+                                       .filter(f -> f.resources().compatibleWith(resources))
+                                       .findFirst()
+                                       .orElseThrow(() -> new IllegalArgumentException("No host flavor found satisfying " + resources));
+            List<ProvisionedHost> hosts = new ArrayList<>();
+            for (int index : provisionIndexes) {
+                hosts.add(new ProvisionedHost("host" + index,
+                                              "hostname" + index,
+                                              hostFlavor,
+                                              "nodename" + index,
+                                              resources,
+                                              osVersion));
+            }
+            provisionedHosts.addAll(hosts);
+            return hosts;
+        }
+
+        @Override
+        public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
+            if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)");
+            assertSame(Node.State.provisioned, host.state());
+            List<Node> result = new ArrayList<>();
+            result.add(withIpAssigned(host));
+            for (var child : children) {
+                assertSame(Node.State.reserved, child.state());
+                result.add(withIpAssigned(child));
+            }
+            return result;
+        }
+
+        @Override
+        public void deprovision(Node host) {
+            if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node");
+            provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname()));
+            deprovisionedHosts++;
+        }
+
+        private MockHostProvisioner with(Behaviour first, Behaviour... rest) {
+            this.behaviours = EnumSet.of(first, rest);
+            return this;
+        }
+
+        private Node withIpAssigned(Node node) {
+            if (node.parentHostname().isPresent()) return node;
+            int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", ""));
+            Set<String> addresses = Set.of("::" + hostIndex + ":0");
+            nameResolver.addRecord(node.hostname(), addresses.iterator().next());
+            Set<String> pool = new HashSet<>();
+            for (int i = 1; i <= 2; i++) {
+                String ip = "::" + hostIndex + ":" + i;
+                pool.add(ip);
+                nameResolver.addRecord(node.hostname() + "-" + i, ip);
+            }
+            return node.with(node.ipConfig().with(addresses).with(IP.Pool.of(pool)));
+        }
+
+        enum Behaviour {
+            failProvisioning,
+            failDeprovisioning,
+        }
+
     }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 4fb1c3bcab2..24a0020df4f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -258,7 +258,7 @@ public class FailedExpirerTest {
                                                      zone,
                                                      new MockNameResolver().mockAnyLookup(),
                                                      DockerImage.fromString("docker-image"),
-                                                     true);
+                                                     true, false);
             this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
             this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30));
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index 11df6146b06..754870e798e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -3,21 +3,17 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
-import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
-import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 
 import java.time.Instant;
@@ -44,7 +40,7 @@ public class MaintenanceTester {
                                                                     zone,
                                                                     new MockNameResolver().mockAnyLookup(),
                                                                     DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                                    true);
+                                                                    true, false);
 
     public MaintenanceTester() {
         curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index e1cb57c4eb1..9fc2f666d27 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -84,7 +84,7 @@ public class MetricsReporterTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true);
+                                                           true, false);
         Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant);
         nodeRepository.addNodes(List.of(node), Agent.system);
         Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy);
@@ -150,7 +150,7 @@ public class MetricsReporterTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true);
+                                                           true, false);
 
         // Allow 4 containers
         Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 0a95845898b..9cf03c6f33b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -82,7 +82,7 @@ public class NodeFailTester {
                                             zone,
                                             new MockNameResolver().mockAnyLookup(),
                                             DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                            true);
+                                            true, false);
         provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
         hostLivenessTracker = new TestHostLivenessTracker(clock);
         orchestrator = new OrchestratorMock();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index e57d57d4c4c..b9c57e013c3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -61,7 +61,7 @@ public class OperatorChangeApplicationMaintainerTest {
                                                  zone,
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                 true);
+                                                 true, false);
         this.fixture = new Fixture(zone, nodeRepository);
 
         createReadyNodes(15, this.fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index 8a2a69bb437..b6ffe4ebe26 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -67,7 +67,7 @@ public class PeriodicApplicationMaintainerTest {
                                                  zone,
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                 true);
+                                                 true, false);
         this.fixture = new Fixture(zone, nodeRepository);
 
         createReadyNodes(15, fixture.nodeResources, nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
index 19dc3699dd9..05e5b4829e9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
@@ -150,7 +150,7 @@ public class RebalancerTest {
                     cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))),
                     memoryApp, new MockDeployer.ApplicationContext(memoryApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources))));
             deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps);
-            rebalancer = new Rebalancer(deployer, tester.nodeRepository(), Optional.empty(), metric, tester.clock(), Duration.ofMinutes(1));
+            rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, tester.clock(), Duration.ofMinutes(1));
             tester.makeReadyNodes(3, "flat", NodeType.host, 8);
             tester.deployZoneApp();
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index 59514cb3c95..2c0dac10f9f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -52,7 +52,7 @@ public class ReservationExpirerTest {
                                                            Zone.defaultZone(),
                                                            new MockNameResolver().mockAnyLookup(),
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                           true);
+                                                           true, false);
         NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource());
 
         List<Node> nodes = new ArrayList<>(2);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index addbf717811..29451d7b690 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -71,7 +71,7 @@ public class RetiredExpirerTest {
                                                                      zone,
                                                                      new MockNameResolver().mockAnyLookup(),
                                                                      DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                                     true);
+                                                                     true, false);
     private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
     private final Orchestrator orchestrator = mock(Orchestrator.class);
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
index 087f8f83f0c..ea0d78e3015 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
@@ -33,7 +33,6 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static com.yahoo.config.provision.NodeResources.DiskSpeed.any;
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
 import static com.yahoo.config.provision.NodeResources.StorageType.local;
 import static com.yahoo.config.provision.NodeResources.StorageType.remote;
@@ -52,7 +51,7 @@ import static org.mockito.Mockito.verify;
 public class DynamicDockerProvisionTest {
 
     private static final Zone zone = new Zone(
-            Cloud.defaultCloud().withDynamicProvisioning(true).withAllowHostSharing(false),
+            Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(),
             SystemName.main,
             Environment.prod,
             RegionName.from("us-east"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 92e64e752a7..9a02bc17b4b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -94,6 +94,7 @@ public class ProvisioningTester {
         this.curator = curator;
         this.nodeFlavors = nodeFlavors;
         this.clock = new ManualClock();
+        ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner);
         this.nodeRepository = new NodeRepository(nodeFlavors,
                                                  resourcesCalculator,
                                                  curator,
@@ -101,9 +102,9 @@ public class ProvisioningTester {
                                                  zone,
                                                  nameResolver,
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
-                                                 true);
+                                                 true,
+                                                 provisionServiceProvider.getHostProvisioner().isPresent());
         this.orchestrator = orchestrator;
-        ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner);
         this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource);
         this.capacityPolicies = new CapacityPolicies(nodeRepository);
         this.provisionLogger = new NullProvisionLogger();