Merge pull request #16752 from vespa-engine/mpolden/provision-config-servers

Provision config servers dynamically

21 files changed, 539 insertions, 321 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java
index e356ee06ac6..63ad6ae1237 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java
@@ -75,4 +75,16 @@ public enum NodeType {
     public boolean canRun(NodeType type) {
         return childNodeTypes.contains(type);
     }
+
+    /** Returns the host type of this */
+    public NodeType hostType() {
+        if (isHost()) return this;
+        for (NodeType nodeType : values()) {
+            if (nodeType.childNodeTypes.size() == 1 && nodeType.canRun(this)) {
+                return nodeType;
+            }
+        }
+        throw new IllegalArgumentException("No host of " + this + " exists");
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index c8763f7154e..68caea53517 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -103,9 +103,9 @@ public final class Node implements Nodelike {
         }
 
         if (type != NodeType.host && reservedTo.isPresent())
-            throw new IllegalArgumentException("Only hosts can be reserved to a tenant");
+            throw new IllegalArgumentException("Only tenant hosts can be reserved to a tenant");
 
-        if (type != NodeType.host && exclusiveTo.isPresent())
+        if (!type.isHost() && exclusiveTo.isPresent())
             throw new IllegalArgumentException("Only hosts can be exclusive to an application");
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 204d4eea1c4..2979940ee22 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -90,13 +90,13 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
 
     /** Resume provisioning of already provisioned hosts and their children */
     private void resumeProvisioning(NodeList nodes, Mutex lock) {
-        Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream()
+        Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant, NodeType.config).asList().stream()
                                                                        .filter(node -> node.parentHostname().isPresent())
                                                                        .collect(Collectors.groupingBy(
                                                                                node -> node.parentHostname().get(),
                                                                                Collectors.toSet()));
 
-        nodes.state(Node.State.provisioned).hosts().forEach(host -> {
+        nodes.state(Node.State.provisioned).nodeType(NodeType.host, NodeType.confighost).forEach(host -> {
             Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of());
             try {
                 List<Node> updatedNodes = hostProvisioner.provision(host, children);
@@ -197,10 +197,10 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                 .collect(Collectors.toMap(Node::hostname, Function.identity())));
 
         nodes.stream()
-                .filter(node -> node.allocation().isPresent())
-                .flatMap(node -> node.parentHostname().stream())
-                .distinct()
-                .forEach(hostsByHostname::remove);
+             .filter(node -> node.allocation().isPresent())
+             .flatMap(node -> node.parentHostname().stream())
+             .distinct()
+             .forEach(hostsByHostname::remove);
 
         return List.copyOf(hostsByHostname.values());
     }
@@ -246,8 +246,8 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
     private List<Node> provisionHosts(int count, NodeResources nodeResources) {
         try {
             Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
-            List<Integer> provisionIndexes = nodeRepository().database().getProvisionIndexes(count);
-            List<Node> hosts = hostProvisioner.provisionHosts(provisionIndexes, nodeResources,
+            List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count);
+            List<Node> hosts = hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources,
                     ApplicationId.defaultId(), osVersion, HostSharing.shared)
                     .stream()
                     .map(ProvisionedHost::generateHost)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
index 5b9cd6a69e1..b720bf004ff 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -55,6 +55,10 @@ public abstract class Expirer extends NodeRepositoryMaintainer {
     }
 
     protected boolean isExpired(Node node) {
+        return isExpired(node, expiryTime);
+    }
+
+    protected final boolean isExpired(Node node, Duration expiryTime) {
         return node.history().hasEventBefore(eventType, clock().instant().minus(expiryTime));
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
index ae6e716bffe..238f89fc448 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -10,6 +11,7 @@ import com.yahoo.vespa.hosted.provision.node.Status;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Maintenance job which moves inactive nodes to dirty or parked after timeout.
@@ -30,10 +32,15 @@ import java.util.List;
 public class InactiveExpirer extends Expirer {
 
     private final NodeRepository nodeRepository;
+    private final Duration defaultTimeout;
+    private final Map<NodeType, Duration> inactiveTimeouts;
 
-    InactiveExpirer(NodeRepository nodeRepository, Duration inactiveTimeout, Metric metric) {
-        super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, inactiveTimeout, metric);
+    InactiveExpirer(NodeRepository nodeRepository, Duration defaultTimeout, Map<NodeType, Duration> inactiveTimeouts,
+                    Metric metric) {
+        super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, defaultTimeout, metric);
         this.nodeRepository = nodeRepository;
+        this.defaultTimeout = defaultTimeout;
+        this.inactiveTimeouts = Map.copyOf(inactiveTimeouts);
     }
 
     @Override
@@ -45,8 +52,12 @@ public class InactiveExpirer extends Expirer {
 
     @Override
     protected boolean isExpired(Node node) {
-        return    super.isExpired(node)
-               || node.allocation().get().owner().instance().isTester();
+        return super.isExpired(node, timeout(node)) ||
+               node.allocation().get().owner().instance().isTester();
+    }
+
+    private Duration timeout(Node node) {
+        return inactiveTimeouts.getOrDefault(node.type(), defaultTimeout);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 0154b030baa..f2ef0168e2f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -8,18 +8,20 @@ import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostLivenessTracker;
 import com.yahoo.config.provision.InfraDeployer;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.autoscale.MetricsFetcher;
 import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb;
+import com.yahoo.vespa.hosted.provision.autoscale.MetricsFetcher;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -52,7 +54,9 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval));
         maintainers.add(new ReservationExpirer(nodeRepository, defaults.reservationExpiry, metric));
         maintainers.add(new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry));
-        maintainers.add(new InactiveExpirer(nodeRepository, defaults.inactiveExpiry, metric));
+        maintainers.add(new InactiveExpirer(nodeRepository, defaults.inactiveExpiry, Map.of(NodeType.config, defaults.inactiveConfigServerExpiry,
+                                                                                            NodeType.controller, defaults.inactiveControllerExpiry),
+                                            metric));
         maintainers.add(new FailedExpirer(nodeRepository, zone, defaults.failedExpirerInterval, metric));
         maintainers.add(new DirtyExpirer(nodeRepository, defaults.dirtyExpiry, metric));
         maintainers.add(new ProvisionedExpirer(nodeRepository, defaults.provisionedExpiry, metric));
@@ -99,6 +103,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         
         private final Duration reservationExpiry;
         private final Duration inactiveExpiry;
+        private final Duration inactiveConfigServerExpiry;
+        private final Duration inactiveControllerExpiry;
         private final Duration retiredExpiry;
         private final Duration failedExpirerInterval;
         private final Duration dirtyExpiry;
@@ -148,6 +154,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days
             dedicatedClusterControllerMigratorInterval = zone.environment() == Environment.staging || zone.system().isCd() ? Duration.ofMinutes(3)
                                                                                                                            : Duration.ofHours(2);
+            inactiveConfigServerExpiry = Duration.ofMinutes(5);
+            inactiveControllerExpiry = Duration.ofMinutes(5);
 
             if (zone.environment()  == Environment.prod && ! zone.system().isCd()) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
index d637236e1b8..534115342f3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
@@ -560,6 +560,22 @@ public class Nodes {
         return performOn(filter, (node, lock) -> write(node.withWantToRetire(true, agent, instant), lock));
     }
 
+    /** Retire and deprovision given host and all of its children */
+    public List<Node> deprovision(Node host, Agent agent, Instant instant) {
+        if (!host.type().isHost()) throw new IllegalArgumentException("Cannot deprovision non-host " + host);
+        Optional<NodeMutex> nodeMutex = lockAndGet(host);
+        if (nodeMutex.isEmpty()) return List.of();
+        List<Node> result;
+        try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) {
+            // This takes allocationLock to prevent any further allocation of nodes on this host
+            host = lock.node();
+            NodeList children = list(allocationLock).childrenOf(host);
+            result = retire(NodeListFilter.from(children.asList()), agent, instant);
+            result.add(write(host.withWantToRetire(true, true, agent, instant), lock));
+        }
+        return result;
+    }
+
     /**
      * Writes this node after it has changed some internal state but NOT changed its state field.
      * This does NOT lock the node repository implicitly, but callers are expected to already hold the lock.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java
index 8118556f4c1..72967cca98a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java
@@ -3,10 +3,8 @@ package com.yahoo.vespa.hosted.provision.os;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
@@ -52,7 +50,7 @@ public class RetiringUpgrader implements Upgrader {
                    .not().deprovisioning()
                    .byIncreasingOsVersion()
                    .first(1)
-                   .forEach(node -> deprovision(node, target.version(), now, allNodes));
+                   .forEach(node -> upgrade(node, target.version(), now));
     }
 
     @Override
@@ -60,26 +58,15 @@ public class RetiringUpgrader implements Upgrader {
         // No action needed in this implementation.
     }
 
-    /** Retire and deprovision given host and its children */
-    private void deprovision(Node host, Version target, Instant now, NodeList allNodes) {
-        if (!host.type().isHost()) throw new IllegalArgumentException("Cannot retire non-host " + host);
-        Optional<NodeMutex> nodeMutex = nodeRepository.nodes().lockAndGet(host);
-        if (nodeMutex.isEmpty()) return;
-        // Take allocationLock to prevent any further allocation of nodes on this host
-        try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) {
-            host = lock.node();
-            NodeType nodeType = host.type();
-
-            LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " +
-                     host.status().osVersion().current().map(Version::toFullString).orElse("<unset>") +
-                     ", want " + target);
-            NodeList children = allNodes.childrenOf(host);
-            nodeRepository.nodes().retire(NodeListFilter.from(children.asList()), Agent.RetiringUpgrader, now);
-            host = host.withWantToRetire(true, true, Agent.RetiringUpgrader, now);
-            host = host.with(host.status().withOsVersion(host.status().osVersion().withWanted(Optional.of(target))));
-            nodeRepository.nodes().write(host, lock);
-            nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, nodeType));
-        }
+    /** Upgrade given host by retiring and deprovisioning it */
+    private void upgrade(Node host, Version target, Instant now) {
+        LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " +
+                 host.status().osVersion().current().map(Version::toFullString).orElse("<unset>") +
+                 ", want " + target);
+        nodeRepository.nodes().deprovision(host, Agent.RetiringUpgrader, now);
+        nodeRepository.nodes().upgradeOs(NodeListFilter.from(host), Optional.of(target));
+        NodeType nodeType = host.type();
+        nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, nodeType));
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index c2fe063dae6..3ed29e14527 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -533,15 +533,15 @@ public class CuratorDatabaseClient {
                  .collect(Collectors.toUnmodifiableList());
     }
 
-    /** Returns a given number of unique provision indexes */
-    public List<Integer> getProvisionIndexes(int numIndexes) {
-        if (numIndexes < 1)
-            throw new IllegalArgumentException("numIndexes must be a positive integer, was " + numIndexes);
-
-        int firstProvisionIndex = (int) provisionIndexCounter.add(numIndexes) - numIndexes;
-        return IntStream.range(0, numIndexes)
-                .mapToObj(i -> firstProvisionIndex + i)
-                .collect(Collectors.toList());
+    /** Returns a given number of unique provision indices */
+    public List<Integer> readProvisionIndices(int count) {
+        if (count < 1)
+            throw new IllegalArgumentException("count must be a positive integer, was " + count);
+
+        int firstIndex = (int) provisionIndexCounter.add(count) - count;
+        return IntStream.range(0, count)
+                        .mapToObj(i -> firstIndex + i)
+                        .collect(Collectors.toList());
     }
 
     public CacheStats cacheStats() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 0a22cc1cc58..e6473e62922 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.OutOfCapacityException;
-import com.yahoo.lang.MutableInteger;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
@@ -90,19 +89,23 @@ public class GroupPreparer {
                                                           allocateOsRequirement);
 
             if (nodeRepository.zone().getCloud().dynamicProvisioning()) {
+                NodeType hostType = allocation.nodeType().hostType();
                 final Version osVersion;
                 if (allocateOsRequirement.equals("rhel8")) {
                     osVersion = new Version(8, Integer.MAX_VALUE /* always use latest 8 version */, 0);
                 } else {
-                    osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
+                    osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion);
                 }
-
-                List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit()
-                        .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()),
-                                                                             deficit.getFlavor(),
-                                                                             application,
-                                                                             osVersion,
-                                                                             requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any))
+                List<ProvisionedHost> provisionedHosts = allocation.nodeDeficit()
+                        .map(deficit -> {
+                            HostSharing sharing = requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any;
+                            return hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.getCount()),
+                                                                        hostType,
+                                                                        deficit.getFlavor(),
+                                                                        application,
+                                                                        osVersion,
+                                                                        sharing);
+                        })
                         .orElseGet(List::of);
 
                 // At this point we have started provisioning of the hosts, the first priority is to make sure that
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index ae8c6757b5a..bfb526a518f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.util.List;
@@ -32,8 +33,9 @@ public interface HostProvisioner {
     /**
      * Schedule provisioning of a given number of hosts.
      *
-     * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames
+     * @param provisionIndices list of unique provision indices which will be used to generate the node hostnames
      *                         on the form of <code>[prefix][index].[domain]</code>
+     * @param hostType The host type to provision
      * @param resources the resources needed per node - the provisioned host may be significantly larger
      * @param applicationId id of the application that will own the provisioned host
      * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable
@@ -41,8 +43,11 @@ public interface HostProvisioner {
      * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned.
      * @return list of {@link ProvisionedHost} describing the provisioned nodes
      */
-    List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
-                                         ApplicationId applicationId, Version osVersion,
+    List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices,
+                                         NodeType hostType,
+                                         NodeResources resources,
+                                         ApplicationId applicationId,
+                                         Version osVersion,
                                          HostSharing sharing);
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index 0eb933a7dcc..f6e0ede4e7d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -281,7 +281,7 @@ class NodeAllocation {
 
     /** Returns true if the content of this list is sufficient to meet the request */
     boolean fulfilled() {
-        return requestedNodes.fulfilledBy(accepted);
+        return requestedNodes.fulfilledBy(accepted());
     }
 
     /** Returns true if this allocation was already fulfilled and resulted in no new changes */
@@ -290,17 +290,49 @@ class NodeAllocation {
     }
 
     /**
-     * Returns {@link FlavorCount} describing the docker node deficit for the given {@link NodeSpec}.
+     * Returns {@link FlavorCount} describing the node deficit for the given {@link NodeSpec}.
      *
-     * @return empty if the requested spec is not count based or the requested flavor type is not docker or
-     *         the request is already fulfilled. Otherwise returns {@link FlavorCount} containing the required flavor
-     *         and node count to cover the deficit.
+     * @return empty if the requested spec is already fulfilled. Otherwise returns {@link FlavorCount} containing the
+     * flavor and node count required to cover the deficit.
      */
-    Optional<FlavorCount> getFulfilledDockerDeficit() {
-        return Optional.of(requestedNodes)
-                .filter(NodeSpec.CountNodeSpec.class::isInstance)
-                .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(accepted)))
-                .filter(flavorCount -> flavorCount.getCount() > 0);
+    Optional<FlavorCount> nodeDeficit() {
+        if (nodeType() != NodeType.config && nodeType() != NodeType.tenant) {
+            return Optional.empty(); // Requests for these node types never have a deficit
+        }
+        return Optional.of(new FlavorCount(requestedNodes.resources().orElseGet(NodeResources::unspecified),
+                                           requestedNodes.fulfilledDeficitCount(accepted())))
+                       .filter(flavorCount -> flavorCount.getCount() > 0);
+    }
+
+    /** Returns the indices to use when provisioning hosts for this */
+    List<Integer> provisionIndices(int count) {
+        if (count < 1) throw new IllegalArgumentException("Count must be positive");
+        NodeType hostType = requestedNodes.type().hostType();
+
+        // Tenant hosts have a continuously increasing index
+        if (hostType == NodeType.host) return nodeRepository.database().readProvisionIndices(count);
+
+        // Infrastructure hosts have fixed indices, starting at 1
+        Set<Integer> currentIndices = allNodes.nodeType(hostType)
+                                              .stream()
+                                              .map(Node::hostname)
+                                              // TODO(mpolden): Use cluster index instead of parsing hostname, once all
+                                              //                config servers have been replaced once and have switched
+                                              //                to compact indices
+                                              .map(NodeAllocation::parseIndex)
+                                              .collect(Collectors.toSet());
+        List<Integer> indices = new ArrayList<>(count);
+        for (int i = 1; indices.size() < count; i++) {
+            if (!currentIndices.contains(i)) {
+                indices.add(i);
+            }
+        }
+        return indices;
+    }
+
+    /** The node type this is allocating */
+    NodeType nodeType() {
+        return requestedNodes.type();
     }
 
     /**
@@ -367,6 +399,14 @@ class NodeAllocation {
                 .collect(Collectors.toList());
     }
 
+    /** Returns the number of nodes accepted this far */
+    private int accepted() {
+        if (nodeType() == NodeType.tenant) return accepted;
+        // Infrastructure nodes are always allocated by type. Count all nodes as accepted so that we never exceed
+        // the wanted number of nodes for the type.
+        return allNodes.nodeType(nodeType()).size();
+    }
+
     /** Prefer to retire nodes we want the least */
     private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) {
         return candidates.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList());
@@ -395,6 +435,15 @@ class NodeAllocation {
         return ": Not enough nodes available due to " + String.join(", ", reasons);
     }
 
+    private static Integer parseIndex(String hostname) {
+        // Node index is the first number appearing in the hostname, before the first dot
+        try {
+            return Integer.parseInt(hostname.replaceFirst("^\\D+(\\d+)\\..*", "$1"));
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("Could not parse index from hostname '" + hostname + "'", e);
+        }
+    }
+
     static class FlavorCount {
 
         private final NodeResources flavor;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
index b37c7b92ea4..c3cb805499c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -170,6 +171,8 @@ public interface NodeSpec {
     /** A node spec specifying a node type. This will accept all nodes of this type. */
     class TypeNodeSpec implements NodeSpec {
 
+        private static final Map<NodeType, Integer> WANTED_NODE_COUNT = Map.of(NodeType.config, 3);
+
         private final NodeType type;
 
         public TypeNodeSpec(NodeType type) {
@@ -204,14 +207,17 @@ public interface NodeSpec {
 
         @Override
         public int fulfilledDeficitCount(int count) {
-            return 0;
+            // If no wanted count is specified for this node type, then any count fulfills the deficit
+            return Math.max(0, WANTED_NODE_COUNT.getOrDefault(type, 0) - count);
         }
 
         @Override
         public NodeSpec fraction(int divisor) { return this; }
 
         @Override
-        public Optional<NodeResources> resources() { return Optional.empty(); }
+        public Optional<NodeResources> resources() {
+            return Optional.empty();
+        }
 
         @Override
         public boolean needsResize(Node node) { return false; }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
index 6ff4d4ca5f8..caaea1167b5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
@@ -27,20 +27,23 @@ public class ProvisionedHost {
     private final String id;
     private final String hostHostname;
     private final Flavor hostFlavor;
+    private final NodeType hostType;
     private final Optional<ApplicationId> exclusiveTo;
     private final List<Address> nodeAddresses;
     private final NodeResources nodeResources;
     private final Version osVersion;
 
-    public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, Optional<ApplicationId> exclusiveTo,
+    public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, Optional<ApplicationId> exclusiveTo,
                            List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion) {
         this.id = Objects.requireNonNull(id, "Host id must be set");
         this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set");
         this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set");
+        this.hostType = Objects.requireNonNull(hostType, "Host type must be set");
         this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be set");
         this.nodeAddresses = validateNodeAddresses(nodeAddresses);
         this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set");
         this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set");
+        if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host");
     }
 
     private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) {
@@ -54,7 +57,7 @@ public class ProvisionedHost {
     /** Generate {@link Node} instance representing the provisioned physical host */
     public Node generateHost() {
         Node.Builder builder = Node
-                .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, NodeType.host)
+                .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, hostType)
                 .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion))));
         exclusiveTo.ifPresent(builder::exclusiveTo);
         return builder.build();
@@ -62,7 +65,7 @@ public class ProvisionedHost {
 
     /** Generate {@link Node} instance representing the node running on this physical host */
     public Node generateNode() {
-        return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, NodeType.tenant).build();
+        return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, hostType.childNodeType()).build();
     }
 
     public String getId() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
new file mode 100644
index 00000000000..25e74df677b
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
@@ -0,0 +1,182 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.OutOfCapacityException;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Address;
+import com.yahoo.vespa.hosted.provision.node.IP;
+import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * @author mpolden
+ */
+public class MockHostProvisioner implements HostProvisioner {
+
+    private final List<ProvisionedHost> provisionedHosts = new ArrayList<>();
+    private final List<Flavor> flavors;
+    private final MockNameResolver nameResolver;
+    private final int memoryTaxGb;
+
+    private int deprovisionedHosts = 0;
+    private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class);
+    private Optional<Flavor> hostFlavor = Optional.empty();
+
+    public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) {
+        this.flavors = List.copyOf(flavors);
+        this.nameResolver = nameResolver;
+        this.memoryTaxGb = memoryTaxGb;
+    }
+
+    public MockHostProvisioner(List<Flavor> flavors) {
+        this(flavors, 0);
+    }
+
+    public MockHostProvisioner(List<Flavor> flavors, int memoryTaxGb) {
+        this(flavors, new MockNameResolver().mockAnyLookup(), memoryTaxGb);
+    }
+
+    @Override
+    public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources,
+                                                ApplicationId applicationId, Version osVersion, HostSharing sharing) {
+        Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources))
+                                                                   .findFirst()
+                                                                   .orElseThrow(() -> new OutOfCapacityException("No host flavor matches " + resources)));
+        List<ProvisionedHost> hosts = new ArrayList<>();
+        for (int index : provisionIndices) {
+            String hostHostname = hostType == NodeType.host ? "hostname" + index : hostType.name() + index;
+            hosts.add(new ProvisionedHost("id-of-" + hostType.name() + index,
+                                          hostHostname,
+                                          hostFlavor,
+                                          hostType,
+                                          Optional.empty(),
+                                          createAddressesForHost(hostType, hostFlavor, index),
+                                          resources,
+                                          osVersion));
+        }
+        provisionedHosts.addAll(hosts);
+        return hosts;
+    }
+
+    @Override
+    public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
+        if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)");
+        if (host.state() != Node.State.provisioned) throw new IllegalStateException("Host to provision must be in " + Node.State.provisioned);
+        List<Node> result = new ArrayList<>();
+        result.add(withIpAssigned(host));
+        for (var child : children) {
+            if (child.state() != Node.State.reserved) throw new IllegalStateException("Child to provisioned must be in " + Node.State.reserved);
+            result.add(withIpAssigned(child));
+        }
+        return result;
+    }
+
+    @Override
+    public void deprovision(Node host) {
+        if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node");
+        provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname()));
+        deprovisionedHosts++;
+    }
+
+    /** Returns the hosts that have been provisioned by this  */
+    public List<ProvisionedHost> provisionedHosts() {
+        return Collections.unmodifiableList(provisionedHosts);
+    }
+
+    /** Returns the number of hosts deprovisioned by this */
+    public int deprovisionedHosts() {
+        return deprovisionedHosts;
+    }
+
+    public MockHostProvisioner with(Behaviour first, Behaviour... rest) {
+        this.behaviours = EnumSet.of(first, rest);
+        return this;
+    }
+
+    public MockHostProvisioner without(Behaviour first, Behaviour... rest) {
+        Set<Behaviour> behaviours = new HashSet<>(this.behaviours);
+        behaviours.removeAll(EnumSet.of(first, rest));
+        this.behaviours = behaviours.isEmpty() ? EnumSet.noneOf(Behaviour.class) : EnumSet.copyOf(behaviours);
+        return this;
+    }
+
+    public MockHostProvisioner overrideHostFlavor(String flavorName) {
+        Flavor flavor = flavors.stream().filter(f -> f.name().equals(flavorName))
+                               .findFirst()
+                               .orElseThrow(() -> new IllegalArgumentException("No such flavor '" + flavorName + "'"));
+        hostFlavor = Optional.of(flavor);
+        return this;
+    }
+
+    public boolean compatible(Flavor flavor, NodeResources resources) {
+        NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb);
+
+        if (flavor.resources().storageType() == NodeResources.StorageType.remote
+            && flavor.resources().diskGb() >= resources.diskGb())
+            resourcesToVerify = resourcesToVerify.withDiskGb(flavor.resources().diskGb());
+        if (flavor.resources().bandwidthGbps() >= resources.bandwidthGbps())
+            resourcesToVerify = resourcesToVerify.withBandwidthGbps(flavor.resources().bandwidthGbps());
+        return flavor.resources().compatibleWith(resourcesToVerify);
+    }
+
+    private List<Address> createAddressesForHost(NodeType hostType, Flavor flavor, int hostIndex) {
+        long numAddresses = Math.max(1, Math.round(flavor.resources().bandwidthGbps()));
+        return IntStream.range(0, (int) numAddresses)
+                        .mapToObj(i -> {
+                            String hostname = hostType == NodeType.host
+                                    ? "nodename" + hostIndex + "_" + i
+                                    : hostType.childNodeType().name() + i;
+                            return new Address(hostname);
+                        })
+                        .collect(Collectors.toList());
+    }
+
+    private Node withIpAssigned(Node node) {
+        if (!node.type().isHost()) {
+            return node.with(node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname())));
+        }
+        int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", ""));
+        Set<String> addresses = Set.of("::" + hostIndex + ":0");
+        Set<String> ipAddressPool = new HashSet<>();
+        if (!behaviours.contains(Behaviour.failDnsUpdate)) {
+            nameResolver.addRecord(node.hostname(), addresses.iterator().next());
+            for (int i = 1; i <= 2; i++) {
+                String ip = "::" + hostIndex + ":" + i;
+                ipAddressPool.add(ip);
+                nameResolver.addRecord(node.hostname() + "-" + i, ip);
+            }
+        }
+        IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool);
+        return node.with(node.ipConfig().withPrimary(addresses).withPool(pool));
+    }
+
+    public enum Behaviour {
+
+        /** Fail all calls to {@link MockHostProvisioner#provision(com.yahoo.vespa.hosted.provision.Node, java.util.Set)} */
+        failProvisioning,
+
+        /** Fail all calls to {@link MockHostProvisioner#deprovision(com.yahoo.vespa.hosted.provision.Node)} */
+        failDeprovisioning,
+
+        /** Fail DNS updates of provisioned hosts */
+        failDnsUpdate,
+
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index 0619b0ad645..77c3a5209e2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import com.yahoo.collections.Pair;
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterResources;
@@ -21,17 +20,12 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.Nodelike;
 import com.yahoo.vespa.hosted.provision.applications.Application;
-import com.yahoo.vespa.hosted.provision.node.Address;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
-import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
-import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
-import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -39,6 +33,9 @@ import java.util.Set;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+/**
+ * @author bratseth
+ */
 class AutoscalingTester {
 
     private final ProvisioningTester provisioningTester;
@@ -296,45 +293,14 @@ class AutoscalingTester {
 
     }
 
-    private class MockHostProvisioner implements HostProvisioner {
+    private class MockHostProvisioner extends com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner {
 
-        private final List<Flavor> hostFlavors;
-
-        public MockHostProvisioner(List<Flavor> hostFlavors) {
-            this.hostFlavors = hostFlavors;
-        }
-
-        @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
-                                                    ApplicationId applicationId, Version osVersion,
-                                                    HostSharing sharing) {
-            Flavor hostFlavor = hostFlavors.stream().filter(f -> matches(f, resources)).findAny()
-                                           .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors));
-
-            List<ProvisionedHost> hosts = new ArrayList<>();
-            for (int index : provisionIndexes) {
-                hosts.add(new ProvisionedHost("host" + index,
-                                              "hostname" + index,
-                                              hostFlavor,
-                                              Optional.empty(),
-                                              List.of(new Address("nodename" + index)),
-                                              resources,
-                                              osVersion));
-            }
-            return hosts;
+        public MockHostProvisioner(List<Flavor> flavors) {
+            super(flavors);
         }
 
         @Override
-        public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
-            throw new RuntimeException("Not implemented");
-        }
-
-        @Override
-        public void deprovision(Node host) {
-            throw new RuntimeException("Not implemented");
-        }
-
-        private boolean matches(Flavor flavor, NodeResources resources) {
+        public boolean compatible(Flavor flavor, NodeResources resources) {
             NodeResources flavorResources = hostResourcesCalculator.advertisedResourcesOf(flavor);
             if (flavorResources.storageType() == NodeResources.StorageType.remote
                 && resources.diskGb() <= flavorResources.diskGb())
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
index 56cf8d02149..4a1b3df0514 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.ParentHostUnavailableException;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
@@ -27,11 +28,10 @@ import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.hosted.provision.node.IP;
-import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
-import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
 import com.yahoo.vespa.service.duper.ConfigServerHostApplication;
@@ -39,21 +39,19 @@ import org.junit.Test;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashSet;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.MockHostProvisioner.Behaviour;
+import static com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner.Behaviour;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author freva
@@ -115,7 +113,7 @@ public class DynamicProvisioningMaintainerTest {
 
         tester.maintainer.maintain();
         assertTrue("Failed host is deprovisioned", tester.nodeRepository.nodes().node(failedHost.get().hostname()).isEmpty());
-        assertEquals(1, tester.hostProvisioner.deprovisionedHosts);
+        assertEquals(1, tester.hostProvisioner.deprovisionedHosts());
     }
 
     @Test
@@ -126,7 +124,7 @@ public class DynamicProvisioningMaintainerTest {
                                                new ClusterCapacity(1, 16, 24, 100, 1.0)),
                                        ClusterCapacity.class);
 
-        assertEquals(0, tester.hostProvisioner.provisionedHosts.size());
+        assertEquals(0, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(11, tester.nodeRepository.nodes().list().size());
         assertTrue(tester.nodeRepository.nodes().node("host2").isPresent());
         assertTrue(tester.nodeRepository.nodes().node("host2-1").isPresent());
@@ -136,7 +134,7 @@ public class DynamicProvisioningMaintainerTest {
 
         tester.maintainer.maintain();
 
-        assertEquals(2, tester.hostProvisioner.provisionedHosts.size());
+        assertEquals(2, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(2, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10)));
         NodeList nodesAfter = tester.nodeRepository.nodes().list();
         assertEquals(11, nodesAfter.size());  // 2 removed, 2 added
@@ -151,13 +149,13 @@ public class DynamicProvisioningMaintainerTest {
     public void preprovision_with_shared_host() {
         var tester = new DynamicProvisioningTester().addInitialNodes();
         // Makes provisioned hosts 48-128-1000-10
-        tester.hostProvisioner.provisionSharedHost("host4");
+        tester.hostProvisioner.overrideHostFlavor("host4");
 
         tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(),
                 List.of(new ClusterCapacity(2, 1, 30, 20, 3.0)),
                 ClusterCapacity.class);
 
-        assertEquals(0, tester.hostProvisioner.provisionedHosts.size());
+        assertEquals(0, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(11, tester.nodeRepository.nodes().list().size());
         assertTrue(tester.nodeRepository.nodes().node("host2").isPresent());
         assertTrue(tester.nodeRepository.nodes().node("host2-1").isPresent());
@@ -196,7 +194,7 @@ public class DynamicProvisioningMaintainerTest {
 
         tester.maintainer.maintain();
 
-        assertEquals(2, tester.hostProvisioner.provisionedHosts.size());
+        assertEquals(2, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(2, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10)));
         assertEquals(10, tester.nodeRepository.nodes().list().size());  // 3 removed, 2 added
         assertTrue("preprovision capacity is prefered on shared hosts", tester.nodeRepository.nodes().node("host3").isEmpty());
@@ -212,7 +210,7 @@ public class DynamicProvisioningMaintainerTest {
         tester.maintainer.maintain();
 
         assertEquals("one provisioned host has been deprovisioned, so there are 2 -> 1 provisioned hosts",
-                1, tester.hostProvisioner.provisionedHosts.size());
+                     1, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(1, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10)));
         assertEquals(9, tester.nodeRepository.nodes().list().size());  // 4 removed, 2 added
         if (tester.nodeRepository.nodes().node("hostname100").isPresent()) {
@@ -226,7 +224,7 @@ public class DynamicProvisioningMaintainerTest {
     }
 
     private void verifyFirstMaintain(DynamicProvisioningTester tester) {
-        assertEquals(1, tester.hostProvisioner.provisionedHosts.size());
+        assertEquals(1, tester.hostProvisioner.provisionedHosts().size());
         assertEquals(1, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10)));
         assertEquals(10, tester.nodeRepository.nodes().list().size());  // 2 removed, 1 added
         assertTrue("Failed host 'host2' is deprovisioned", tester.nodeRepository.nodes().node("host2").isEmpty());
@@ -266,17 +264,17 @@ public class DynamicProvisioningMaintainerTest {
 
     private void assertWithMinCount(int minCount, int provisionCount, int deprovisionCount) {
         var tester = new DynamicProvisioningTester().addInitialNodes();
-        tester.hostProvisioner.provisionSharedHost("host4");
+        tester.hostProvisioner.overrideHostFlavor("host4");
 
         tester.flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(null, minCount), SharedHost.class);
         tester.maintainer.maintain();
-        assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts.size());
-        assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts);
+        assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts().size());
+        assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts());
 
         // Verify next maintain is a no-op
         tester.maintainer.maintain();
-        assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts.size());
-        assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts);
+        assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts().size());
+        assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts());
     }
 
     @Test
@@ -300,14 +298,14 @@ public class DynamicProvisioningMaintainerTest {
 
         // Hosts are provisioned
         assertEquals(2, tester.provisionedHostsMatching(resources1));
-        assertEquals(0, tester.hostProvisioner.deprovisionedHosts);
+        assertEquals(0, tester.hostProvisioner.deprovisionedHosts());
 
         // Next maintenance run does nothing
         tester.assertNodesUnchanged();
 
         // Pretend shared-host flag has been set to host4's flavor
         var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote);
-        tester.hostProvisioner.provisionSharedHost("host4");
+        tester.hostProvisioner.overrideHostFlavor("host4");
 
         // Next maintenance run does nothing
         tester.assertNodesUnchanged();
@@ -421,6 +419,93 @@ public class DynamicProvisioningMaintainerTest {
         assertCfghost3IsDeprovisioned(tester);
     }
 
+    @Test
+    public void replace_config_server() {
+        Cloud cloud = Cloud.builder().dynamicProvisioning(true).build();
+        DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup());
+        ProvisioningTester tester = dynamicProvisioningTester.provisioningTester;
+        dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default");
+
+        // Initial config server hosts are provisioned manually
+        ApplicationId hostApp = ApplicationId.from("hosted-vespa", "configserver-host", "default");
+        List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", NodeType.confighost).stream()
+                                            .sorted(Comparator.comparing(Node::hostname))
+                                            .collect(Collectors.toList());
+        tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost);
+
+        // Provision config servers
+        ApplicationId configSrvApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default");
+        for (int i = 0; i < provisionedHosts.size(); i++) {
+            tester.makeReadyChildren(1, i + 1, NodeResources.unspecified(), NodeType.config,
+                                     provisionedHosts.get(i).hostname(), (nodeIndex) -> "cfg" + nodeIndex);
+        }
+        tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+
+        // Expected number of hosts and children are provisioned
+        NodeList allNodes = tester.nodeRepository().nodes().list();
+        NodeList configHosts = allNodes.nodeType(NodeType.confighost);
+        NodeList configNodes = allNodes.nodeType(NodeType.config);
+        assertEquals(3, configHosts.size());
+        assertEquals(3, configNodes.size());
+        String hostnameToRemove = provisionedHosts.get(1).hostname();
+        Supplier<Node> hostToRemove = () -> tester.nodeRepository().nodes().node(hostnameToRemove).get();
+        Supplier<Node> nodeToRemove = () -> tester.nodeRepository().nodes().node(configNodes.childrenOf(hostnameToRemove).first().get().hostname()).get();
+
+        // Retire and deprovision host
+        tester.nodeRepository().nodes().deprovision(hostToRemove.get(), Agent.system, tester.clock().instant());
+        tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName());
+        assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state());
+        assertSame("Node remains active", Node.State.active, nodeToRemove.get().state());
+        assertTrue("Node wants to retire", nodeToRemove.get().status().wantToRetire());
+
+        // Redeployment of config server application retires node
+        tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+        assertTrue("Redeployment retires node", nodeToRemove.get().allocation().get().membership().retired());
+
+        // Config server becomes removable (done by RetiredExpirer in a real system) and redeployment moves it
+        // to inactive
+        tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get()));
+        tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+        assertEquals("Node moves to inactive", Node.State.inactive, nodeToRemove.get().state());
+
+        // Node is completely removed (done by InactiveExpirer and host-admin in a real system)
+        Node inactiveConfigServer = nodeToRemove.get();
+        int removedIndex = inactiveConfigServer.allocation().get().membership().index();
+        tester.nodeRepository().nodes().removeRecursively(inactiveConfigServer, true);
+        assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.config).size());
+
+        // Host is removed
+        dynamicProvisioningTester.maintainer.maintain();
+        assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.confighost).size());
+
+        // Next deployment starts provisioning a new host and child
+        try {
+            tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+            fail("Expected provisioning to fail");
+        } catch (ParentHostUnavailableException ignored) {}
+        Node newNode = tester.nodeRepository().nodes().list(Node.State.reserved).nodeType(NodeType.config).first().get();
+
+        // Resume provisioning and activate host
+        dynamicProvisioningTester.maintainer.maintain();
+        List<ProvisionedHost> newHosts = dynamicProvisioningTester.hostProvisioner.provisionedHosts();
+        assertEquals(1, newHosts.size());
+        tester.nodeRepository().nodes().setReady(newHosts.get(0).hostHostname(), Agent.operator, getClass().getSimpleName());
+        tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost);
+        assertEquals(3, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.confighost).size());
+
+        // Redeployment of config server app actives new node
+        tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+        newNode = tester.nodeRepository().nodes().node(newNode.hostname()).get();
+        assertSame(Node.State.active, newNode.state());
+        assertEquals("Removed index is reused", removedIndex, newNode.allocation().get().membership().index());
+
+        // Next redeployment does nothing
+        NodeList nodesBefore = tester.nodeRepository().nodes().list().nodeType(NodeType.config);
+        tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config);
+        NodeList nodesAfter = tester.nodeRepository().nodes().list().nodeType(NodeType.config);
+        assertEquals(nodesBefore, nodesAfter);
+    }
+
     private void assertCfghost3IsActive(DynamicProvisioningTester tester) {
         assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size());
         assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size());
@@ -451,12 +536,11 @@ public class DynamicProvisioningMaintainerTest {
         private final ProvisioningTester provisioningTester;
 
         public DynamicProvisioningTester() {
-            this(Cloud.builder().dynamicProvisioning(true).build());
+            this(Cloud.builder().dynamicProvisioning(true).build(), new MockNameResolver());
         }
 
-        public DynamicProvisioningTester(Cloud cloud) {
-            MockNameResolver nameResolver = new MockNameResolver();
-            this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver);
+        public DynamicProvisioningTester(Cloud cloud, MockNameResolver nameResolver) {
+            this.hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), nameResolver, 0);
             this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(),
                                                                                      Environment.defaultEnvironment(),
                                                                                      RegionName.defaultName()))
@@ -529,9 +613,9 @@ public class DynamicProvisioningMaintainerTest {
         }
 
         private long provisionedHostsMatching(NodeResources resources) {
-            return hostProvisioner.provisionedHosts.stream()
-                                                   .filter(host -> host.generateHost().resources().compatibleWith(resources))
-                                                   .count();
+            return hostProvisioner.provisionedHosts().stream()
+                                  .filter(host -> host.generateHost().resources().compatibleWith(resources))
+                                  .count();
         }
 
         private void assertNodesUnchanged() {
@@ -542,113 +626,5 @@ public class DynamicProvisioningMaintainerTest {
 
     }
 
-    static class MockHostProvisioner implements HostProvisioner {
-
-        private final List<ProvisionedHost> provisionedHosts = new ArrayList<>();
-        private final NodeFlavors flavors;
-        private final MockNameResolver nameResolver;
-
-        private int deprovisionedHosts = 0;
-        private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class);
-        private Optional<Flavor> provisionHostFlavor = Optional.empty();
-
-        public MockHostProvisioner(NodeFlavors flavors, MockNameResolver nameResolver) {
-            this.flavors = flavors;
-            this.nameResolver = nameResolver;
-        }
-
-        public MockHostProvisioner provisionSharedHost(String flavorName) {
-            provisionHostFlavor = Optional.of(flavors.getFlavorOrThrow(flavorName));
-            return this;
-        }
-
-        @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
-                                                    ApplicationId applicationId, Version osVersion, HostSharing sharing) {
-            Flavor hostFlavor = provisionHostFlavor
-                    .orElseGet(() -> flavors.getFlavors().stream()
-                            .filter(f -> !f.isDocker())
-                            .filter(f -> f.resources().compatibleWith(resources))
-                            .findFirst()
-                            .orElseThrow(() -> new IllegalArgumentException("No host flavor found satisfying " + resources)));
-
-            List<ProvisionedHost> hosts = new ArrayList<>();
-            for (int index : provisionIndexes) {
-                hosts.add(new ProvisionedHost("host" + index,
-                                              "hostname" + index,
-                                              hostFlavor,
-                                              Optional.empty(),
-                                              createAddressesForHost(hostFlavor, index),
-                                              resources,
-                                              osVersion));
-            }
-            provisionedHosts.addAll(hosts);
-            return hosts;
-        }
-
-        private List<Address> createAddressesForHost(Flavor flavor, int hostIndex) {
-            long numAddresses = Math.max(1, Math.round(flavor.resources().bandwidthGbps()));
-            return IntStream.range(0, (int) numAddresses)
-                    .mapToObj(i -> new Address("nodename" + hostIndex + "_" + i))
-                    .collect(Collectors.toList());
-        }
-
-        @Override
-        public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
-            if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)");
-            assertSame(Node.State.provisioned, host.state());
-            List<Node> result = new ArrayList<>();
-            result.add(withIpAssigned(host));
-            for (var child : children) {
-                assertSame(Node.State.reserved, child.state());
-                result.add(withIpAssigned(child));
-            }
-            return result;
-        }
-
-        @Override
-        public void deprovision(Node host) {
-            if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node");
-            provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname()));
-            deprovisionedHosts++;
-        }
-
-        private MockHostProvisioner with(Behaviour first, Behaviour... rest) {
-            this.behaviours = EnumSet.of(first, rest);
-            return this;
-        }
-
-        private MockHostProvisioner without(Behaviour first, Behaviour... rest) {
-            Set<Behaviour> behaviours = new HashSet<>(this.behaviours);
-            behaviours.removeAll(EnumSet.of(first, rest));
-            this.behaviours = behaviours.isEmpty() ? EnumSet.noneOf(Behaviour.class) : EnumSet.copyOf(behaviours);
-            return this;
-        }
-
-        private Node withIpAssigned(Node node) {
-            if (node.parentHostname().isPresent()) return node;
-            int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", ""));
-            Set<String> addresses = Set.of("::" + hostIndex + ":0");
-            Set<String> ipAddressPool = new HashSet<>();
-            if (!behaviours.contains(Behaviour.failDnsUpdate)) {
-                nameResolver.addRecord(node.hostname(), addresses.iterator().next());
-                for (int i = 1; i <= 2; i++) {
-                    String ip = "::" + hostIndex + ":" + i;
-                    ipAddressPool.add(ip);
-                    nameResolver.addRecord(node.hostname() + "-" + i, ip);
-                }
-            }
-
-            IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool);
-            return node.with(node.ipConfig().withPrimary(addresses).withPool(pool));
-        }
-
-        enum Behaviour {
-            failProvisioning,
-            failDeprovisioning,
-            failDnsUpdate,
-        }
-
-    }
-
 }
+
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index eda744e9ee1..3bd40670631 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
@@ -18,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
@@ -28,9 +30,12 @@ import java.time.Duration;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doThrow;
@@ -64,7 +69,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Inactive times out
         tester.advanceTime(Duration.ofMinutes(14));
-        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run();
+        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run();
         assertEquals(0, tester.nodeRepository().nodes().list(Node.State.inactive).size());
         NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty);
         assertEquals(2, dirty.size());
@@ -105,7 +110,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Inactive times out and node is moved to dirty
         tester.advanceTime(Duration.ofMinutes(14));
-        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run();
+        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run();
         NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty);
         assertEquals(2, dirty.size());
 
@@ -156,7 +161,7 @@ public class InactiveAndFailedExpirerTest {
 
         // Inactive times out and one node is moved to parked
         tester.advanceTime(Duration.ofMinutes(11)); // Trigger InactiveExpirer
-        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run();
+        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run();
         assertEquals(1, tester.nodeRepository().nodes().list(Node.State.parked).size());
     }
 
@@ -178,7 +183,7 @@ public class InactiveAndFailedExpirerTest {
         assertEquals(1, inactiveNodes.size());
 
         // See that nodes are moved to dirty immediately.
-        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run();
+        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run();
         assertEquals(0, tester.nodeRepository().nodes().list(Node.State.inactive).size());
         NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty);
         assertEquals(1, dirty.size());
@@ -202,8 +207,31 @@ public class InactiveAndFailedExpirerTest {
         // Nodes marked for deprovisioning are moved to parked
         tester.patchNodes(inactiveNodes, (node) -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant()));
         tester.advanceTime(Duration.ofMinutes(11));
-        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run();
+        new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run();
         assertEquals(2, tester.nodeRepository().nodes().list(Node.State.parked).size());
     }
 
+    @Test
+    public void inactive_config_server_expires_according_to_custom_timeout() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+        InactiveExpirer expirer = new InactiveExpirer(tester.nodeRepository(), Duration.ofHours(1),
+                                                      Map.of(NodeType.config, Duration.ofMinutes(5)),
+                                                      new TestMetric());
+        NodeList nodes = tester.makeConfigServers(3, "default", Vtag.currentVersion);
+        Supplier<Node> firstNode = () -> tester.nodeRepository().nodes().node(nodes.first().get().hostname()).get();
+        ApplicationId application = firstNode.get().allocation().get().owner();
+
+        // Retired config server is moved to inactive
+        tester.nodeRepository().nodes().retire(NodeListFilter.from(firstNode.get()), Agent.system, tester.clock().instant());
+        tester.prepareAndActivateInfraApplication(application, NodeType.config);
+        assertSame(Node.State.inactive, firstNode.get().state());
+        expirer.maintain();
+        assertSame(Node.State.inactive, firstNode.get().state());
+
+        // Config server expires
+        tester.clock().advance(Duration.ofMinutes(5));
+        expirer.maintain();
+        assertSame(Node.State.dirty, firstNode.get().state());
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
index 131c02015a1..4db1b86419b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
@@ -14,22 +14,20 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeResources.DiskSpeed;
 import com.yahoo.config.provision.NodeResources.StorageType;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.OutOfCapacityException;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
-import com.yahoo.vespa.hosted.provision.node.Address;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing;
+import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Test;
 
 import java.time.Instant;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -73,7 +71,7 @@ public class DynamicDockerProvisionTest {
 
         mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts
         prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources);
-        verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), resources, application1,
+        verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), NodeType.host, resources, application1,
                 Version.emptyVersion, HostSharing.any);
 
         // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes
@@ -99,7 +97,7 @@ public class DynamicDockerProvisionTest {
         ApplicationId application3 = ProvisioningTester.applicationId();
         mockHostProvisioner(hostProvisioner, "large", 3, application3);
         prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, resources);
-        verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), resources, application3,
+        verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), NodeType.host, resources, application3,
                 Version.emptyVersion, HostSharing.exclusive);
 
         // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes
@@ -429,7 +427,7 @@ public class DynamicDockerProvisionTest {
         doAnswer(invocation -> {
             Flavor hostFlavor = tester.nodeRepository().flavors().getFlavorOrThrow(hostFlavorName);
             List<Integer> provisionIndexes = (List<Integer>) invocation.getArguments()[0];
-            NodeResources nodeResources = (NodeResources) invocation.getArguments()[1];
+            NodeResources nodeResources = (NodeResources) invocation.getArguments()[2];
 
             return provisionIndexes.stream()
                     .map(hostIndex -> {
@@ -451,52 +449,7 @@ public class DynamicDockerProvisionTest {
                         return provisionedHost;
                     })
                     .collect(Collectors.toList());
-        }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any());
-    }
-
-    private static class MockHostProvisioner implements HostProvisioner {
-
-        private final List<Flavor> hostFlavors;
-        private final int memoryTaxGb;
-
-        public MockHostProvisioner(List<Flavor> hostFlavors, int memoryTaxGb) {
-            this.hostFlavors = List.copyOf(hostFlavors);
-            this.memoryTaxGb = memoryTaxGb;
-        }
-
-        @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
-                                                    ApplicationId applicationId, Version osVersion, HostSharing sharing) {
-            Optional<Flavor> hostFlavor = hostFlavors.stream().filter(f -> compatible(f, resources)).findFirst();
-            if (hostFlavor.isEmpty())
-                throw new OutOfCapacityException("No host flavor matches " + resources);
-            return provisionIndexes.stream()
-                                   .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor.get(), Optional.empty(),
-                                           List.of(new Address("host-" + i + "-1")), resources, osVersion))
-                                   .collect(Collectors.toList());
-        }
-
-        private boolean compatible(Flavor hostFlavor, NodeResources resources) {
-            NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb);
-
-            if (hostFlavor.resources().storageType() == NodeResources.StorageType.remote
-                && hostFlavor.resources().diskGb() >= resources.diskGb())
-                resourcesToVerify = resourcesToVerify.withDiskGb(hostFlavor.resources().diskGb());
-            if (hostFlavor.resources().bandwidthGbps() >= resources.bandwidthGbps())
-                resourcesToVerify = resourcesToVerify.withBandwidthGbps(hostFlavor.resources().bandwidthGbps());
-            return hostFlavor.resources().compatibleWith(resourcesToVerify);
-        }
-
-        @Override
-        public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
-            throw new RuntimeException("Not implemented: provision");
-        }
-
-        @Override
-        public void deprovision(Node host) {
-            throw new RuntimeException("Not implemented: deprovision");
-        }
-
+        }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any());
     }
 
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 0986f2954a7..c269b4642ea 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -33,7 +33,6 @@ import org.junit.Test;
 
 import java.time.Duration;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -1017,7 +1016,7 @@ public class ProvisioningTest {
 
     private Set<HostSpec> prepare(ApplicationId application, ProvisioningTester tester, ClusterSpec cluster, int nodeCount, int groups,
                                   boolean required, NodeResources nodeResources) {
-        if (nodeCount == 0) return Collections.emptySet(); // this is a shady practice
+        if (nodeCount == 0) return Set.of(); // this is a shady practice
         return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, nodeResources));
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 97baddf93fa..eefbd03ce4e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -227,7 +227,10 @@ public class ProvisioningTester {
     }
 
     public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build();
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()))
+                                         .vespaVersion(version)
+                                         .stateful(nodeType == NodeType.config || nodeType == NodeType.controller)
+                                         .build();
         Capacity capacity = Capacity.fromRequiredNodeType(nodeType);
         List<HostSpec> hostSpecs = prepare(application, cluster, capacity);
         activate(application, hostSpecs);
@@ -510,17 +513,18 @@ public class ProvisioningTester {
                                      index -> UUID.randomUUID().toString());
     }
 
-    /** Creates a set of virtual nodes on a single parent host */
-    public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname,
-                                        Function<Integer, String> nodeNamer) {
+    /** Create one or more child nodes on given parent host */
+    public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, NodeType nodeType,
+                                        String parentHostname, Function<Integer, String> nodeNamer) {
+        if (nodeType.isHost()) throw new IllegalArgumentException("Non-child node type: " + nodeType);
         List<Node> nodes = new ArrayList<>(count);
         for (int i = startIndex; i < count + startIndex; i++) {
             String hostname = nodeNamer.apply(i);
             IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), Set.of());
-
-            Node.Builder builder = Node.create("node-id", ipConfig, hostname, new Flavor(resources), NodeType.tenant);
-            builder.parentHostname(parentHostname);
-            nodes.add(builder.build());
+            Node node = Node.create("node-id", ipConfig, hostname, new Flavor(resources), nodeType)
+                            .parentHostname(parentHostname)
+                            .build();
+            nodes.add(node);
         }
         nodes = nodeRepository.nodes().addNodes(nodes, Agent.system);
         nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName());
@@ -528,6 +532,12 @@ public class ProvisioningTester {
         return nodes;
     }
 
+    /** Create one or more child nodes on given parent host */
+    public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname,
+                                        Function<Integer, String> nodeNamer) {
+        return makeReadyChildren(count, startIndex, resources, NodeType.tenant, parentHostname, nodeNamer);
+    }
+
     public void activateTenantHosts() {
         prepareAndActivateInfraApplication(applicationId(), NodeType.host);
     }