Merge pull request #21063 from vespa-engine/freva/do-not-allocate-to-suspended

Do not allocate nodes on suspended hosts

22 files changed, 110 insertions, 81 deletions

diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml
index a44f1845d4e..08521216736 100644
--- a/athenz-identity-provider-service/pom.xml
+++ b/athenz-identity-provider-service/pom.xml
@@ -93,6 +93,12 @@
             <scope>test</scope>
         </dependency>
         <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>orchestrator</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <scope>test</scope>
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 7f54fff5c70..a387bc28aa4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages;
 import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
+import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.time.Clock;
 import java.util.List;
@@ -59,6 +60,7 @@ public class NodeRepository extends AbstractComponent {
     private final LoadBalancers loadBalancers;
     private final FlagSource flagSource;
     private final MetricsDb metricsDb;
+    private final Orchestrator orchestrator;
     private final int spareCount;
 
     /**
@@ -72,7 +74,8 @@ public class NodeRepository extends AbstractComponent {
                           Curator curator,
                           Zone zone,
                           FlagSource flagSource,
-                          MetricsDb metricsDb) {
+                          MetricsDb metricsDb,
+                          Orchestrator orchestrator) {
         this(flavors,
              provisionServiceProvider,
              curator,
@@ -83,6 +86,7 @@ public class NodeRepository extends AbstractComponent {
              Optional.of(config.tenantContainerImage()).filter(s -> !s.isEmpty()).map(DockerImage::fromString),
              flagSource,
              metricsDb,
+             orchestrator,
              config.useCuratorClientCache(),
              zone.environment().isProduction() && !zone.getCloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0,
              config.nodeCacheSize());
@@ -102,6 +106,7 @@ public class NodeRepository extends AbstractComponent {
                           Optional<DockerImage> tenantContainerImage,
                           FlagSource flagSource,
                           MetricsDb metricsDb,
+                          Orchestrator orchestrator,
                           boolean useCuratorClientCache,
                           int spareCount,
                           long nodeCacheSize) {
@@ -113,7 +118,7 @@ public class NodeRepository extends AbstractComponent {
         this.db = new CuratorDatabaseClient(flavors, curator, clock, useCuratorClientCache, nodeCacheSize);
         this.zone = zone;
         this.clock = clock;
-        this.nodes = new Nodes(db, zone, clock);
+        this.nodes = new Nodes(db, zone, clock, orchestrator);
         this.flavors = flavors;
         this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator();
         this.nameResolver = nameResolver;
@@ -127,6 +132,7 @@ public class NodeRepository extends AbstractComponent {
         this.loadBalancers = new LoadBalancers(db);
         this.flagSource = flagSource;
         this.metricsDb = metricsDb;
+        this.orchestrator = orchestrator;
         this.spareCount = spareCount;
         nodes.rewrite();
     }
@@ -172,6 +178,8 @@ public class NodeRepository extends AbstractComponent {
 
     public MetricsDb metricsDb() { return metricsDb; }
 
+    public Orchestrator orchestrator() { return orchestrator; }
+
     public NodeRepoStats computeStats() { return NodeRepoStats.computeOver(this); }
 
     /** Returns the time keeper of this system */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 341ab1f785c..e6476cd7373 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.NodesAndHosts;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.node.IP;
-import com.yahoo.vespa.hosted.provision.node.Nodes;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing;
@@ -205,7 +204,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
 
     private Map<String, Node> findSharedHosts(NodeList nodeList) {
         return nodeList.stream()
-                .filter(node -> Nodes.canAllocateTenantNodeTo(node, true))
+                .filter(node -> nodeRepository().nodes().canAllocateTenantNodeTo(node, true))
                 .filter(node -> node.reservedTo().isEmpty())
                 .filter(node -> node.exclusiveToApplicationId().isEmpty())
                 .collect(Collectors.toMap(Node::hostname, Function.identity()));
@@ -298,7 +297,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
         int wantedGroups = 1;
 
         NodePrioritizer prioritizer = new NodePrioritizer(nodesAndHosts, applicationId, clusterSpec, nodeSpec, wantedGroups,
-                true, nodeRepository().nameResolver(), nodeRepository().resourcesCalculator(),
+                true, nodeRepository().nameResolver(), nodeRepository().nodes(), nodeRepository().resourcesCalculator(),
                 nodeRepository().spareCount());
         List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of());
         MutableInteger index = new MutableInteger(0);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 7cf806eb16c..636884cef0a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -22,7 +22,6 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.ClusterId;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.persistence.CacheStats;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
@@ -47,20 +46,17 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
 
     private final Set<Pair<Metric.Context, String>> nonZeroMetrics = new HashSet<>();
     private final Metric metric;
-    private final Orchestrator orchestrator;
     private final ServiceMonitor serviceMonitor;
     private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>();
     private final Supplier<Integer> pendingRedeploymentsSupplier;
 
     MetricsReporter(NodeRepository nodeRepository,
                     Metric metric,
-                    Orchestrator orchestrator,
                     ServiceMonitor serviceMonitor,
                     Supplier<Integer> pendingRedeploymentsSupplier,
                     Duration interval) {
         super(nodeRepository, interval, metric);
         this.metric = metric;
-        this.orchestrator = orchestrator;
         this.serviceMonitor = serviceMonitor;
         this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
     }
@@ -212,7 +208,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
 
         serviceModel.getApplication(hostname)
                 .map(ApplicationInstance::reference)
-                .map(reference -> orchestrator.getHostInfo(reference, hostname))
+                .map(reference -> nodeRepository().orchestrator().getHostInfo(reference, hostname))
                 .ifPresent(info -> {
                     int suspended = info.status().isSuspended() ? 1 : 0;
                     metric.set("suspended", suspended, context);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 71e6fb8521e..a1916d7dc20 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -7,7 +7,6 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TransientException;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.transaction.Mutex;
-import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeMutex;
@@ -15,8 +14,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException;
-import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.yolean.Exceptions;
 
@@ -54,19 +51,16 @@ public class NodeFailer extends NodeRepositoryMaintainer {
     private final Deployer deployer;
     private final Duration downTimeLimit;
     private final Duration suspendedDownTimeLimit;
-    private final Orchestrator orchestrator;
     private final ThrottlePolicy throttlePolicy;
     private final Metric metric;
 
     public NodeFailer(Deployer deployer, NodeRepository nodeRepository,
-                      Duration downTimeLimit, Duration interval, Orchestrator orchestrator,
-                      ThrottlePolicy throttlePolicy, Metric metric) {
+                      Duration downTimeLimit, Duration interval, ThrottlePolicy throttlePolicy, Metric metric) {
         // check ping status every interval, but at least twice as often as the down time limit
         super(nodeRepository, min(downTimeLimit.dividedBy(2), interval), metric);
         this.deployer = deployer;
         this.downTimeLimit = downTimeLimit;
         this.suspendedDownTimeLimit = downTimeLimit.multipliedBy(4); // Allow more downtime when a node is suspended
-        this.orchestrator = orchestrator;
         this.throttlePolicy = throttlePolicy;
         this.metric = metric;
     }
@@ -160,7 +154,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
         NodeList activeNodes = nodeRepository().nodes().list(Node.State.active);
 
         for (Node node : activeNodes) {
-            Instant graceTimeStart = clock().instant().minus(suspended(node) ? suspendedDownTimeLimit : downTimeLimit);
+            Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit);
             if (node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node)) {
                 // Allow a grace period after node re-activation
                 if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart))
@@ -201,7 +195,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
 
     private boolean applicationSuspended(Node node) {
         try {
-            return orchestrator.getApplicationInstanceStatus(node.allocation().get().owner())
+            return nodeRepository().orchestrator().getApplicationInstanceStatus(node.allocation().get().owner())
                    == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
         } catch (ApplicationIdNotFoundException e) {
             // Treat it as not suspended and allow to fail the node anyway
@@ -209,23 +203,14 @@ public class NodeFailer extends NodeRepositoryMaintainer {
         }
     }
 
-    private boolean suspended(Node node) {
-        try {
-            return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
-        } catch (HostNameNotFoundException e) {
-            // Treat it as not suspended
-            return false;
-        }
-    }
-
     /** Is the node and all active children suspended? */
     private boolean allSuspended(Node node, NodeList activeNodes) {
-        if (!suspended(node)) return false;
+        if (!nodeRepository().nodes().suspended(node)) return false;
         if (node.parentHostname().isPresent()) return true; // optimization
         return activeNodes.stream()
                 .filter(childNode -> childNode.parentHostname().isPresent() &&
                         childNode.parentHostname().get().equals(node.hostname()))
-                .allMatch(this::suspended);
+                .allMatch(nodeRepository().nodes()::suspended);
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 2f200032492..15decde0d7c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.autoscale.MetricsFetcher;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.time.Duration;
@@ -35,7 +34,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     @Inject
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
-                                     Zone zone, Orchestrator orchestrator, Metric metric,
+                                     Zone zone, Metric metric,
                                      ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource,
                                      MetricsFetcher metricsFetcher) {
         DefaultTimes defaults = new DefaultTimes(zone, deployer);
@@ -46,11 +45,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(periodicApplicationMaintainer);
         maintainers.add(infrastructureProvisioner);
 
-        maintainers.add(new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, orchestrator, defaults.throttlePolicy, metric));
+        maintainers.add(new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, defaults.throttlePolicy, metric));
         maintainers.add(new NodeHealthTracker(hostLivenessTracker, serviceMonitor, nodeRepository, defaults.nodeFailureStatusUpdateInterval, metric));
         maintainers.add(new ExpeditedChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.expeditedChangeRedeployInterval));
         maintainers.add(new ReservationExpirer(nodeRepository, defaults.reservationExpiry, metric));
-        maintainers.add(new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry));
+        maintainers.add(new RetiredExpirer(nodeRepository, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry));
         maintainers.add(new InactiveExpirer(nodeRepository, defaults.inactiveExpiry, Map.of(NodeType.config, defaults.inactiveConfigServerExpiry,
                                                                                             NodeType.controller, defaults.inactiveControllerExpiry),
                                             metric));
@@ -58,7 +57,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         maintainers.add(new DirtyExpirer(nodeRepository, defaults.dirtyExpiry, metric));
         maintainers.add(new ProvisionedExpirer(nodeRepository, defaults.provisionedExpiry, metric));
         maintainers.add(new NodeRebooter(nodeRepository, flagSource, metric));
-        maintainers.add(new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval));
+        maintainers.add(new MetricsReporter(nodeRepository, metric, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval));
         maintainers.add(new SpareCapacityMaintainer(deployer, nodeRepository, metric, defaults.spareCapacityMaintenanceInterval));
         maintainers.add(new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval, metric));
         maintainers.add(new Rebalancer(deployer, nodeRepository, metric, defaults.rebalancerInterval));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
index 55c225c3dad..73c9a1ab55a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
@@ -31,11 +30,9 @@ public class RetiredExpirer extends NodeRepositoryMaintainer {
 
     private final Deployer deployer;
     private final Metric metric;
-    private final Orchestrator orchestrator;
     private final Duration retiredExpiry;
 
     public RetiredExpirer(NodeRepository nodeRepository,
-                          Orchestrator orchestrator,
                           Deployer deployer,
                           Metric metric,
                           Duration maintenanceInterval,
@@ -43,7 +40,6 @@ public class RetiredExpirer extends NodeRepositoryMaintainer {
         super(nodeRepository, maintenanceInterval, metric);
         this.deployer = deployer;
         this.metric = metric;
-        this.orchestrator = orchestrator;
         this.retiredExpiry = retiredExpiry;
     }
 
@@ -126,7 +122,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer {
         }
 
         try {
-            orchestrator.acquirePermissionToRemove(new HostName(node.hostname()));
+            nodeRepository().orchestrator().acquirePermissionToRemove(new HostName(node.hostname()));
             log.info("Node " + node + " has been granted permission to be removed");
             return true;
         } catch (UncheckedTimeoutException e) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
index 4c96e58d954..57a3b436e37 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -18,6 +19,8 @@ import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -53,14 +56,16 @@ public class Nodes {
 
     private static final Logger log = Logger.getLogger(Nodes.class.getName());
 
+    private final CuratorDatabaseClient db;
     private final Zone zone;
     private final Clock clock;
-    private final CuratorDatabaseClient db;
+    private final Orchestrator orchestrator;
 
-    public Nodes(CuratorDatabaseClient db, Zone zone, Clock clock) {
+    public Nodes(CuratorDatabaseClient db, Zone zone, Clock clock, Orchestrator orchestrator) {
         this.zone = zone;
         this.clock = clock;
         this.db = db;
+        this.orchestrator = orchestrator;
     }
 
     /** Read and write all nodes to make sure they are stored in the latest version of the serialized format */
@@ -728,10 +733,11 @@ public class Nodes {
         return canAllocateTenantNodeTo(host, zone.getCloud().dynamicProvisioning());
     }
 
-    public static boolean canAllocateTenantNodeTo(Node host, boolean dynamicProvisioning) {
+    public boolean canAllocateTenantNodeTo(Node host, boolean dynamicProvisioning) {
         if ( ! host.type().canRun(NodeType.tenant)) return false;
         if (host.status().wantToRetire()) return false;
         if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false;
+        if (suspended(host)) return false;
 
         if (dynamicProvisioning)
             return EnumSet.of(Node.State.active, Node.State.ready, Node.State.provisioned).contains(host.state());
@@ -739,6 +745,15 @@ public class Nodes {
             return host.state() == Node.State.active;
     }
 
+    public boolean suspended(Node node) {
+        try {
+            return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
+        } catch (HostNameNotFoundException e) {
+            // Treat it as not suspended
+            return false;
+        }
+    }
+
     /** Create a lock which provides exclusive rights to making changes to the given application */
     // TODO: Move to Applications
     public Mutex lock(ApplicationId application) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 2d93763c631..ae65f367684 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -149,6 +149,7 @@ public class GroupPreparer {
                                                           wantedGroups,
                                                           nodeRepository.zone().getCloud().dynamicProvisioning(),
                                                           nodeRepository.nameResolver(),
+                                                          nodeRepository.nodes(),
                                                           nodeRepository.resourcesCalculator(),
                                                           nodeRepository.spareCount());
         allocation.offer(prioritizer.collect(surplusActiveNodes));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
index 85a43e38e07..fe4eb5d68c9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
@@ -30,13 +30,14 @@ import java.util.stream.Collectors;
  */
 public class NodePrioritizer {
 
-    private final List<NodeCandidate> nodes = new ArrayList<>();
+    private final List<NodeCandidate> candidates = new ArrayList<>();
     private final NodesAndHosts<LockedNodeList> allNodesAndHosts;
     private final HostCapacity capacity;
     private final NodeSpec requestedNodes;
     private final ApplicationId application;
     private final ClusterSpec clusterSpec;
     private final NameResolver nameResolver;
+    private final Nodes nodes;
     private final boolean dynamicProvisioning;
     /** Whether node specification allows new nodes to be allocated. */
     private final boolean canAllocateNew;
@@ -46,7 +47,7 @@ public class NodePrioritizer {
     private final Set<Node> spareHosts;
 
     public NodePrioritizer(NodesAndHosts<LockedNodeList> allNodesAndHosts, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec,
-                           int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver,
+                           int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes,
                            HostResourcesCalculator hostResourcesCalculator, int spareCount) {
         this.allNodesAndHosts = allNodesAndHosts;
         this.capacity = new HostCapacity(this.allNodesAndHosts, hostResourcesCalculator);
@@ -58,6 +59,7 @@ public class NodePrioritizer {
                 capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodesAndHosts.nodes().asList()) :
                 capacity.findSpareHosts(this.allNodesAndHosts.nodes().asList(), spareCount);
         this.nameResolver = nameResolver;
+        this.nodes = nodes;
 
         NodeList nodesInCluster = this.allNodesAndHosts.nodes().owner(application).type(clusterSpec.type()).cluster(clusterSpec.id());
         NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired();
@@ -95,12 +97,12 @@ public class NodePrioritizer {
     /** Returns the list of nodes sorted by {@link NodeCandidate#compareTo(NodeCandidate)} */
     private List<NodeCandidate> prioritize() {
         // Group candidates by their switch hostname
-        Map<String, List<NodeCandidate>> candidatesBySwitch = this.nodes.stream()
+        Map<String, List<NodeCandidate>> candidatesBySwitch = this.candidates.stream()
                 .collect(Collectors.groupingBy(candidate -> candidate.parent.orElseGet(candidate::toNode)
                                                                             .switchHostname()
                                                                             .orElse("")));
         // Mark lower priority nodes on shared switch as non-exclusive
-        List<NodeCandidate> nodes = new ArrayList<>(this.nodes.size());
+        List<NodeCandidate> nodes = new ArrayList<>(this.candidates.size());
         for (var clusterSwitch : candidatesBySwitch.keySet()) {
             List<NodeCandidate> switchCandidates = candidatesBySwitch.get(clusterSwitch);
             if (clusterSwitch.isEmpty()) {
@@ -126,7 +128,7 @@ public class NodePrioritizer {
         for (Node node : surplusNodes) {
             NodeCandidate candidate = candidateFrom(node, true);
             if (!candidate.violatesSpares || canAllocateToSpareHosts) {
-                nodes.add(candidate);
+                candidates.add(candidate);
             }
         }
     }
@@ -136,7 +138,7 @@ public class NodePrioritizer {
         if ( !canAllocateNew) return;
 
         for (Node host : allNodesAndHosts.nodes()) {
-            if ( ! Nodes.canAllocateTenantNodeTo(host, dynamicProvisioning)) continue;
+            if ( ! nodes.canAllocateTenantNodeTo(host, dynamicProvisioning)) continue;
             if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue;
             if (host.reservedTo().isPresent() && application.instance().isTester()) continue;
             if (host.exclusiveToApplicationId().isPresent()) continue; // Never allocate new nodes to exclusive hosts
@@ -144,7 +146,7 @@ public class NodePrioritizer {
             if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue;
             if ( ! capacity.hasCapacity(host, requestedNodes.resources().get())) continue;
             if ( ! allNodesAndHosts.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue;
-            nodes.add(NodeCandidate.createNewChild(requestedNodes.resources().get(),
+            candidates.add(NodeCandidate.createNewChild(requestedNodes.resources().get(),
                                                    capacity.availableCapacityOf(host),
                                                    host,
                                                    spareHosts.contains(host),
@@ -164,7 +166,7 @@ public class NodePrioritizer {
                 .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id()))
                 .filter(node -> node.state() == Node.State.active || canStillAllocate(node))
                 .map(node -> candidateFrom(node, false))
-                .forEach(nodes::add);
+                .forEach(candidates::add);
     }
 
     /** Add nodes already provisioned, but not allocated to any application */
@@ -174,7 +176,7 @@ public class NodePrioritizer {
                 .filter(node -> node.state() == Node.State.ready)
                 .map(node -> candidateFrom(node, false))
                 .filter(n -> !n.violatesSpares || canAllocateToSpareHosts)
-                .forEach(nodes::add);
+                .forEach(candidates::add);
     }
 
     /** Create a candidate from given pre-existing node */
@@ -218,7 +220,7 @@ public class NodePrioritizer {
     private boolean canStillAllocate(Node node) {
         if (node.type() != NodeType.tenant || node.parentHostname().isEmpty()) return true;
         Optional<Node> parent = allNodesAndHosts.parentOf(node);
-        return parent.isPresent() ? Nodes.canAllocateTenantNodeTo(parent.get(), dynamicProvisioning) : null;
+        return parent.isPresent() && nodes.canAllocateTenantNodeTo(parent.get(), dynamicProvisioning);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 1a2d5294aa5..ff406efdc39 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -70,6 +70,7 @@ public class MockNodeRepository extends NodeRepository {
               Optional.empty(),
               new InMemoryFlagSource(),
               new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))),
+              new OrchestratorMock(),
               true,
               0, 1000);
         this.flavors = flavors;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
index b391292884f..65a57ebd53e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 
 import java.util.List;
 import java.util.Optional;
@@ -47,6 +48,7 @@ public class NodeRepositoryTester {
                                             Optional.empty(),
                                             new InMemoryFlagSource(),
                                             new MemoryMetricsDb(clock),
+                                            new OrchestratorMock(),
                                             true,
                                             0, 1000);
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index e06bdba90fb..68aea0e9056 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -31,6 +31,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -73,6 +74,7 @@ public class CapacityCheckerTester {
                                             Optional.empty(),
                                             new InMemoryFlagSource(),
                                             new MemoryMetricsDb(clock),
+                                            new OrchestratorMock(),
                                             true,
                                             0, 1000);
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index 5e9137f8713..5211b855fff 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -126,7 +126,7 @@ public class InactiveAndFailedExpirerTest {
         );
         Orchestrator orchestrator = mock(Orchestrator.class);
         doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any());
-        new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(),
+        new RetiredExpirer(tester.nodeRepository(), deployer, new TestMetric(),
                            Duration.ofDays(30), Duration.ofMinutes(10)).run();
         assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).size());
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 8c649243d61..98d3ffa92f8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -339,7 +339,6 @@ public class MetricsReporterTest {
     private MetricsReporter metricsReporter(TestMetric metric, ProvisioningTester tester) {
         return new MetricsReporter(tester.nodeRepository(),
                                    metric,
-                                   tester.orchestrator(),
                                    serviceMonitor,
                                    () -> 42,
                                    LONG_INTERVAL);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 0ea7011a930..f67e9cd8345 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -26,10 +26,8 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
-import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 import com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub;
 import com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker;
-import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -66,14 +64,11 @@ public class NodeFailTester {
     public MockDeployer deployer;
     public TestMetric metric;
     private final TestHostLivenessTracker hostLivenessTracker;
-    private final Orchestrator orchestrator;
     private final NodeRepositoryProvisioner provisioner;
     private final Curator curator;
 
     private NodeFailTester() {
-        orchestrator = new OrchestratorMock();
-        tester = new ProvisioningTester.Builder().orchestrator(orchestrator)
-                                                 .flavors(hostFlavors.getFlavors())
+        tester = new ProvisioningTester.Builder().flavors(hostFlavors.getFlavors())
                                                  .spareCount(1).build();
         clock = tester.clock();
         curator = tester.getCurator();
@@ -215,7 +210,7 @@ public class NodeFailTester {
 
     public void suspend(ApplicationId app) {
         try {
-            orchestrator.suspend(app);
+            nodeRepository.orchestrator().suspend(app);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -223,7 +218,7 @@ public class NodeFailTester {
 
     public void suspend(String hostName) {
         try {
-            orchestrator.suspend(new HostName(hostName));
+            nodeRepository.orchestrator().suspend(new HostName(hostName));
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -231,7 +226,7 @@ public class NodeFailTester {
 
     public NodeFailer createFailer() {
         return new NodeFailer(deployer, nodeRepository, downtimeLimitOneHour,
-                              Duration.ofMinutes(5), orchestrator, NodeFailer.ThrottlePolicy.hosted, metric);
+                              Duration.ofMinutes(5), NodeFailer.ThrottlePolicy.hosted, metric);
     }
 
     public NodeHealthTracker createUpdater() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 4575f7b4355..1237ede5345 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.testutils.MockDuperModel;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
 import org.junit.Before;
 import org.junit.Test;
@@ -60,11 +61,11 @@ public class RetiredExpirerTest {
     private final NodeResources hostResources = new NodeResources(64, 128, 2000, 10);
     private final NodeResources nodeResources = new NodeResources(2, 8, 50, 1);
 
-    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
+    private final Orchestrator orchestrator = mock(Orchestrator.class);
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().orchestrator(orchestrator).build();
     private final ManualClock clock = tester.clock();
     private final NodeRepository nodeRepository = tester.nodeRepository();
     private final NodeRepositoryProvisioner provisioner = tester.provisioner();
-    private final Orchestrator orchestrator = mock(Orchestrator.class);
 
     private static final Duration RETIRED_EXPIRATION = Duration.ofHours(12);
 
@@ -72,6 +73,7 @@ public class RetiredExpirerTest {
     public void setup() throws OrchestrationException {
         // By default, orchestrator should deny all request for suspension so we can test expiration
         doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any());
+        when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS);
     }
 
     @Test
@@ -269,7 +271,6 @@ public class RetiredExpirerTest {
 
     private RetiredExpirer createRetiredExpirer(Deployer deployer) {
         return new RetiredExpirer(nodeRepository,
-                                  orchestrator,
                                   deployer,
                                   new TestMetric(),
                                   Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
index 373bfe20162..2fa18681ece 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java
@@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvid
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -267,6 +268,7 @@ public class SpareCapacityMaintainerTest {
                                                 Optional.empty(),
                                                 new InMemoryFlagSource(),
                                                 new MemoryMetricsDb(clock),
+                                                new OrchestratorMock(),
                                                 true,
                                                 1, 1000);
             deployer = new MockDeployer(nodeRepository);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
index df9e8efde93..583ccdda656 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
@@ -17,6 +17,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.config.provisioning.FlavorsConfig;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.Node.State;
@@ -267,6 +268,35 @@ public class DynamicAllocationTest {
     }
 
     @Test
+    public void does_not_allocate_to_suspended_hosts() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
+        tester.makeReadyNodes(4, "host-small", NodeType.host, 32);
+        tester.activateTenantHosts();
+
+        HostName randomHost = new HostName(tester.nodeRepository().nodes().list(State.active).first().get().hostname());
+        tester.orchestrator().suspend(randomHost);
+
+        ApplicationId application1 = ProvisioningTester.applicationId();
+        ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1");
+        NodeResources flavor = new NodeResources(1, 4, 100, 1);
+
+        try {
+            tester.prepare(application1, clusterSpec, 4, 1, flavor);
+            fail("Should not be able to deploy 4 nodes on 4 hosts because 1 is suspended");
+        } catch (OutOfCapacityException ignored) { }
+
+        // Resume the host, the deployment goes through
+        tester.orchestrator().resume(randomHost);
+        tester.activate(application1, tester.prepare(application1, clusterSpec, 4, 1, flavor));
+        Set<String> hostnames = tester.getNodes(application1, State.active).hostnames();
+
+        // Verify that previously allocated nodes are not affected by host suspension
+        tester.orchestrator().suspend(randomHost);
+        tester.activate(application1, tester.prepare(application1, clusterSpec, 4, 1, flavor));
+        assertEquals(hostnames, tester.getNodes(application1, State.active).hostnames());
+    }
+
+    @Test
     public void non_prod_zones_do_not_have_spares() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
         tester.makeReadyNodes(3, "host-small", NodeType.host, 32);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
index 482b798d736..3db8a71e4a7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
@@ -237,7 +237,6 @@ public class MultigroupProvisioningTest {
                                                       new MockDeployer.ApplicationContext(application1, cluster(), 
                                                                                           Capacity.from(new ClusterResources(8, 1, large), false, true))));
         new RetiredExpirer(tester.nodeRepository(),
-                           tester.orchestrator(),
                            deployer,
                            new TestMetric(),
                            Duration.ofDays(30),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
index 4a9707f52f8..12f1abf0cf5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
@@ -98,7 +98,6 @@ public class NodeTypeProvisioningTest {
                                                                                                               clusterSpec,
                                                                                                               capacity)));
         RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(),
-                                                            tester.orchestrator(),
                                                             deployer,
                                                             new TestMetric(),
                                                             Duration.ofDays(30),
@@ -166,7 +165,6 @@ public class NodeTypeProvisioningTest {
                                                  Collections.singletonMap(application,
                                                                           new MockDeployer.ApplicationContext(application, clusterSpec, capacity)));
         RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(),
-                                                            tester.orchestrator(),
                                                             deployer,
                                                             new TestMetric(),
                                                             Duration.ofDays(30),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index c478840780f..b781f397f70 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -43,6 +43,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
 import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
 
@@ -80,7 +81,6 @@ public class ProvisioningTester {
     private final NodeFlavors nodeFlavors;
     private final ManualClock clock;
     private final NodeRepository nodeRepository;
-    private final Orchestrator orchestrator;
     private final NodeRepositoryProvisioner provisioner;
     private final CapacityPolicies capacityPolicies;
     private final ProvisionLogger provisionLogger;
@@ -114,10 +114,10 @@ public class ProvisioningTester {
                                                  Optional.empty(),
                                                  flagSource,
                                                  new MemoryMetricsDb(clock),
+                                                 orchestrator,
                                                  true,
                                                  spareCount,
                                                  1000);
-        this.orchestrator = orchestrator;
         this.provisioner = new NodeRepositoryProvisioner(nodeRepository,
                                                          zone,
                                                          provisionServiceProvider,
@@ -144,7 +144,7 @@ public class ProvisioningTester {
 
     public void advanceTime(TemporalAmount duration) { clock.advance(duration); }
     public NodeRepository nodeRepository() { return nodeRepository; }
-    public Orchestrator orchestrator() { return orchestrator; }
+    public Orchestrator orchestrator() { return nodeRepository.orchestrator(); }
     public ManualClock clock() { return clock; }
     public NodeRepositoryProvisioner provisioner() { return provisioner; }
     public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; }
@@ -689,20 +689,13 @@ public class ProvisioningTester {
         }
 
         public ProvisioningTester build() {
-            Orchestrator orchestrator = Optional.ofNullable(this.orchestrator)
-                    .orElseGet(() -> {
-                        Orchestrator orch = mock(Orchestrator.class);
-                        doThrow(new RuntimeException()).when(orch).acquirePermissionToRemove(any());
-                        return orch;
-                    });
-
             return new ProvisioningTester(Optional.ofNullable(curator).orElseGet(MockCurator::new),
                                           new NodeFlavors(Optional.ofNullable(flavorsConfig).orElseGet(ProvisioningTester::createConfig)),
                                           resourcesCalculator,
                                           Optional.ofNullable(zone).orElseGet(Zone::defaultZone),
                                           Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()),
                                           defaultImage,
-                                          orchestrator,
+                                          Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new),
                                           hostProvisioner,
                                           new LoadBalancerServiceMock(),
                                           Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new),