Merge pull request #9930 from vespa-engine/freva/dynamic-provisioning-internal-redeploy

Dynamic provisioning internal redeploy

10 files changed, 99 insertions, 118 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index cd0519ff115..d490b1b49e9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -7,11 +7,13 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployment;
+import com.yahoo.config.provision.TransientException;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.state.StateMonitor;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.server.rpc.RpcServer;
 import com.yahoo.vespa.config.server.version.VersionState;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -241,10 +243,13 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         }
 
         for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
+            ApplicationId app = f.getKey();
             try {
                 f.getValue().get();
+            } catch (TransientException e) {
+                log.log(LogLevel.INFO, "Redeploying " + app +
+                        " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e));
             } catch (ExecutionException e) {
-                ApplicationId app = f.getKey();
                 log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e);
                 failedDeployments.add(app);
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index 96c8fe21959..c1a05a3c32d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -5,10 +5,12 @@ import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
+import com.yahoo.config.provision.TransientException;
 import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -88,6 +90,8 @@ public abstract class ApplicationMaintainer extends Maintainer {
             if ( ! deployment.isPresent()) return; // this will be done at another config server
             log.log(LogLevel.DEBUG, this.getClass().getSimpleName() + " deploying " + application);
             deployment.get().activate();
+        } catch (TransientException e) {
+            log.log(LogLevel.INFO, "Failed to redeploy " + application + " with a transient error: " + Exceptions.toMessageString(e));
         } catch (RuntimeException e) {
             log.log(LogLevel.WARNING, "Exception on maintenance redeploy", e);
         } finally {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index b7e8395cc92..a7b750c4e46 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -5,7 +5,9 @@ import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.HostLivenessTracker;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TransientException;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
@@ -21,6 +23,7 @@ import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -368,8 +371,11 @@ public class NodeFailer extends Maintainer {
             try {
                 deployment.get().activate();
                 return true;
-            }
-            catch (RuntimeException e) {
+            } catch (TransientException e) {
+                log.log(LogLevel.INFO, "Failed to redeploy " + node.allocation().get().owner() +
+                        " with a transient error, will be retried by application maintainer: " + Exceptions.toMessageString(e));
+                return true;
+            } catch (RuntimeException e) {
                 // The expected reason for deployment to fail here is that there is no capacity available to redeploy.
                 // In that case we should leave the node in the active state to avoid failing additional nodes.
                 nodeRepository().reactivate(node.hostname(), Agent.system,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 52e7a28acc8..b9b1200d473 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -65,7 +65,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
 
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric);
         periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, defaults.redeployMaintainerInterval, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval));
-        operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, clock, durationFromEnv("operator_change_redeploy_interval").orElse(defaults.operatorChangeRedeployInterval));
+        operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, durationFromEnv("operator_change_redeploy_interval").orElse(defaults.operatorChangeRedeployInterval));
         reservationExpirer = new ReservationExpirer(nodeRepository, clock, durationFromEnv("reservation_expiry").orElse(defaults.reservationExpiry));
         retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, clock, durationFromEnv("retired_interval").orElse(defaults.retiredInterval), durationFromEnv("retired_expiry").orElse(defaults.retiredExpiry));
         inactiveExpirer = new InactiveExpirer(nodeRepository, clock, durationFromEnv("inactive_expiry").orElse(defaults.inactiveExpiry));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
index 46571fd0deb..ab7a565688e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
@@ -7,12 +7,12 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
-import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.History;
 
-import java.time.Clock;
 import java.time.Duration;
-import java.time.Instant;
 import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -28,31 +28,25 @@ import java.util.stream.Collectors;
  * @author bratseth
  */
 public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer {
-
-    private final Clock clock;
-    
-    private Instant previousRun;
     
-    OperatorChangeApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Clock clock, Duration interval) {
+    OperatorChangeApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval) {
         super(deployer, nodeRepository, interval);
-        this.clock = clock;
-        previousRun = clock.instant(); // Changes before this will be caught by the first PeriodicApplicationMaintainer run
     }
 
     @Override
     protected Set<ApplicationId> applicationsNeedingMaintenance() {
-        Instant windowEnd = clock.instant();
-        Instant windowStart = previousRun;
-        previousRun = windowEnd;
-        return nodeRepository().getNodes(NodeType.tenant).stream()
-                               .filter(node -> hasManualStateChangeSince(windowStart, node))
-                               .flatMap(node -> node.allocation().map(Allocation::owner).stream())
-                               .collect(Collectors.toCollection(LinkedHashSet::new));
-    }
-
-    private boolean hasManualStateChangeSince(Instant instant, Node node) {
-        return node.history().events().stream()
-                .anyMatch(event -> event.agent() == Agent.operator && event.at().isAfter(instant));
+        Map<ApplicationId, List<Node>> nodesByApplication = nodeRepository().getNodes(NodeType.tenant).stream()
+                .filter(node -> node.allocation().isPresent())
+                .collect(Collectors.groupingBy(node -> node.allocation().get().owner(), Collectors.toList()));
+
+        return nodesByApplication.entrySet().stream()
+                .filter(entry -> entry.getValue().stream()
+                        .flatMap(node -> node.history().events().stream())
+                        .filter(event -> event.agent() == Agent.operator)
+                        .map(History.Event::at)
+                        .anyMatch(getLastDeployTime(entry.getKey())::isBefore))
+                .map(Map.Entry::getKey)
+                .collect(Collectors.toCollection(LinkedHashSet::new));
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index 174591b0836..6ab85e76ba2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -40,21 +39,24 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
 
     @Override
     protected boolean canDeployNow(ApplicationId application) {
-        // Don't deploy if a regular deploy just happened
-        return getLastDeployTime(application).isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments));
+        return deployer().lastDeployTime(application)
+                // Don't deploy if a regular deploy just happened
+                .map(lastDeployTime -> lastDeployTime.isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments)))
+                // We only know last deploy time for applications that were deployed on this config server,
+                // the rest will be deployed on another config server
+                .orElse(false);
     }
 
     // Returns the applications that need to be redeployed by this config server at this point in time.
     @Override
     protected Set<ApplicationId> applicationsNeedingMaintenance() {
-        if (waitInitially()) return Collections.emptySet();
+        if (waitInitially()) return Set.of();
 
         // Collect all deployment times before sorting as deployments may happen while we build the set, breaking
         // the comparable contract. Stale times are fine as the time is rechecked in ApplicationMaintainer#deployWithLock
         Map<ApplicationId, Instant> deploymentTimes = nodesNeedingMaintenance().stream()
                                                                                .map(node -> node.allocation().get().owner())
                                                                                .distinct()
-                                                                               .filter(this::shouldBeDeployedOnThisServer)
                                                                                .filter(this::canDeployNow)
                                                                                .collect(Collectors.toMap(Function.identity(), this::getLastDeployTime));
 
@@ -64,12 +66,6 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
                               .collect(Collectors.toCollection(LinkedHashSet::new));
     }
 
-    // We only know last deploy time for applications that were deployed on this config server,
-    // the rest will be deployed on another config server
-    protected boolean shouldBeDeployedOnThisServer(ApplicationId application) {
-        return deployer().lastDeployTime(application).isPresent();
-    }
-
     // TODO: Do not start deploying until some time has gone (ideally only until bootstrap of config server is finished)
     private boolean waitInitially() {
         return clock.instant().isBefore(start.plus(minTimeBetweenRedeployments));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
index c27989cb852..dea0b8c19d0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -4,12 +4,15 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
+import com.yahoo.config.provision.TransientException;
+import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -73,6 +76,9 @@ public class RetiredExpirer extends Maintainer {
 
                 String nodeList = nodesToRemove.stream().map(Node::hostname).collect(Collectors.joining(", "));
                 log.info("Redeployed " + application + " to deactivate retired nodes: " +  nodeList);
+            } catch (TransientException e) {
+                log.log(LogLevel.INFO, "Failed to redeploy " + application +
+                        " with a transient error, will be retried by application maintainer: " + Exceptions.toMessageString(e));
             } catch (RuntimeException e) {
                 String nodeList = retiredNodes.stream().map(Node::hostname).collect(Collectors.joining(", "));
                 log.log(Level.WARNING, "Exception trying to deactivate retired nodes from " + application
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
index 299dc66c547..e628e823025 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,7 +40,7 @@ public class MockDeployer implements Deployer {
     @Inject
     @SuppressWarnings("unused")
     public MockDeployer() {
-        this(null, Clock.systemUTC(), Collections.emptyMap());
+        this(null, Clock.systemUTC(), Map.of());
     }
 
     /**
@@ -53,7 +52,7 @@ public class MockDeployer implements Deployer {
                         Map<ApplicationId, ApplicationContext> applications) {
         this.provisioner = provisioner;
         this.clock = clock;
-        this.applications = applications;
+        this.applications = new HashMap<>(applications);
     }
 
     public ReentrantLock lock() {
@@ -73,8 +72,8 @@ public class MockDeployer implements Deployer {
             throw new RuntimeException(e);
         }
         try {
-            lastDeployTimes.put(id, clock.instant());
-            return Optional.of(new MockDeployment(provisioner, applications.get(id)));
+            return Optional.ofNullable(applications.get(id))
+                    .map(application -> new MockDeployment(provisioner, application));
         } finally {
             lock.unlock();
         }
@@ -90,6 +89,13 @@ public class MockDeployer implements Deployer {
         return Optional.ofNullable(lastDeployTimes.get(application));
     }
 
+    public void removeApplication(ApplicationId applicationId) {
+        new MockDeployment(provisioner, new ApplicationContext(applicationId, List.of())).activate();
+
+        applications.remove(applicationId);
+        lastDeployTimes.remove(applicationId);
+    }
+
     public class MockDeployment implements Deployment {
 
         private final NodeRepositoryProvisioner provisioner;
@@ -116,6 +122,7 @@ public class MockDeployer implements Deployer {
             try (NestedTransaction t = new NestedTransaction()) {
                 provisioner.activate(t, application.id(), preparedHosts);
                 t.commit();
+                lastDeployTimes.put(application.id, clock.instant());
             }
         }
 
@@ -136,7 +143,7 @@ public class MockDeployer implements Deployer {
         }
 
         public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) {
-            this(id, Collections.singletonList(new ClusterContext(id, cluster, capacity, groups)));
+            this(id, List.of(new ClusterContext(id, cluster, capacity, groups)));
         }
 
         public ApplicationId id() { return id; }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index 5979ed37a98..e1ac0430ee4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
-import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
@@ -18,10 +17,8 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
-import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -36,7 +33,6 @@ import org.junit.Test;
 
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -62,33 +58,34 @@ public class OperatorChangeApplicationMaintainerTest {
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
                                                  true);
-        this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator);
+        this.fixture = new Fixture(zone, nodeRepository, nodeFlavors);
 
         createReadyNodes(15, this.fixture.nodeResources, nodeRepository);
         createHostNodes(2, nodeRepository, nodeFlavors);
 
         // Create applications
         fixture.activate();
-        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer, nodeRepository, clock, Duration.ofMinutes(1));
+        assertEquals("Initial applications are deployed", 2, fixture.deployer.redeployments);
+        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer, nodeRepository, Duration.ofMinutes(1));
         
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
-        assertEquals("No changes -> no redeployments", 0, fixture.deployer.redeployments);
+        assertEquals("No changes -> no redeployments", 2, fixture.deployer.redeployments);
 
         nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname(), Agent.system, "Failing to unit test");
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
-        assertEquals("System change -> no redeployments", 0, fixture.deployer.redeployments);
+        assertEquals("System change -> no redeployments", 2, fixture.deployer.redeployments);
 
         clock.advance(Duration.ofSeconds(1));
         nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(4).hostname(), Agent.operator, "Manual node failing");
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
-        assertEquals("Operator change -> redeployment", 1, fixture.deployer.redeployments);
+        assertEquals("Operator change -> redeployment", 3, fixture.deployer.redeployments);
 
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
-        assertEquals("No further operator changes -> no (new) redeployments", 1, fixture.deployer.redeployments);
+        assertEquals("No further operator changes -> no (new) redeployments", 3, fixture.deployer.redeployments);
     }
 
     private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
@@ -118,50 +115,33 @@ public class OperatorChangeApplicationMaintainerTest {
 
     private class Fixture {
 
-        final NodeResources nodeResources = new NodeResources(2, 8, 50);
         final NodeRepository nodeRepository;
-        final NodeRepositoryProvisioner provisioner;
-        final Curator curator;
+        final MockDeployer deployer;
 
+        final NodeResources nodeResources = new NodeResources(2, 8, 50);
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
         final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
         final int wantedNodesApp1 = 5;
         final int wantedNodesApp2 = 7;
-        MockDeployer deployer; // created on activation
 
-        Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors, Curator curator) {
+        Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors) {
             this.nodeRepository = nodeRepository;
-            this.curator = curator;
-            this.provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
+            NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(
+                    nodeRepository, flavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
+
+            Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
+                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1),
+                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1));
+            this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
         }
 
         void activate() {
-            activate(app1, clusterApp1, wantedNodesApp1, provisioner);
-            activate(app2, clusterApp2, wantedNodesApp2, provisioner);
+            deployer.deployFromLocalActive(app1, false).get().activate();
+            deployer.deployFromLocalActive(app2, false).get().activate();
             assertEquals(wantedNodesApp1, nodeRepository.getNodes(app1, Node.State.active).size());
             assertEquals(wantedNodesApp2, nodeRepository.getNodes(app2, Node.State.active).size());
-
-            Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
-            apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1,
-                                                               Capacity.fromCount(wantedNodesApp1, nodeResources), 1));
-            apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2,
-                                                               Capacity.fromCount(wantedNodesApp2, nodeResources), 1));
-            this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
-        }
-
-        private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) {
-            List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromCount(nodeCount, nodeResources), 1, null);
-            NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
-            provisioner.activate(transaction, applicationId, hosts);
-            transaction.commit();
-        }
-
-        void remove(ApplicationId application) {
-            NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
-            provisioner.remove(transaction, application);
-            transaction.commit();
         }
 
         NodeList getNodes(Node.State ... states) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index dff18912288..211b4a4472f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -10,7 +10,6 @@ import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
-import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
@@ -19,10 +18,8 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
-import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -46,7 +43,6 @@ import java.util.Map;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 /**
  * @author bratseth
@@ -68,7 +64,7 @@ public class PeriodicApplicationMaintainerTest {
                                                  new MockNameResolver().mockAnyLookup(),
                                                  DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
                                                  true);
-        this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator);
+        this.fixture = new Fixture(zone, nodeRepository, nodeFlavors);
 
         createReadyNodes(15, fixture.nodeResources, nodeRepository);
         createHostNodes(2, nodeRepository, nodeFlavors);
@@ -149,27 +145,32 @@ public class PeriodicApplicationMaintainerTest {
     public void application_deploy_inhibits_redeploy_for_a_while() {
         fixture.activate();
 
+        assertEquals("No deployment expected", 2, fixture.deployer.redeployments);
+
         // Holds off on deployments a while after starting
         fixture.runApplicationMaintainer();
-        assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app1).isPresent());
-        assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app2).isPresent());
+        assertEquals("No deployment expected", 2, fixture.deployer.redeployments);
+
         // Exhaust initial wait period
         clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
 
         // First deployment of applications
         fixture.runApplicationMaintainer();
+        assertEquals("No deployment expected", 4, fixture.deployer.redeployments);
         Instant firstDeployTime = clock.instant();
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
         clock.advance(Duration.ofMinutes(5));
         fixture.runApplicationMaintainer();
         // Too soon: Not redeployed:
+        assertEquals("No deployment expected", 4, fixture.deployer.redeployments);
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
 
         clock.advance(Duration.ofMinutes(30));
         fixture.runApplicationMaintainer();
         // Redeployed:
+        assertEquals("No deployment expected", 6, fixture.deployer.redeployments);
         assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app2).get());
     }
@@ -238,12 +239,10 @@ public class PeriodicApplicationMaintainerTest {
 
     private class Fixture {
 
-        final NodeResources nodeResources = new NodeResources(2, 8, 50);
         final NodeRepository nodeRepository;
-        final NodeRepositoryProvisioner provisioner;
-        final Curator curator;
         final MockDeployer deployer;
 
+        final NodeResources nodeResources = new NodeResources(2, 8, 50);
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
@@ -253,39 +252,28 @@ public class PeriodicApplicationMaintainerTest {
 
         private final TestablePeriodicApplicationMaintainer maintainer;
 
-        Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors, Curator curator) {
+        Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors) {
             this.nodeRepository = nodeRepository;
-            this.curator = curator;
-            this.provisioner =  new NodeRepositoryProvisioner(nodeRepository, flavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
-
-            Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
-            apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1,
-                                                               Capacity.fromCount(wantedNodesApp1, nodeResources), 1));
-            apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2,
-                                                               Capacity.fromCount(wantedNodesApp2, nodeResources), 1));
+            NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(
+                    nodeRepository, flavors, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
+
+            Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
+                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1),
+                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1));
             this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
             this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test
                                                                         Duration.ofMinutes(30));
         }
 
         void activate() {
-            activate(app1, clusterApp1, wantedNodesApp1, provisioner);
-            activate(app2, clusterApp2, wantedNodesApp2, provisioner);
+            deployer.deployFromLocalActive(app1, false).get().activate();
+            deployer.deployFromLocalActive(app2, false).get().activate();
             assertEquals(wantedNodesApp1, nodeRepository.getNodes(app1, Node.State.active).size());
             assertEquals(wantedNodesApp2, nodeRepository.getNodes(app2, Node.State.active).size());
         }
 
-        private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) {
-            List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromCount(nodeCount, nodeResources), 1, null);
-            NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
-            provisioner.activate(transaction, applicationId, hosts);
-            transaction.commit();
-        }
-
         void remove(ApplicationId application) {
-            NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
-            provisioner.remove(transaction, application);
-            transaction.commit();
+            deployer.removeApplication(application);
         }
 
         void runApplicationMaintainer() {
@@ -324,11 +312,6 @@ public class PeriodicApplicationMaintainerTest {
                     : super.nodesNeedingMaintenance();
         }
 
-        @Override
-        protected boolean shouldBeDeployedOnThisServer(ApplicationId application) {
-            return true;
-        }
-
     }
 
 }