Merge pull request #6309 from vespa-engine/bratseth/application-deploy-inhibits-maintenance-deploy

Application deployments inhibits periodic redeploys for a while

11 files changed, 96 insertions, 21 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index 1d894f80eca..42dfb38e4d3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
@@ -38,21 +39,28 @@ public abstract class ApplicationMaintainer extends Maintainer {
     protected final void maintain() {
         Set<ApplicationId> applications = applicationsNeedingMaintenance();
         for (ApplicationId application : applications) {
-            deploy(application);
+            if (canDeployNow(application))
+                deploy(application);
             throttle(applications.size());
         }
     }
 
+    protected boolean canDeployNow(ApplicationId application) {
+        return true;
+    }
+
     /**
      * Redeploy this application.
      *
-     * The default implementation deploys asynchronously to make sure we do all applications timely 
+     * The default implementation deploys asynchronously to make sure we do all applications timely
      * even when deployments are slow.
      */
     protected void deploy(ApplicationId application) {
         deploymentExecutor.execute(() -> deployWithLock(application));
     }
 
+    protected Deployer deployer() { return deployer; }
+
     /** Block in this method until the next application should be maintained */
     protected abstract void throttle(int applicationCount);
 
@@ -78,7 +86,6 @@ public abstract class ApplicationMaintainer extends Maintainer {
             if ( ! isActive(application)) return; // became inactive since deployment was requested
             Optional<Deployment> deployment = deployer.deployFromLocalActive(application);
             if ( ! deployment.isPresent()) return; // this will be done at another config server
-
             deployment.get().activate();
         } catch (RuntimeException e) {
             log.log(Level.WARNING, "Exception on maintenance redeploy", e);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index cdc5ecf77ec..c0e2d94ebea 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -1,12 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * The application maintainer regularly redeploys all applications to make sure the node repo and application
@@ -29,6 +32,15 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
     }
 
     @Override
+    protected boolean canDeployNow(ApplicationId application) {
+        Optional<Instant> lastDeploy = deployer().lastDeployTime(application);
+        if (lastDeploy.isPresent() &&
+            lastDeploy.get().isAfter(nodeRepository().clock().instant().minus(interval())))
+            return false; // Don't deploy if a regular deploy just happened
+        return true;
+    }
+
+    @Override
     protected List<Node> nodesNeedingMaintenance() {
         return nodeRepository().getNodes(Node.State.active);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
index c880c66abc7..470aa4f39f2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
@@ -12,8 +12,11 @@ import com.yahoo.config.provision.HostSpec;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 
+import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -26,22 +29,28 @@ public class MockDeployer implements Deployer {
 
     private final NodeRepositoryProvisioner provisioner;
     private final Map<ApplicationId, ApplicationContext> applications;
+    private final Map<ApplicationId, Instant> lastDeployTimes = new HashMap<>();
 
     /** The number of redeployments done to this */
     public int redeployments = 0;
 
+    private final Clock clock;
+
     @Inject
     @SuppressWarnings("unused")
     public MockDeployer() {
-        this(null, Collections.emptyMap());
+        this(null, Clock.systemUTC(), Collections.emptyMap());
     }
 
     /**
      * Create a mock deployer which contains a substitute for an application repository, fullfilled to
      * be able to call provision with the right parameters.
      */
-    public MockDeployer(NodeRepositoryProvisioner provisioner, Map<ApplicationId, ApplicationContext> applications) {
+    public MockDeployer(NodeRepositoryProvisioner provisioner,
+                        Clock clock,
+                        Map<ApplicationId, ApplicationContext> applications) {
         this.provisioner = provisioner;
+        this.clock = clock;
         this.applications = applications;
     }
 
@@ -52,9 +61,15 @@ public class MockDeployer implements Deployer {
 
     @Override
     public Optional<Deployment> deployFromLocalActive(ApplicationId id, Duration timeout) {
+        lastDeployTimes.put(id, clock.instant());
         return Optional.of(new MockDeployment(provisioner, applications.get(id)));
     }
 
+    @Override
+    public Optional<Instant> lastDeployTime(ApplicationId application) {
+        return Optional.ofNullable(lastDeployTimes.get(application));
+    }
+
     public class MockDeployment implements Deployment {
 
         private final NodeRepositoryProvisioner provisioner;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index f4fa8461f12..84e366c9510 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -13,6 +13,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
@@ -142,6 +143,7 @@ public class InactiveAndFailedExpirerTest {
         tester.advanceTime(Duration.ofMinutes(11)); // Trigger RetiredExpirer
         MockDeployer deployer = new MockDeployer(
                 tester.provisioner(),
+                tester.clock(),
                 Collections.singletonMap(
                         applicationId,
                         new MockDeployer.ApplicationContext(applicationId, cluster,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index b1bceec6d9f..ea28f7bafc8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub;
 import com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 
+import java.time.Clock;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -111,7 +112,7 @@ public class NodeFailTester {
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
         apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromNodeCount(wantedNodesApp1, Optional.of("default"), false, true), 1));
         apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1));
-        tester.deployer = new MockDeployer(tester.provisioner, apps);
+        tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
         tester.metric = new MetricsReporterTest.TestMetric();
         tester.failer = tester.createFailer();
@@ -147,7 +148,7 @@ public class NodeFailTester {
         apps.put(nodeAdminApp, new MockDeployer.ApplicationContext(nodeAdminApp, clusterNodeAdminApp, allHosts, 1));
         apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1));
         apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1));
-        tester.deployer = new MockDeployer(tester.provisioner, apps);
+        tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
         tester.metric = new MetricsReporterTest.TestMetric();
         tester.failer = tester.createFailer();
@@ -170,7 +171,7 @@ public class NodeFailTester {
 
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
         apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, allProxies, 1));
-        tester.deployer = new MockDeployer(tester.provisioner, apps);
+        tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
         tester.metric = new MetricsReporterTest.TestMetric();
         tester.failer = tester.createFailer();
@@ -179,7 +180,7 @@ public class NodeFailTester {
 
     public static NodeFailTester withNoApplications() {
         NodeFailTester tester = new NodeFailTester();
-        tester.deployer = new MockDeployer(tester.provisioner, Collections.emptyMap());
+        tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), Collections.emptyMap());
         tester.serviceMonitor = new ServiceMonitorStub(Collections.emptyMap(), tester.nodeRepository);
         tester.metric = new MetricsReporterTest.TestMetric();
         tester.failer = tester.createFailer();
@@ -210,6 +211,8 @@ public class NodeFailTester {
         }
     }
 
+    public Clock clock() { return clock; }
+
     public List<Node> createReadyNodes(int count) {
         return createReadyNodes(count, 0);
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java
index 421211b7c49..2bf4f831072 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java
@@ -75,7 +75,7 @@ public class NodeRetirerTester {
                                             new DockerImage("docker-registry.domain.tld:8080/dist/vespa"), true);
         jobControl = new JobControl(nodeRepository.database());
         NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone);
-        deployer = new MockDeployer(provisioner, apps);
+        deployer = new MockDeployer(provisioner, clock, apps);
         flavors = nodeFlavors.getFlavors().stream().sorted(Comparator.comparing(Flavor::name)).collect(Collectors.toList());
 
         try {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index c7a649faf98..976831d129d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -136,7 +136,7 @@ public class OperatorChangeApplicationMaintainerTest {
                                                                Capacity.fromNodeCount(wantedNodesApp1, Optional.of("default"), false, true), 1));
             apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2,
                                                                Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1));
-            this.deployer = new MockDeployer(provisioner, apps);
+            this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
         }
 
         private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodeCount, NodeRepositoryProvisioner provisioner) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index de7f4fde6ae..360e8de8d11 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -32,7 +32,9 @@ import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -63,7 +65,7 @@ public class PeriodicApplicationMaintainerTest {
     }
 
     @Test
-    public void test_application_maintenance() throws InterruptedException {
+    public void test_application_maintenance() {
         createReadyNodes(15, nodeRepository, nodeFlavors);
         createHostNodes(2, nodeRepository, nodeFlavors);
 
@@ -101,6 +103,7 @@ public class PeriodicApplicationMaintainerTest {
                      0, fixture.getNodes(Node.State.active).retired().size());
 
         // Cause maintenance deployment which will update the applications with the re-activated nodes
+        ((ManualClock)nodeRepository.clock()).advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited
         fixture.runApplicationMaintainer();
         assertEquals("Superflous content nodes are retired",
                      reactivatedInApp2, fixture.getNodes(Node.State.active).retired().size());
@@ -129,6 +132,31 @@ public class PeriodicApplicationMaintainerTest {
                      nodeRepository.getNodes(fixture.app2, Node.State.inactive).size());
     }
 
+    @Test
+    public void application_deploy_inhibits_redeploy_for_a_while() {
+        ManualClock clock = (ManualClock)nodeRepository.clock();
+        createReadyNodes(15, nodeRepository, nodeFlavors);
+        createHostNodes(2, nodeRepository, nodeFlavors);
+
+        // Create applications
+        fixture.activate();
+        fixture.runApplicationMaintainer();
+        Instant firstDeployTime = clock.instant();
+        assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
+        assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
+        ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(5));
+        fixture.runApplicationMaintainer();
+        // Too soo: Not redeployed:
+        assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
+        assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
+
+        ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(30));
+        fixture.runApplicationMaintainer();
+        // Redeployed:
+        assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app1).get());
+        assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app2).get());
+    }
+
     private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
@@ -152,6 +180,7 @@ public class PeriodicApplicationMaintainerTest {
         final NodeRepository nodeRepository;
         final NodeRepositoryProvisioner provisioner;
         final Curator curator;
+        final Deployer deployer;
 
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
@@ -164,6 +193,13 @@ public class PeriodicApplicationMaintainerTest {
             this.nodeRepository = nodeRepository;
             this.curator = curator;
             this.provisioner =  new NodeRepositoryProvisioner(nodeRepository, flavors, zone);
+
+            Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
+            apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1,
+                                                               Capacity.fromNodeCount(wantedNodesApp1, Optional.of("default"), false, true), 1));
+            apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2,
+                                                               Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1));
+            this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
         }
 
         void activate() {
@@ -191,12 +227,6 @@ public class PeriodicApplicationMaintainerTest {
         }
 
         void runApplicationMaintainer(Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
-            apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, 
-                                                               Capacity.fromNodeCount(wantedNodesApp1, Optional.of("default"), false, true), 1));
-            apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, 
-                                                               Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1));
-            MockDeployer deployer = new MockDeployer(provisioner, apps);
             new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(30), overriddenNodesNeedingMaintenance).run();
         }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index e5a46adef8e..03f82e5c3d7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -92,6 +92,7 @@ public class RetiredExpirerTest {
         clock.advance(Duration.ofHours(30)); // Retire period spent
         MockDeployer deployer =
             new MockDeployer(provisioner,
+                             clock,
                              Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(wantedNodes, Optional.of("default"), false, true), 1)));
         createRetiredExpirer(deployer).run();
         assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size());
@@ -120,6 +121,7 @@ public class RetiredExpirerTest {
         clock.advance(Duration.ofHours(30)); // Retire period spent
         MockDeployer deployer =
             new MockDeployer(provisioner,
+                             clock,
                              Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(2, Optional.of("default"), false, true), 1)));
         createRetiredExpirer(deployer).run();
         assertEquals(2, nodeRepository.getNodes(applicationId, Node.State.active).size());
@@ -151,9 +153,10 @@ public class RetiredExpirerTest {
         // Cause inactivation of retired nodes
         MockDeployer deployer =
                 new MockDeployer(provisioner,
-                        Collections.singletonMap(
-                                applicationId,
-                                new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(wantedNodes, Optional.of("default"), false, true), 1)));
+                                 clock,
+                                 Collections.singletonMap(
+                                     applicationId,
+                                     new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(wantedNodes, Optional.of("default"), false, true), 1)));
 
 
         // Allow the 1st and 3rd retired nodes permission to inactivate
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
index 62ff978047b..86daf636875 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
@@ -133,6 +133,7 @@ public class MultigroupProvisioningTest {
         tester.advanceTime(Duration.ofDays(7));
         MockDeployer deployer =
             new MockDeployer(tester.provisioner(),
+                             tester.clock(),
                              Collections.singletonMap(application1, 
                                                       new MockDeployer.ApplicationContext(application1, cluster(), 
                                                                                           Capacity.fromNodeCount(8, Optional.of("large"), false, true), 1)));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
index 5f3976ab137..03cc3a3c20b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
@@ -97,6 +97,7 @@ public class NodeTypeProvisioningTest {
     public void retire_proxy() {
         MockDeployer deployer = new MockDeployer(
                 tester.provisioner(),
+                tester.clock(),
                 Collections.singletonMap(
                         application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
         RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer,
@@ -161,6 +162,7 @@ public class NodeTypeProvisioningTest {
     public void retire_multiple_proxy_simultaneously() {
         MockDeployer deployer = new MockDeployer(
                 tester.provisioner(),
+                tester.clock(),
                 Collections.singletonMap(
                         application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
         RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer,