Merge pull request #8454 from vespa-engine/freva/delayed-scheduling

Node-Admin: Delayed scheduling

12 files changed, 103 insertions, 55 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
index 1bb0cf3fb10..0e6000c651b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory;
 import com.yahoo.vespa.athenz.identity.SiaIdentityProvider;
-import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo;
 import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
 import org.apache.http.HttpHeaders;
@@ -40,8 +39,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
-import static java.util.Collections.singleton;
-
 /**
  * Retries request on config server a few times before giving up. Assumes that all requests should be sent with
  * content-type application/json
@@ -58,11 +55,6 @@ public class ConfigServerApiImpl implements ConfigServerApi {
 
     private final CloseableHttpClient client;
 
-    // TODO: Remove after 2018-12-01
-    public static ConfigServerApiImpl create(ConfigServerInfo info, SiaIdentityProvider provider) {
-        return create(info, provider, new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity())));
-    }
-
     public static ConfigServerApiImpl create(ConfigServerInfo info, SiaIdentityProvider provider, HostnameVerifier hostnameVerifier) {
         return new ConfigServerApiImpl(
                 info.getConfigServerUris(),
@@ -70,13 +62,6 @@ public class ConfigServerApiImpl implements ConfigServerApi {
                 provider);
     }
 
-    // TODO: Remove after 2018-12-01
-    public static ConfigServerApiImpl createFor(ConfigServerInfo info,
-                                                SiaIdentityProvider provider,
-                                                HostName configServerHostname) {
-        return createFor(info, provider, new AthenzIdentityVerifier(singleton(info.getConfigServerIdentity())), configServerHostname);
-    }
-
     public static ConfigServerApiImpl createFor(ConfigServerInfo info,
                                                 SiaIdentityProvider provider,
                                                 HostnameVerifier hostnameVerifier,
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java
index 93d00cb2cc5..a9954200f8a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java
@@ -11,6 +11,8 @@ import java.util.List;
 
 /**
  * An editor that assumes all rules in the filter table are exactly as the the wanted rules
+ *
+ * @author smorgrav
  */
 class FilterTableLineEditor implements LineEditor {
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java
index 419432b54f7..e1dc4a661ff 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java
@@ -10,6 +10,8 @@ import java.util.List;
 
 /**
  * An editor that only cares about the REDIRECT statement
+ *
+ * @author smorgrav
  */
 class NatTableLineEditor implements LineEditor {
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java
index 37d79d97e74..456391c65c2 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java
@@ -18,6 +18,9 @@ public interface NodeAdmin {
      */
     void refreshContainersToRun(final List<NodeSpec> containersToRun);
 
+    /** Gather node agent and its docker container metrics and forward them to the {@code MetricReceiverWrapper} */
+    void updateNodeAgentMetrics();
+
     /**
      * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that
      * they will not pick up any changes from NodeRepository.
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index 2303f78217c..2b37dcdf69c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -38,17 +38,18 @@ import java.util.stream.Collectors;
 public class NodeAdminImpl implements NodeAdmin {
     private static final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(NodeAdmin.class);
     private static final Duration NODE_AGENT_FREEZE_TIMEOUT = Duration.ofSeconds(5);
+    private static final Duration NODE_AGENT_SPREAD = Duration.ofSeconds(3);
 
     private final ScheduledExecutorService aclScheduler =
             Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("aclscheduler"));
-    private final ScheduledExecutorService metricsScheduler =
-            Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler"));
 
     private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory;
     private final NodeAgentContextFactory nodeAgentContextFactory;
     private final Optional<AclMaintainer> aclMaintainer;
 
     private final Clock clock;
+    private final Duration freezeTimeout;
+    private final Duration spread;
     private boolean previousWantFrozen;
     private boolean isFrozen;
     private Instant startOfFreezeConvergence;
@@ -64,19 +65,25 @@ public class NodeAdminImpl implements NodeAdmin {
                          MetricReceiverWrapper metricReceiver,
                          Clock clock) {
         this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext),
-                nodeAgentContextFactory, aclMaintainer, metricReceiver, clock);
+                nodeAgentContextFactory, aclMaintainer, metricReceiver, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD);
+    }
+
+    public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, NodeAgentContextFactory nodeAgentContextFactory,
+                         Optional<AclMaintainer> aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock, Duration freezeTimeout, Duration spread) {
+        this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext),
+                nodeAgentContextFactory, aclMaintainer, metricReceiver, clock, freezeTimeout, spread);
     }
 
     NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory,
-                  NodeAgentContextFactory nodeAgentContextFactory,
-                  Optional<AclMaintainer> aclMaintainer,
-                  MetricReceiverWrapper metricReceiver,
-                  Clock clock) {
+                  NodeAgentContextFactory nodeAgentContextFactory, Optional<AclMaintainer> aclMaintainer, MetricReceiverWrapper metricReceiver,
+                  Clock clock, Duration freezeTimeout, Duration spread) {
         this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory;
         this.nodeAgentContextFactory = nodeAgentContextFactory;
         this.aclMaintainer = aclMaintainer;
 
         this.clock = clock;
+        this.freezeTimeout = freezeTimeout;
+        this.spread = spread;
         this.previousWantFrozen = true;
         this.isFrozen = true;
         this.startOfFreezeConvergence = clock.instant();
@@ -102,19 +109,25 @@ public class NodeAdminImpl implements NodeAdmin {
             nodeAgentWithSchedulerByHostname.put(hostname, naws);
         });
 
+        Duration timeBetweenNodeAgents = spread.dividedBy(Math.max(nodeAgentContextsByHostname.size() - 1, 1));
+        Instant nextAgentStart = clock.instant();
         // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys
-        nodeAgentContextsByHostname.forEach((hostname, context) ->
-            nodeAgentWithSchedulerByHostname.get(hostname).scheduleTickWith(context)
-        );
+        for (String hostname : nodeAgentContextsByHostname.keySet()) {
+            NodeAgentContext context = nodeAgentContextsByHostname.get(hostname);
+            nodeAgentWithSchedulerByHostname.get(hostname).scheduleTickWith(context, nextAgentStart);
+            nextAgentStart = nextAgentStart.plus(timeBetweenNodeAgents);
+        }
     }
 
-    private void updateNodeAgentMetrics() {
+    @Override
+    public void updateNodeAgentMetrics() {
         int numberContainersWaitingImage = 0;
         int numberOfNewUnhandledExceptions = 0;
 
         for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) {
             if (nodeAgentWithScheduler.isDownloadingImage()) numberContainersWaitingImage++;
             numberOfNewUnhandledExceptions += nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions();
+            nodeAgentWithScheduler.updateContainerNodeMetrics();
         }
 
         numberOfContainersInLoadImageState.sample(numberContainersWaitingImage);
@@ -135,7 +148,7 @@ public class NodeAdminImpl implements NodeAdmin {
 
         // Use filter with count instead of allMatch() because allMatch() will short circuit on first non-match
         boolean allNodeAgentsConverged = nodeAgentWithSchedulerByHostname.values().parallelStream()
-                .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, NODE_AGENT_FREEZE_TIMEOUT))
+                .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, freezeTimeout))
                 .count() == 0;
 
         if (wantFrozen) {
@@ -173,15 +186,6 @@ public class NodeAdminImpl implements NodeAdmin {
 
     @Override
     public void start() {
-        metricsScheduler.scheduleAtFixedRate(() -> {
-            try {
-                updateNodeAgentMetrics();
-                nodeAgentWithSchedulerByHostname.values().forEach(NodeAgent::updateContainerNodeMetrics);
-            } catch (Throwable e) {
-                logger.warning("Metric fetcher scheduler failed", e);
-            }
-        }, 10, 55, TimeUnit.SECONDS);
-
         aclMaintainer.ifPresent(maintainer -> {
             int delay = 120; // WARNING: Reducing this will increase the load on config servers.
             aclScheduler.scheduleWithFixedDelay(() -> {
@@ -192,7 +196,6 @@ public class NodeAdminImpl implements NodeAdmin {
 
     @Override
     public void stop() {
-        metricsScheduler.shutdown();
         aclScheduler.shutdown();
 
         // Stop all node-agents in parallel, will block until the last NodeAgent is stopped
@@ -200,12 +203,11 @@ public class NodeAdminImpl implements NodeAdmin {
 
         do {
             try {
-                metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
                 aclScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
             } catch (InterruptedException e) {
                 logger.info("Was interrupted while waiting for metricsScheduler and aclScheduler to shutdown");
             }
-        } while (!metricsScheduler.isTerminated() || !aclScheduler.isTerminated());
+        } while (!aclScheduler.isTerminated());
     }
 
     // Set-difference. Returns minuend minus subtrahend.
@@ -232,7 +234,7 @@ public class NodeAdminImpl implements NodeAdmin {
         @Override public boolean isDownloadingImage() { return nodeAgent.isDownloadingImage(); }
         @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); }
 
-        @Override public void scheduleTickWith(NodeAgentContext context) { nodeAgentScheduler.scheduleTickWith(context); }
+        @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); }
         @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); }
     }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
index 13d3f3307d2..18c3a836e41 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
+import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
@@ -10,11 +11,17 @@ import com.yahoo.vespa.hosted.provision.Node;
 
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED;
+import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED;
 import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN;
 import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.TRANSITIONING;
 
@@ -27,6 +34,9 @@ public class NodeAdminStateUpdater {
     private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName());
     private static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5);
 
+    private final ScheduledExecutorService metricsScheduler =
+            Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler"));
+
     private final NodeRepository nodeRepository;
     private final Orchestrator orchestrator;
     private final NodeAdmin nodeAdmin;
@@ -34,7 +44,7 @@ public class NodeAdminStateUpdater {
 
     public enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED }
 
-    private State currentState = SUSPENDED_NODE_ADMIN;
+    private volatile State currentState = SUSPENDED_NODE_ADMIN;
 
     public NodeAdminStateUpdater(
             NodeRepository nodeRepository,
@@ -49,6 +59,31 @@ public class NodeAdminStateUpdater {
 
     public void start() {
         nodeAdmin.start();
+
+        EnumSet<State> suspendedStates = EnumSet.of(SUSPENDED_NODE_ADMIN, SUSPENDED);
+        metricsScheduler.scheduleAtFixedRate(() -> {
+            try {
+                if (suspendedStates.contains(currentState)) return;
+                nodeAdmin.updateNodeAgentMetrics();
+            } catch (Throwable e) {
+                log.log(Level.WARNING, "Metric fetcher scheduler failed", e);
+            }
+        }, 10, 55, TimeUnit.SECONDS);
+    }
+
+    public void stop() {
+        metricsScheduler.shutdown();
+
+        // Stop all node-agents in parallel, will block until the last NodeAgent is stopped
+        nodeAdmin.stop();
+
+        do {
+            try {
+                metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+            } catch (InterruptedException e) {
+                log.info("Was interrupted while waiting for metricsScheduler and shutdown");
+            }
+        } while (!metricsScheduler.isTerminated());
     }
 
     /**
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java
index 54f357d5f29..237b7d0daf7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
 import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Objects;
 
 /**
@@ -17,6 +18,7 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg
 
     private NodeAgentContext currentContext;
     private NodeAgentContext nextContext;
+    private Instant nextContextAt;
     private boolean wantFrozen = false;
     private boolean isFrozen = true;
     private boolean pendingInterrupt = false;
@@ -27,9 +29,10 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg
     }
 
     @Override
-    public void scheduleTickWith(NodeAgentContext context) {
+    public void scheduleTickWith(NodeAgentContext context, Instant at) {
         synchronized (monitor) {
             nextContext = Objects.requireNonNull(context);
+            nextContextAt = Objects.requireNonNull(at);
             monitor.notifyAll(); // Notify of new context
         }
     }
@@ -58,14 +61,17 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg
     @Override
     public NodeAgentContext nextContext() throws InterruptedException {
         synchronized (monitor) {
-            while (setAndGetIsFrozen(wantFrozen) || nextContext == null) {
+            Duration untilNextContext = Duration.ZERO;
+            while (setAndGetIsFrozen(wantFrozen) ||
+                    nextContext == null ||
+                    (untilNextContext = Duration.between(Instant.now(), nextContextAt)).toMillis() > 0) {
                 if (pendingInterrupt) {
                     pendingInterrupt = false;
                     throw new InterruptedException("interrupt() was called before next context was scheduled");
                 }
 
                 try {
-                    monitor.wait(); // Wait until scheduler provides a new context
+                    monitor.wait(Math.max(untilNextContext.toMillis(), 0L)); // Wait until scheduler provides a new context
                 } catch (InterruptedException ignored) { }
             }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
index 540601ffa4f..a5daab8dcfd 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
@@ -2,14 +2,15 @@
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
 import java.time.Duration;
+import java.time.Instant;
 
 /**
  * @author freva
  */
 public interface NodeAgentScheduler {
 
-    /** Schedule a tick for NodeAgent to run with the given NodeAgentContext */
-     void scheduleTickWith(NodeAgentContext context);
+    /** Schedule a tick for NodeAgent to run with the given NodeAgentContext, at no earlier than given instant */
+     void scheduleTickWith(NodeAgentContext context, Instant at);
 
     /**
      * Will eventually freeze/unfreeze the node agent
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java
index 9a35188a372..4588a9edcf4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java
@@ -12,8 +12,6 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 /**
  * Utils related to IOException.
  *
- * todo: replace much of the below with com.yahoo.yolean.Exceptions::uncheck
- *
  * @author hakonhall
  */
 public class IOExceptionUtil {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 88cc833f1f8..0254f58e7eb 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -98,7 +98,7 @@ public class DockerTester implements AutoCloseable {
                 Optional.empty(), Optional.empty(), Optional.empty());
         NodeAgentContextFactory nodeAgentContextFactory = nodeSpec ->
                 new NodeAgentContextImpl.Builder(nodeSpec).fileSystem(fileSystem).build();
-        nodeAdmin = new NodeAdminImpl(nodeAgentFactory, nodeAgentContextFactory, Optional.empty(), mr, Clock.systemUTC());
+        nodeAdmin = new NodeAdminImpl(nodeAgentFactory, nodeAgentContextFactory, Optional.empty(), mr, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO);
         nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator,
                 nodeAdmin, HOST_HOSTNAME);
 
@@ -139,7 +139,7 @@ public class DockerTester implements AutoCloseable {
     @Override
     public void close() {
         // First, stop NodeAdmin and all the NodeAgents
-        nodeAdmin.stop();
+        nodeAdminStateUpdater.stop();
 
         terminated = true;
         do {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index 47e220a968b..f8e87ccef53 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -45,7 +45,7 @@ public class NodeAdminImplTest {
     private final ManualClock clock = new ManualClock();
 
     private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, nodeAgentContextFactory,
-            Optional.empty(), new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock);
+            Optional.empty(), new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock, Duration.ZERO, Duration.ZERO);
 
     @Test
     public void nodeAgentsAreProperlyLifeCycleManaged() {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java
index f32e3d91e34..5aeccb4ab7d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java
@@ -5,6 +5,7 @@ import org.junit.Test;
 
 import java.time.Clock;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
@@ -26,7 +27,7 @@ public class NodeAgentContextManagerTest {
     @Test(timeout = TIMEOUT)
     public void returns_immediately_if_next_context_is_ready() throws InterruptedException {
         NodeAgentContext context1 = generateContext();
-        manager.scheduleTickWith(context1);
+        manager.scheduleTickWith(context1, clock.instant());
 
         assertSame(initialContext, manager.currentContext());
         assertSame(context1, manager.nextContext());
@@ -34,6 +35,19 @@ public class NodeAgentContextManagerTest {
     }
 
     @Test(timeout = TIMEOUT)
+    public void returns_no_earlier_than_at_given_time() throws InterruptedException {
+        NodeAgentContext context1 = generateContext();
+        Instant returnAt = clock.instant().plusMillis(500);
+        manager.scheduleTickWith(context1, returnAt);
+
+        assertSame(initialContext, manager.currentContext());
+        assertSame(context1, manager.nextContext());
+        assertSame(context1, manager.currentContext());
+        // Is accurate to a millisecond
+        assertFalse(clock.instant().plusMillis(1).isBefore(returnAt));
+    }
+
+    @Test(timeout = TIMEOUT)
     public void blocks_in_nextContext_until_one_is_scheduled() throws InterruptedException {
         AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext);
         assertFalse(async.response.isPresent());
@@ -41,7 +55,7 @@ public class NodeAgentContextManagerTest {
         assertFalse(async.response.isPresent());
 
         NodeAgentContext context1 = generateContext();
-        manager.scheduleTickWith(context1);
+        manager.scheduleTickWith(context1, clock.instant());
 
         async.awaitResult();
         assertEquals(Optional.of(context1), async.response);
@@ -68,7 +82,7 @@ public class NodeAgentContextManagerTest {
 
         // Generate new context and get it from the supplier, this completes the unfreeze
         NodeAgentContext context1 = generateContext();
-        manager.scheduleTickWith(context1);
+        manager.scheduleTickWith(context1, clock.instant());
         assertSame(context1, manager.nextContext());
 
         assertTrue(manager.setFrozen(false, Duration.ZERO));
@@ -91,7 +105,7 @@ public class NodeAgentContextManagerTest {
         assertFalse(async.response.isPresent());
 
         NodeAgentContext context1 = generateContext();
-        manager.scheduleTickWith(context1);
+        manager.scheduleTickWith(context1, clock.instant());
         assertSame(context1, manager.nextContext());
 
         async.awaitResult();