Enforce CC timeouts in Orchestrator 2

7 files changed, 245 insertions, 135 deletions

diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 76d9398c44e..80174d05a54 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -43,6 +43,8 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
@@ -245,6 +247,8 @@ public class OrchestratorImplTest {
         // A spy is preferential because suspendAll() relies on delegating the hard work to suspend() and resume().
         OrchestratorImpl orchestrator = spy(this.orchestrator);
 
+        OrchestratorContext context = mock(OrchestratorContext.class);
+
         orchestrator.suspendAll(
                 new HostName("parentHostname"),
                 Arrays.asList(
@@ -257,9 +261,9 @@ public class OrchestratorImplTest {
         //   TEST6: tenant-id-3:application-instance-3:default
         //   TEST1: test-tenant-id:application:instance
         InOrder order = inOrder(orchestrator);
-        order.verify(orchestrator).suspendGroup(DummyInstanceLookupService.TEST3_NODE_GROUP);
-        order.verify(orchestrator).suspendGroup(DummyInstanceLookupService.TEST6_NODE_GROUP);
-        order.verify(orchestrator).suspendGroup(DummyInstanceLookupService.TEST1_NODE_GROUP);
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST3_NODE_GROUP));
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP));
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST1_NODE_GROUP));
         order.verifyNoMoreInteractions();
     }
 
@@ -272,7 +276,7 @@ public class OrchestratorImplTest {
                 DummyInstanceLookupService.TEST6_HOST_NAME,
                 "some-constraint",
                 "error message");
-        doThrow(supensionFailure).when(orchestrator).suspendGroup(DummyInstanceLookupService.TEST6_NODE_GROUP);
+        doThrow(supensionFailure).when(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP));
 
         try {
             orchestrator.suspendAll(
@@ -291,8 +295,8 @@ public class OrchestratorImplTest {
         }
 
         InOrder order = inOrder(orchestrator);
-        order.verify(orchestrator).suspendGroup(DummyInstanceLookupService.TEST3_NODE_GROUP);
-        order.verify(orchestrator).suspendGroup(DummyInstanceLookupService.TEST6_NODE_GROUP);
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST3_NODE_GROUP));
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP));
         order.verifyNoMoreInteractions();
     }
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
index 228174a9b3d..91909391fe7 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
@@ -6,7 +6,9 @@ import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import org.junit.Test;
 
-import static org.mockito.Matchers.anyFloat;
+import java.time.Duration;
+
+import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -28,7 +30,9 @@ public class ClusterControllerClientTest {
         final ClusterControllerNodeState wantedState = ClusterControllerNodeState.MAINTENANCE;
 
         OrchestratorContext context = mock(OrchestratorContext.class);
-        when(context.getSuboperationTimeoutInSeconds(anyFloat())).thenReturn(1.0f);
+        ClusterControllerClientTimeouts timeouts = mock(ClusterControllerClientTimeouts.class);
+        when(context.getClusterControllerTimeouts(any())).thenReturn(timeouts);
+        when(timeouts.getServerTimeout()).thenReturn(Duration.ofSeconds(1));
         clusterControllerClient.setNodeState(context, STORAGE_NODE_INDEX, wantedState);
 
         final ClusterControllerStateRequest expectedNodeStateRequest = new ClusterControllerStateRequest(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java
new file mode 100644
index 00000000000..36dd4c4a83f
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java
@@ -0,0 +1,155 @@
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.test.ManualClock;
+import com.yahoo.time.TimeBudget;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Optional;
+
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.CONNECT_TIMEOUT;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD_PER_CALL;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.MIN_SERVER_TIMEOUT;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.NETWORK_OVERHEAD_PER_CALL;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.NUM_CALLS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ClusterControllerClientTimeoutsTest {
+    // The minimum time left for any invocation of prepareForImmediateJaxRsCall().
+    private static final Duration MINIMUM_TIME_LEFT = IN_PROCESS_OVERHEAD_PER_CALL
+            .plus(CONNECT_TIMEOUT)
+            .plus(NETWORK_OVERHEAD_PER_CALL)
+            .plus(MIN_SERVER_TIMEOUT);
+    static {
+        assertEquals(Duration.ofMillis(160), MINIMUM_TIME_LEFT);
+    }
+
+    // The minimum time left (= original time) which is required to allow any requests to the CC.
+    private static final Duration MINIMUM_ORIGINAL_TIMEOUT = MINIMUM_TIME_LEFT
+            .multipliedBy(NUM_CALLS)
+            .plus(IN_PROCESS_OVERHEAD);
+    static {
+        assertEquals(Duration.ofMillis(420), MINIMUM_ORIGINAL_TIMEOUT);
+    }
+
+    private final ManualClock clock = new ManualClock();
+
+    private Duration originalTimeout;
+    private TimeBudget timeBudget;
+    private ClusterControllerClientTimeouts timeouts;
+
+    private void makeTimeouts(Duration originalTimeout) {
+        this.originalTimeout = originalTimeout;
+        this.timeBudget = TimeBudget.from(clock, clock.instant(), Optional.of(originalTimeout));
+        this.timeouts = new ClusterControllerClientTimeouts("clustername", timeBudget);
+    }
+
+    @Before
+    public void setUp() {
+        makeTimeouts(Duration.ofSeconds(3));
+    }
+
+    @Test
+    public void makes2RequestsWithMaxProcessingTime() {
+        assertEquals(Duration.ofMillis(50), timeouts.getConnectTimeout());
+        assertEquals(Duration.ofMillis(1350), timeouts.getReadTimeout());
+        assertEquals(Duration.ofMillis(1300), timeouts.getServerTimeout());
+
+        Duration maxProcessingTime = IN_PROCESS_OVERHEAD_PER_CALL
+                .plus(CONNECT_TIMEOUT)
+                .plus(timeouts.getReadTimeout());
+        assertEquals(1450, maxProcessingTime.toMillis());
+        clock.advance(maxProcessingTime);
+
+        assertEquals(Duration.ofMillis(50), timeouts.getConnectTimeout());
+        assertEquals(Duration.ofMillis(1350), timeouts.getReadTimeout());
+        assertEquals(Duration.ofMillis(1300), timeouts.getServerTimeout());
+
+        clock.advance(maxProcessingTime);
+
+        try {
+            timeouts.getServerTimeout();
+            fail();
+        } catch (UncheckedTimeoutException e) {
+            assertEquals(
+                    "Too little time left (PT0.1S) to call content cluster 'clustername', original timeout was PT3S",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void makesAtLeast3RequestsWithShortProcessingTime() {
+        assertEquals(Duration.ofMillis(50), timeouts.getConnectTimeout());
+        assertEquals(Duration.ofMillis(1350), timeouts.getReadTimeout());
+        assertEquals(Duration.ofMillis(1300), timeouts.getServerTimeout());
+
+        Duration shortPocessingTime = Duration.ofMillis(200);
+        clock.advance(shortPocessingTime);
+
+        assertEquals(Duration.ofMillis(50), timeouts.getConnectTimeout());
+        assertEquals(Duration.ofMillis(1350), timeouts.getReadTimeout());
+        assertEquals(Duration.ofMillis(1300), timeouts.getServerTimeout());
+
+        clock.advance(shortPocessingTime);
+
+        assertEquals(Duration.ofMillis(50), timeouts.getConnectTimeout());
+        assertEquals(Duration.ofMillis(1350), timeouts.getReadTimeout());
+        assertEquals(Duration.ofMillis(1300), timeouts.getServerTimeout());
+    }
+
+    @Test
+    public void alreadyTimedOut() {
+        clock.advance(Duration.ofSeconds(4));
+
+        try {
+            timeouts.getServerTimeout();
+            fail();
+        } catch (UncheckedTimeoutException e) {
+            assertEquals(
+                    "Exceeded the timeout PT3S against content cluster 'clustername' by PT1S",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void justTooLittleTime() {
+        clock.advance(originalTimeout.minus(MINIMUM_TIME_LEFT).plus(Duration.ofMillis(1)));
+        try {
+            timeouts.getServerTimeout();
+            fail();
+        } catch (UncheckedTimeoutException e) {
+            assertEquals(
+                    "Server would be given too little time to complete: PT0.009S. Original timeout was PT3S",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void justEnoughTime() {
+        clock.advance(originalTimeout.minus(MINIMUM_TIME_LEFT));
+        timeouts.getServerTimeout();
+    }
+
+    @Test
+    public void justTooLittleInitialTime() {
+        makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT.minus(Duration.ofMillis(1)));
+        try {
+            timeouts.getServerTimeout();
+            fail();
+        } catch (UncheckedTimeoutException e) {
+            assertEquals(
+                    "Server would be given too little time to complete: PT0.0095S. Original timeout was PT0.419S",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void justEnoughInitialTime() {
+        makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT);
+        timeouts.getServerTimeout();
+    }
+}
\ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java
new file mode 100644
index 00000000000..1f505991476
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java
@@ -0,0 +1,54 @@
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RetryingClusterControllerClientFactoryTest {
+    private final Clock clock = new ManualClock();
+
+    @Test
+    public void verifyJerseyCallForSetNodeState() throws IOException {
+        JaxRsClientFactory clientFactory = mock(JaxRsClientFactory.class);
+        ClusterControllerJaxRsApi api = mock(ClusterControllerJaxRsApi.class);
+        when(clientFactory.createClient(any())).thenReturn(api);
+        RetryingClusterControllerClientFactory factory = new RetryingClusterControllerClientFactory(clientFactory);
+        String clusterName = "clustername";
+        HostName host1 = new HostName("host1");
+        HostName host2 = new HostName("host2");
+        HostName host3 = new HostName("host3");
+        List<HostName> clusterControllers = Arrays.asList(host1, host2, host3);
+        ClusterControllerClient client = factory.createClient(clusterControllers, clusterName);
+        OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
+        int storageNode = 2;
+        ClusterControllerNodeState wantedState = ClusterControllerNodeState.MAINTENANCE;
+        client.setNodeState(context, storageNode, wantedState);
+
+        ArgumentCaptor<ClusterControllerStateRequest> requestCaptor = ArgumentCaptor.forClass(ClusterControllerStateRequest.class);
+
+        verify(api, times(1)).setNodeState(eq(clusterName), eq(storageNode), eq(4.8f), requestCaptor.capture());
+        ClusterControllerStateRequest request = requestCaptor.getValue();
+        assertEquals(ClusterControllerStateRequest.Condition.SAFE, request.condition);
+        Map<String, Object> expectedState = new HashMap<>();
+        expectedState.put("user", new ClusterControllerStateRequest.State(wantedState, "Orchestrator"));
+        assertEquals(expectedState, request.state);
+    }
+}
\ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java
deleted file mode 100644
index 4dabae14add..00000000000
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.controller;
-
-import com.yahoo.vespa.applicationmodel.ConfigId;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory;
-import com.yahoo.vespa.orchestrator.OrchestratorContext;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-import static org.hamcrest.CoreMatchers.anyOf;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyFloat;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class SingleInstanceClusterControllerClientFactoryTest {
-    private static final int PORT = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_HARDCODED_PORT;
-    private static final String PATH = SingleInstanceClusterControllerClientFactory.CLUSTERCONTROLLER_API_PATH;
-
-    private static final HostName HOST_NAME_1 = new HostName("host1");
-    private static final HostName HOST_NAME_2 = new HostName("host2");
-    private static final HostName HOST_NAME_3 = new HostName("host3");
-
-    OrchestratorContext context = mock(OrchestratorContext.class);
-    private final ClusterControllerJaxRsApi mockApi = mock(ClusterControllerJaxRsApi.class);
-    private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class);
-    private final ClusterControllerClientFactory clientFactory
-            = new SingleInstanceClusterControllerClientFactory(jaxRsClientFactory);
-
-    @Before
-    public void setup() {
-        when(
-                jaxRsClientFactory.createClient(
-                        eq(ClusterControllerJaxRsApi.class),
-                        any(HostName.class),
-                        anyInt(),
-                        anyString(),
-                        anyString()))
-                .thenReturn(mockApi);
-    }
-
-    @Test
-    public void testCreateClientWithNoClusterControllerInstances() throws Exception {
-        final List<HostName> clusterControllers = Arrays.asList();
-
-        try {
-            clientFactory.createClient(clusterControllers, "clusterName");
-            fail();
-        } catch (IllegalArgumentException e) {
-            // As expected.
-        }
-    }
-
-    @Test
-    public void testCreateClientWithSingleClusterControllerInstance() throws Exception {
-        final List<HostName> clusterControllers = Arrays.asList(HOST_NAME_1);
-
-        when(context.getSuboperationTimeoutInSeconds(anyFloat())).thenReturn(1.0f);
-        clientFactory.createClient(clusterControllers, "clusterName")
-                .setNodeState(context, 0, ClusterControllerNodeState.MAINTENANCE);
-
-        verify(jaxRsClientFactory).createClient(
-                ClusterControllerJaxRsApi.class,
-                HOST_NAME_1,
-                PORT,
-                PATH,
-                "http");
-    }
-
-    @Test
-    public void testCreateClientWithoutClusterControllerInstances() throws Exception {
-        final List<HostName> clusterControllers = Arrays.asList();
-
-        try {
-            clientFactory.createClient(clusterControllers, "clusterName");
-            fail();
-        } catch (IllegalArgumentException e) {
-            // As expected.
-        }
-    }
-
-    @Test
-    public void testCreateClientWithThreeClusterControllerInstances() throws Exception {
-        final List<HostName> clusterControllers = Arrays.asList(HOST_NAME_1, HOST_NAME_2, HOST_NAME_3);
-
-        when(context.getSuboperationTimeoutInSeconds(anyFloat())).thenReturn(1.0f);
-        clientFactory.createClient(clusterControllers, "clusterName")
-                .setNodeState(context, 0, ClusterControllerNodeState.MAINTENANCE);
-
-        verify(jaxRsClientFactory).createClient(
-                eq(ClusterControllerJaxRsApi.class),
-                argThat(is(anyOf(
-                        equalTo(HOST_NAME_1),
-                        equalTo(HOST_NAME_2),
-                        equalTo(HOST_NAME_3)))),
-                eq(PORT),
-                eq(PATH),
-                eq("http"));
-    }
-
-    private static ConfigId clusterControllerConfigId(final int index) {
-        return new ConfigId("admin/cluster-controllers/" + index);
-    }
-}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 45ba862c8f1..a9b8127e7fe 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.resources;
 
-import com.yahoo.jdisc.Timer;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
@@ -32,6 +31,7 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
+import org.junit.Before;
 import org.junit.Test;
 
 import javax.ws.rs.BadRequestException;
@@ -41,6 +41,7 @@ import javax.ws.rs.core.UriBuilder;
 import javax.ws.rs.core.UriInfo;
 import java.net.URI;
 import java.time.Clock;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Optional;
@@ -74,7 +75,7 @@ public class HostResourceTest {
     static {
         when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.forApplicationInstance(eq(APPLICATION_INSTANCE_REFERENCE)))
                 .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
-        when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.lockApplicationInstance_forCurrentThreadOnly(eq(APPLICATION_INSTANCE_REFERENCE)))
+        when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.lockApplicationInstance_forCurrentThreadOnly(eq(APPLICATION_INSTANCE_REFERENCE), any()))
                 .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
         when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getHostStatus(any()))
                 .thenReturn(HostStatus.NO_REMARKS);
@@ -150,6 +151,11 @@ public class HostResourceTest {
 
     private final UriInfo uriInfo = mock(UriInfo.class);
 
+    @Before
+    public void setUp() {
+        when(clock.instant()).thenReturn(Instant.now());
+    }
+
     @Test
     public void returns_200_on_success() {
         HostResource hostResource =
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
index 2e914718e20..44847666670 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
@@ -17,6 +17,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -81,7 +82,8 @@ public class ZookeeperStatusServiceTest {
     @Test
     public void setting_host_state_is_idempotent() {
         try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                TestIds.APPLICATION_INSTANCE_REFERENCE,
+                Duration.ofSeconds(10))) {
 
             //shuffling to catch "clean database" failures for all cases.
             for (HostStatus hostStatus: shuffledList(HostStatus.values())) {
@@ -105,11 +107,12 @@ public class ZookeeperStatusServiceTest {
 
             final CompletableFuture<Void> lockedSuccessfullyFuture;
             try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                    TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                    TestIds.APPLICATION_INSTANCE_REFERENCE,
+                    Duration.ofSeconds(10))) {
 
                 lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> {
                     try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
-                            .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE))
+                            .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(10)))
                     {
                     }
                 });
@@ -131,13 +134,13 @@ public class ZookeeperStatusServiceTest {
             ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator);
 
             try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                    TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                    TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(10))) {
 
                 //must run in separate thread, since having 2 locks in the same thread fails
                 CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> {
                     try {
                         zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(
-                                TestIds.APPLICATION_INSTANCE_REFERENCE,1);
+                                TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(1));
                         fail("Both zookeeper host status services locked simultaneously for the same application instance");
                     } catch (RuntimeException e) {
                     }
@@ -211,7 +214,7 @@ public class ZookeeperStatusServiceTest {
 
         // Suspend
         try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(10))) {
             statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }
 
@@ -223,7 +226,7 @@ public class ZookeeperStatusServiceTest {
 
         // Resume
         try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(10))) {
             statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS);
         }
 
@@ -241,12 +244,12 @@ public class ZookeeperStatusServiceTest {
         assertThat(suspendedApps.size(), is(0));
 
         try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+                TestIds.APPLICATION_INSTANCE_REFERENCE, Duration.ofSeconds(10))) {
             statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }
 
         try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(
-                TestIds.APPLICATION_INSTANCE_REFERENCE2)) {
+                TestIds.APPLICATION_INSTANCE_REFERENCE2, Duration.ofSeconds(10))) {
             statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }