Wrap CC HTTP failures in 409

9 files changed, 91 insertions, 159 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
index 25c6c780130..f1f572621ce 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
@@ -18,15 +18,18 @@ import java.util.Optional;
 public class OrchestratorContext {
     private static final Duration DEFAULT_TIMEOUT_FOR_SINGLE_OP = Duration.ofSeconds(10);
     private static final Duration DEFAULT_TIMEOUT_FOR_BATCH_OP = Duration.ofSeconds(60);
+    private static final Duration TIMEOUT_OVERHEAD = Duration.ofMillis(500);
 
     private final Clock clock;
     private final TimeBudget timeBudget;
     private boolean probe;
 
+    /** Create an OrchestratorContext for operations on multiple applications. */
     public static OrchestratorContext createContextForMultiAppOp(Clock clock) {
         return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), true);
     }
 
+    /** Create an OrchestratorContext for an operation on a single application. */
     public static OrchestratorContext createContextForSingleAppOp(Clock clock) {
         return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), true);
     }
@@ -41,8 +44,8 @@ public class OrchestratorContext {
         return timeBudget.timeLeftOrThrow().get();
     }
 
-    public ClusterControllerClientTimeouts getClusterControllerTimeouts(String clusterName) {
-        return new ClusterControllerClientTimeouts(clusterName, timeBudget.timeLeftAsTimeBudget());
+    public ClusterControllerClientTimeouts getClusterControllerTimeouts() {
+        return new ClusterControllerClientTimeouts(timeBudget.timeLeftAsTimeBudget());
     }
 
 
@@ -57,8 +60,16 @@ public class OrchestratorContext {
         return probe;
     }
 
-    /** Create an OrchestratorContext to use within an application lock. */
-    public OrchestratorContext createSubcontextForApplication() {
+    /** Create OrchestratorContext to use within an application lock. */
+    public OrchestratorContext createSubcontextWithinLock() {
+        // Move deadline towards past by a fixed amount to ensure there's time to process exceptions and
+        // access ZooKeeper before the lock times out.
+        TimeBudget subTimeBudget = timeBudget.withDeadline(timeBudget.deadline().get().minus(TIMEOUT_OVERHEAD));
+        return new OrchestratorContext(clock, subTimeBudget, probe);
+    }
+
+    /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */
+    public OrchestratorContext createSubcontextForSingleAppOp() {
         Instant now = clock.instant();
         Instant deadline = timeBudget.deadline().get();
         Instant maxDeadline = now.plus(DEFAULT_TIMEOUT_FOR_SINGLE_OP);
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 3a2673d9dbc..6811788ffb7 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -141,7 +141,7 @@ public class OrchestratorImpl implements Orchestrator {
 
             ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(appInstance.reference()).getApplicationInstanceStatus();
             if (appStatus == ApplicationInstanceStatus.NO_REMARKS) {
-                policy.releaseSuspensionGrant(context, appInstance, hostName, statusRegistry);
+                policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry);
             }
         }
     }
@@ -167,7 +167,7 @@ public class OrchestratorImpl implements Orchestrator {
                     statusRegistry,
                     clusterControllerClientFactory);
 
-            policy.acquirePermissionToRemove(context, applicationApi);
+            policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi);
         }
     }
 
@@ -193,7 +193,7 @@ public class OrchestratorImpl implements Orchestrator {
                     nodeGroup,
                     hostStatusRegistry,
                     clusterControllerClientFactory);
-            policy.grantSuspensionRequest(context, applicationApi);
+            policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi);
         }
     }
 
@@ -232,7 +232,7 @@ public class OrchestratorImpl implements Orchestrator {
         OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock);
         for (NodeGroup nodeGroup : nodeGroupsOrderedByApplication) {
             try {
-                suspendGroup(context.createSubcontextForApplication(), nodeGroup);
+                suspendGroup(context.createSubcontextForSingleAppOp(), nodeGroup);
             } catch (HostStateChangeDeniedException e) {
                 throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e);
             } catch (RuntimeException e) {
@@ -324,7 +324,7 @@ public class OrchestratorImpl implements Orchestrator {
 
                 // If the clustercontroller throws an error the nodes will be marked as allowed to be down
                 // and be set back up on next resume invocation.
-                setClusterStateInController(context, application, ClusterControllerNodeState.MAINTENANCE);
+                setClusterStateInController(context.createSubcontextWithinLock(), application, ClusterControllerNodeState.MAINTENANCE);
             }
 
             statusRegistry.setApplicationInstanceStatus(status);
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
index 46440682fa0..50ffe13b437 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java
@@ -36,7 +36,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{
                                                        ClusterControllerNodeState wantedState) throws IOException {
         ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON);
         ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.SAFE);
-        ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(clusterName);
+        ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts();
 
         try {
             return clusterControllerApi.apply(api -> api.setNodeState(
@@ -68,7 +68,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{
             ClusterControllerNodeState wantedState) throws IOException {
         ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON);
         ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.FORCE);
-        ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(clusterName);
+        ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts();
 
         try {
             return clusterControllerApi.apply(api -> api.setClusterState(
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java
index 44ecc7ac167..22f2440b408 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java
@@ -16,63 +16,29 @@ import java.time.Duration;
  * the theoretical max time is: T = C + R * NR + TW. With both NR and TW unknown, there's no way to
  * set a proper C and R.</p>
  *
- * <p>The various timeouts is set according to the following considerations:</p>
- *
- * <ol>
- *     <li>Some time is reserved for the execution in this process, e.g. execution leading to the REST call,
- *     handling of the response, exception handling, etc, such that we can finish processing this request
- *     before the {@link #timeBudget} deadline. This is typically in the order of ms.</li>
- *     <li>A timeout will be passed to the Cluster Controller backend. We'll give a timeout such that if one
- *     CC times out, the next CC will be given exactly the same timeout. This may or may not be a good strategy:
- *     (A) There's typically a 3rd CC. But if the first 2 fails with timeout, the chance the last is OK
- *     is negligible. (B) If picking the CC is random, then giving the full timeout to the first
- *     should be sufficient since a later retry will hit the healthy CC. (C) Because we have been using
- *     DROP in networking rules, clients may time out (host out of app or whatever). This would suggest
- *     allowing more than 1 full request.</li>
- *     <li>The timeout passed to the CC backend should be such that if it honors that, the Orchestrator
- *     should not time out. This means some kernel and network overhead should be subtracted from the timeout
- *     passed to the CC.</li>
- *     <li>We're only able to set the connect and read/write timeouts(!) Since we're communicating within
- *     data center, assume connections are in the order of ms, while a single read may stall close up to the CC
- *     timeout.</li>
- * </ol>
- *
  * @author hakonhall
  */
 public class ClusterControllerClientTimeouts implements JaxRsTimeouts {
-    // In data center connect timeout
     static final Duration CONNECT_TIMEOUT = Duration.ofMillis(100);
-    // Per call overhead
-    static final Duration IN_PROCESS_OVERHEAD_PER_CALL = Duration.ofMillis(100);
-    // In-process kernel overhead, network overhead, server kernel overhead, and server in-process overhead.
-    static final Duration DOWNSTREAM_OVERHEAD_PER_CALL = CONNECT_TIMEOUT.plus(Duration.ofMillis(100));
-    // Minimum time reserved for post-RPC processing to finish BEFORE the deadline, including ZK write.
-    static final Duration IN_PROCESS_OVERHEAD = Duration.ofMillis(100);
-    // Number of JAX-RS RPC calls to account for within the time budget.
-    static final int NUM_CALLS = 2;
+    // Time reserved to guarantee that even though the server application honors a server timeout S,
+    // some time will pass before the server sees the timeout, and after it has returned.
+    static final Duration DOWNSTREAM_OVERHEAD = Duration.ofMillis(300);
     // Minimum server-side timeout
-    static final Duration MIN_SERVER_TIMEOUT = Duration.ofMillis(10);
+    static final Duration MIN_SERVER_TIMEOUT = Duration.ofMillis(100);
 
-    private final String clusterName;
     private final TimeBudget timeBudget;
-    private final Duration maxClientTimeout;
 
     /**
      * Creates a timeouts instance.
      *
      * The {@link #timeBudget} SHOULD be the time budget for a single logical call to the Cluster Controller.
-     * A logical call to CC may in fact call the CC several times, if the first onces are down and/or not
+     * A logical call to CC may in fact call the CC several times, if the first ones are down and/or not
      * the master.
      *
-     * @param clusterName The name of the content cluster this request is for.
      * @param timeBudget  The time budget for a single logical call to the the Cluster Controller.
      */
-    public ClusterControllerClientTimeouts(String clusterName, TimeBudget timeBudget) {
-        this.clusterName = clusterName;
+    public ClusterControllerClientTimeouts(TimeBudget timeBudget) {
         this.timeBudget = timeBudget;
-
-        // timeLeft = inProcessOverhead + numCalls * clientTimeout
-        maxClientTimeout = timeBudget.originalTimeout().get().minus(IN_PROCESS_OVERHEAD).dividedBy(NUM_CALLS);
     }
 
     @Override
@@ -83,43 +49,26 @@ public class ClusterControllerClientTimeouts implements JaxRsTimeouts {
     @Override
     public Duration getReadTimeoutOrThrow() {
         Duration timeLeft = timeBudget.timeLeft().get();
-        if (timeLeft.toMillis() <= 0) {
-            throw new UncheckedTimeoutException("Exceeded the timeout " + timeBudget.originalTimeout().get() +
-                    " against content cluster '" + clusterName + "' by " + timeLeft.negated());
-        }
 
-        Duration clientTimeout = min(timeLeft, maxClientTimeout);
-        verifyPositive(timeLeft, clientTimeout);
+        // timeLeft = CONNECT_TIMEOUT + readTimeout
+        Duration readTimeout = timeLeft.minus(CONNECT_TIMEOUT);
 
-        // clientTimeout = overheadPerCall + connectTimeout + readTimeout
-        Duration readTimeout = clientTimeout.minus(IN_PROCESS_OVERHEAD_PER_CALL).minus(CONNECT_TIMEOUT);
-        verifyPositive(timeLeft, readTimeout);
+        if (readTimeout.toMillis() <= 0) {
+            throw new UncheckedTimeoutException("Timed out after " + timeBudget.originalTimeout().get());
+        }
 
         return readTimeout;
     }
 
     public Duration getServerTimeoutOrThrow() {
-        // readTimeout = networkOverhead + serverTimeout
-        Duration serverTimeout = getReadTimeoutOrThrow().minus(DOWNSTREAM_OVERHEAD_PER_CALL);
+        // readTimeout = DOWNSTREAM_OVERHEAD + serverTimeout
+        Duration serverTimeout = getReadTimeoutOrThrow().minus(DOWNSTREAM_OVERHEAD);
+
         if (serverTimeout.toMillis() < MIN_SERVER_TIMEOUT.toMillis()) {
-            throw new UncheckedTimeoutException("Server would be given too little time to complete: " +
-                    serverTimeout + ". Original timeout was " + timeBudget.originalTimeout().get());
+            throw new UncheckedTimeoutException("Timed out after " + timeBudget.originalTimeout().get());
         }
 
         return serverTimeout;
     }
 
-    private static Duration min(Duration a, Duration b) {
-        return a.compareTo(b) < 0 ? a : b;
-    }
-
-    private void verifyLargerThan(Duration timeLeft, Duration availableDuration) {
-        if (availableDuration.toMillis() <= 0) {
-            throw new UncheckedTimeoutException("Too little time left (" + timeLeft +
-                    ") to call content cluster '" + clusterName +
-                    "', original timeout was " + timeBudget.originalTimeout().get());
-        }
-    }
-
-    private void verifyPositive(Duration timeLeft, Duration duration) { verifyLargerThan(timeLeft, duration); }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java
new file mode 100644
index 00000000000..042eca0040e
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.orchestrator.controller;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Error response from cluster controller.
+ *
+ * @author hakonhall
+ */
+public class ClusterControllerStateErrorResponse {
+    @JsonProperty("message")
+    public final String message;
+
+    @JsonCreator
+    public ClusterControllerStateErrorResponse(@JsonProperty("message") String message) {
+        this.message = message;
+    }
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
index 3e387012d2c..9900c8de752 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
@@ -12,10 +12,13 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateErrorResponse;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
 
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
@@ -75,6 +78,17 @@ public class StorageNodeImpl implements StorageNode {
                     HostedVespaPolicy.CLUSTER_CONTROLLER_AVAILABLE_CONSTRAINT,
                     "Failed to communicate with cluster controllers " + clusterControllers + ": " + e,
                     e);
+        } catch (WebApplicationException e) {
+            Response webResponse = e.getResponse();
+            // Response may contain detail message
+            ClusterControllerStateErrorResponse errorResponse = webResponse.readEntity(ClusterControllerStateErrorResponse.class);
+            String detail = errorResponse.message == null ? "" : ": " + errorResponse.message;
+            throw new HostStateChangeDeniedException(
+                    hostName(),
+                    HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT,
+                    "Failure from cluster controllers " + clusterControllers + " when setting node " + nodeIndex +
+                            " in cluster " + clusterId + " to state " + wantedNodeState + detail,
+                    e);
         } catch (UncheckedTimeoutException e) {
             throw new HostStateChangeDeniedException(
                     hostName(),
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
index b12cd5aa7be..35dda403aed 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java
@@ -8,7 +8,6 @@ import org.junit.Test;
 
 import java.time.Duration;
 
-import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -31,7 +30,7 @@ public class ClusterControllerClientTest {
 
         OrchestratorContext context = mock(OrchestratorContext.class);
         ClusterControllerClientTimeouts timeouts = mock(ClusterControllerClientTimeouts.class);
-        when(context.getClusterControllerTimeouts(any())).thenReturn(timeouts);
+        when(context.getClusterControllerTimeouts()).thenReturn(timeouts);
         when(timeouts.getServerTimeoutOrThrow()).thenReturn(Duration.ofSeconds(1));
         clusterControllerClient.setNodeState(context, STORAGE_NODE_INDEX, wantedState);
 
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java
index ee81a89d76c..63b8e498f16 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java
@@ -10,30 +10,18 @@ import java.time.Duration;
 import java.util.Optional;
 
 import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.CONNECT_TIMEOUT;
-import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD;
-import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD_PER_CALL;
+import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.DOWNSTREAM_OVERHEAD;
 import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.MIN_SERVER_TIMEOUT;
-import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.DOWNSTREAM_OVERHEAD_PER_CALL;
-import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.NUM_CALLS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 public class ClusterControllerClientTimeoutsTest {
-    // The minimum time left that allows for a single RPC to CC.
-    private static final Duration MINIMUM_TIME_LEFT = IN_PROCESS_OVERHEAD_PER_CALL
-            .plus(CONNECT_TIMEOUT)
-            .plus(DOWNSTREAM_OVERHEAD_PER_CALL)
+    // The minimum time that allows for a single RPC to CC.
+    private static final Duration MINIMUM_TIME_LEFT = CONNECT_TIMEOUT
+            .plus(DOWNSTREAM_OVERHEAD)
             .plus(MIN_SERVER_TIMEOUT);
     static {
-        assertEquals(Duration.ofMillis(410), MINIMUM_TIME_LEFT);
-    }
-
-    // The minimum time left (= original time) that allows for NUM_CALLS RPCs to CC.
-    private static final Duration MINIMUM_ORIGINAL_TIMEOUT = MINIMUM_TIME_LEFT
-            .multipliedBy(NUM_CALLS)
-            .plus(IN_PROCESS_OVERHEAD);
-    static {
-        assertEquals(Duration.ofMillis(920), MINIMUM_ORIGINAL_TIMEOUT);
+        assertEquals(Duration.ofMillis(500), MINIMUM_TIME_LEFT);
     }
 
     private final ManualClock clock = new ManualClock();
@@ -45,7 +33,7 @@ public class ClusterControllerClientTimeoutsTest {
     private void makeTimeouts(Duration originalTimeout) {
         this.originalTimeout = originalTimeout;
         this.timeBudget = TimeBudget.from(clock, clock.instant(), Optional.of(originalTimeout));
-        this.timeouts = new ClusterControllerClientTimeouts("clustername", timeBudget);
+        this.timeouts = new ClusterControllerClientTimeouts(timeBudget);
     }
 
     @Before
@@ -54,47 +42,22 @@ public class ClusterControllerClientTimeoutsTest {
     }
 
     @Test
-    public void makes2RequestsWithMaxProcessingTime() {
-        assertStandardTimeouts();
-
-        Duration maxProcessingTime = IN_PROCESS_OVERHEAD_PER_CALL
-                .plus(CONNECT_TIMEOUT)
-                .plus(timeouts.getReadTimeoutOrThrow());
-        assertEquals(1450, maxProcessingTime.toMillis());
-        clock.advance(maxProcessingTime);
-
-        assertStandardTimeouts();
-
-        clock.advance(maxProcessingTime);
-
-        try {
-            timeouts.getServerTimeoutOrThrow();
-            fail();
-        } catch (UncheckedTimeoutException e) {
-            assertEquals(
-                    "Too little time left (PT0.1S) to call content cluster 'clustername', original timeout was PT3S",
-                    e.getMessage());
-        }
-    }
-
-    @Test
-    public void makesAtLeast3RequestsWithShortProcessingTime() {
-        assertStandardTimeouts();
-
-        Duration shortProcessingTime = Duration.ofMillis(200);
-        clock.advance(shortProcessingTime);
+    public void makesManyRequestsWithShortProcessingTime() {
+        assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2900), timeouts.getReadTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2600), timeouts.getServerTimeoutOrThrow());
 
-        assertStandardTimeouts();
+        clock.advance(Duration.ofMillis(100));
 
-        clock.advance(shortProcessingTime);
+        assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2800), timeouts.getReadTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2500), timeouts.getServerTimeoutOrThrow());
 
-        assertStandardTimeouts();
-    }
+        clock.advance(Duration.ofMillis(100));
 
-    private void assertStandardTimeouts() {
         assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow());
-        assertEquals(Duration.ofMillis(1250), timeouts.getReadTimeoutOrThrow());
-        assertEquals(Duration.ofMillis(1050), timeouts.getServerTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2700), timeouts.getReadTimeoutOrThrow());
+        assertEquals(Duration.ofMillis(2400), timeouts.getServerTimeoutOrThrow());
     }
 
     @Test
@@ -105,9 +68,7 @@ public class ClusterControllerClientTimeoutsTest {
             timeouts.getServerTimeoutOrThrow();
             fail();
         } catch (UncheckedTimeoutException e) {
-            assertEquals(
-                    "Exceeded the timeout PT3S against content cluster 'clustername' by PT1S",
-                    e.getMessage());
+            assertEquals("Timed out after PT3S", e.getMessage());
         }
     }
 
@@ -118,9 +79,7 @@ public class ClusterControllerClientTimeoutsTest {
             timeouts.getServerTimeoutOrThrow();
             fail();
         } catch (UncheckedTimeoutException e) {
-            assertEquals(
-                    "Server would be given too little time to complete: PT0.009S. Original timeout was PT3S",
-                    e.getMessage());
+            assertEquals("Timed out after PT3S", e.getMessage());
         }
     }
 
@@ -129,23 +88,4 @@ public class ClusterControllerClientTimeoutsTest {
         clock.advance(originalTimeout.minus(MINIMUM_TIME_LEFT));
         timeouts.getServerTimeoutOrThrow();
     }
-
-    @Test
-    public void justTooLittleInitialTime() {
-        makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT.minus(Duration.ofMillis(1)));
-        try {
-            timeouts.getServerTimeoutOrThrow();
-            fail();
-        } catch (UncheckedTimeoutException e) {
-            assertEquals(
-                    "Server would be given too little time to complete: PT0.0095S. Original timeout was PT0.919S",
-                    e.getMessage());
-        }
-    }
-
-    @Test
-    public void justEnoughInitialTime() {
-        makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT);
-        timeouts.getServerTimeoutOrThrow();
-    }
 }
\ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java
index 3b0b1a43085..f47b43fa27b 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java
@@ -44,7 +44,7 @@ public class RetryingClusterControllerClientFactoryTest {
 
         ArgumentCaptor<ClusterControllerStateRequest> requestCaptor = ArgumentCaptor.forClass(ClusterControllerStateRequest.class);
 
-        verify(api, times(1)).setNodeState(eq(clusterName), eq(storageNode), eq(4.55f), requestCaptor.capture());
+        verify(api, times(1)).setNodeState(eq(clusterName), eq(storageNode), eq(9.6f), requestCaptor.capture());
         ClusterControllerStateRequest request = requestCaptor.getValue();
         assertEquals(ClusterControllerStateRequest.Condition.SAFE, request.condition);
         Map<String, Object> expectedState = new HashMap<>();