diff options
author | Håkon Hallingstad <hakon@oath.com> | 2018-11-01 23:57:14 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2018-11-01 23:57:14 +0100 |
commit | ddd668831dd89d6a6d5cc89203c6acbf1d00032c (patch) | |
tree | 7263e95ec856d05da04a6ac7fd9622d10c451ac3 /orchestrator | |
parent | 23f8367787ac42870bd49b2d633f7c8b872c2695 (diff) |
Wrap CC HTTP failures in 409
Diffstat (limited to 'orchestrator')
9 files changed, 91 insertions, 159 deletions
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index 25c6c780130..f1f572621ce 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -18,15 +18,18 @@ import java.util.Optional; public class OrchestratorContext { private static final Duration DEFAULT_TIMEOUT_FOR_SINGLE_OP = Duration.ofSeconds(10); private static final Duration DEFAULT_TIMEOUT_FOR_BATCH_OP = Duration.ofSeconds(60); + private static final Duration TIMEOUT_OVERHEAD = Duration.ofMillis(500); private final Clock clock; private final TimeBudget timeBudget; private boolean probe; + /** Create an OrchestratorContext for operations on multiple applications. */ public static OrchestratorContext createContextForMultiAppOp(Clock clock) { return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), true); } + /** Create an OrchestratorContext for an operation on a single application. */ public static OrchestratorContext createContextForSingleAppOp(Clock clock) { return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), true); } @@ -41,8 +44,8 @@ public class OrchestratorContext { return timeBudget.timeLeftOrThrow().get(); } - public ClusterControllerClientTimeouts getClusterControllerTimeouts(String clusterName) { - return new ClusterControllerClientTimeouts(clusterName, timeBudget.timeLeftAsTimeBudget()); + public ClusterControllerClientTimeouts getClusterControllerTimeouts() { + return new ClusterControllerClientTimeouts(timeBudget.timeLeftAsTimeBudget()); } @@ -57,8 +60,16 @@ public class OrchestratorContext { return probe; } - /** Create an OrchestratorContext to use within an application lock. */ - public OrchestratorContext createSubcontextForApplication() { + /** Create OrchestratorContext to use within an application lock. */ + public OrchestratorContext createSubcontextWithinLock() { + // Move deadline towards past by a fixed amount to ensure there's time to process exceptions and + // access ZooKeeper before the lock times out. + TimeBudget subTimeBudget = timeBudget.withDeadline(timeBudget.deadline().get().minus(TIMEOUT_OVERHEAD)); + return new OrchestratorContext(clock, subTimeBudget, probe); + } + + /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */ + public OrchestratorContext createSubcontextForSingleAppOp() { Instant now = clock.instant(); Instant deadline = timeBudget.deadline().get(); Instant maxDeadline = now.plus(DEFAULT_TIMEOUT_FOR_SINGLE_OP); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 3a2673d9dbc..6811788ffb7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -141,7 +141,7 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(appInstance.reference()).getApplicationInstanceStatus(); if (appStatus == ApplicationInstanceStatus.NO_REMARKS) { - policy.releaseSuspensionGrant(context, appInstance, hostName, statusRegistry); + policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); } } } @@ -167,7 +167,7 @@ public class OrchestratorImpl implements Orchestrator { statusRegistry, clusterControllerClientFactory); - policy.acquirePermissionToRemove(context, applicationApi); + policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi); } } @@ -193,7 +193,7 @@ public class OrchestratorImpl implements Orchestrator { nodeGroup, hostStatusRegistry, clusterControllerClientFactory); - policy.grantSuspensionRequest(context, applicationApi); + policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi); } } @@ -232,7 +232,7 @@ public class OrchestratorImpl implements Orchestrator { OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock); for (NodeGroup nodeGroup : nodeGroupsOrderedByApplication) { try { - suspendGroup(context.createSubcontextForApplication(), nodeGroup); + suspendGroup(context.createSubcontextForSingleAppOp(), nodeGroup); } catch (HostStateChangeDeniedException e) { throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e); } catch (RuntimeException e) { @@ -324,7 +324,7 @@ public class OrchestratorImpl implements Orchestrator { // If the clustercontroller throws an error the nodes will be marked as allowed to be down // and be set back up on next resume invocation. - setClusterStateInController(context, application, ClusterControllerNodeState.MAINTENANCE); + setClusterStateInController(context.createSubcontextWithinLock(), application, ClusterControllerNodeState.MAINTENANCE); } statusRegistry.setApplicationInstanceStatus(status); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 46440682fa0..50ffe13b437 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -36,7 +36,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ ClusterControllerNodeState wantedState) throws IOException { ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.SAFE); - ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(clusterName); + ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(); try { return clusterControllerApi.apply(api -> api.setNodeState( @@ -68,7 +68,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ ClusterControllerNodeState wantedState) throws IOException { ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.FORCE); - ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(clusterName); + ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(); try { return clusterControllerApi.apply(api -> api.setClusterState( diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java index 44ecc7ac167..22f2440b408 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java @@ -16,63 +16,29 @@ import java.time.Duration; * the theoretical max time is: T = C + R * NR + TW. With both NR and TW unknown, there's no way to * set a proper C and R.</p> * - * <p>The various timeouts is set according to the following considerations:</p> - * - * <ol> - * <li>Some time is reserved for the execution in this process, e.g. execution leading to the REST call, - * handling of the response, exception handling, etc, such that we can finish processing this request - * before the {@link #timeBudget} deadline. This is typically in the order of ms.</li> - * <li>A timeout will be passed to the Cluster Controller backend. We'll give a timeout such that if one - * CC times out, the next CC will be given exactly the same timeout. This may or may not be a good strategy: - * (A) There's typically a 3rd CC. But if the first 2 fails with timeout, the chance the last is OK - * is negligible. (B) If picking the CC is random, then giving the full timeout to the first - * should be sufficient since a later retry will hit the healthy CC. (C) Because we have been using - * DROP in networking rules, clients may time out (host out of app or whatever). This would suggest - * allowing more than 1 full request.</li> - * <li>The timeout passed to the CC backend should be such that if it honors that, the Orchestrator - * should not time out. This means some kernel and network overhead should be subtracted from the timeout - * passed to the CC.</li> - * <li>We're only able to set the connect and read/write timeouts(!) Since we're communicating within - * data center, assume connections are in the order of ms, while a single read may stall close up to the CC - * timeout.</li> - * </ol> - * * @author hakonhall */ public class ClusterControllerClientTimeouts implements JaxRsTimeouts { - // In data center connect timeout static final Duration CONNECT_TIMEOUT = Duration.ofMillis(100); - // Per call overhead - static final Duration IN_PROCESS_OVERHEAD_PER_CALL = Duration.ofMillis(100); - // In-process kernel overhead, network overhead, server kernel overhead, and server in-process overhead. - static final Duration DOWNSTREAM_OVERHEAD_PER_CALL = CONNECT_TIMEOUT.plus(Duration.ofMillis(100)); - // Minimum time reserved for post-RPC processing to finish BEFORE the deadline, including ZK write. - static final Duration IN_PROCESS_OVERHEAD = Duration.ofMillis(100); - // Number of JAX-RS RPC calls to account for within the time budget. - static final int NUM_CALLS = 2; + // Time reserved to guarantee that even though the server application honors a server timeout S, + // some time will pass before the server sees the timeout, and after it has returned. + static final Duration DOWNSTREAM_OVERHEAD = Duration.ofMillis(300); // Minimum server-side timeout - static final Duration MIN_SERVER_TIMEOUT = Duration.ofMillis(10); + static final Duration MIN_SERVER_TIMEOUT = Duration.ofMillis(100); - private final String clusterName; private final TimeBudget timeBudget; - private final Duration maxClientTimeout; /** * Creates a timeouts instance. * * The {@link #timeBudget} SHOULD be the time budget for a single logical call to the Cluster Controller. - * A logical call to CC may in fact call the CC several times, if the first onces are down and/or not + * A logical call to CC may in fact call the CC several times, if the first ones are down and/or not * the master. * - * @param clusterName The name of the content cluster this request is for. * @param timeBudget The time budget for a single logical call to the the Cluster Controller. */ - public ClusterControllerClientTimeouts(String clusterName, TimeBudget timeBudget) { - this.clusterName = clusterName; + public ClusterControllerClientTimeouts(TimeBudget timeBudget) { this.timeBudget = timeBudget; - - // timeLeft = inProcessOverhead + numCalls * clientTimeout - maxClientTimeout = timeBudget.originalTimeout().get().minus(IN_PROCESS_OVERHEAD).dividedBy(NUM_CALLS); } @Override @@ -83,43 +49,26 @@ public class ClusterControllerClientTimeouts implements JaxRsTimeouts { @Override public Duration getReadTimeoutOrThrow() { Duration timeLeft = timeBudget.timeLeft().get(); - if (timeLeft.toMillis() <= 0) { - throw new UncheckedTimeoutException("Exceeded the timeout " + timeBudget.originalTimeout().get() + - " against content cluster '" + clusterName + "' by " + timeLeft.negated()); - } - Duration clientTimeout = min(timeLeft, maxClientTimeout); - verifyPositive(timeLeft, clientTimeout); + // timeLeft = CONNECT_TIMEOUT + readTimeout + Duration readTimeout = timeLeft.minus(CONNECT_TIMEOUT); - // clientTimeout = overheadPerCall + connectTimeout + readTimeout - Duration readTimeout = clientTimeout.minus(IN_PROCESS_OVERHEAD_PER_CALL).minus(CONNECT_TIMEOUT); - verifyPositive(timeLeft, readTimeout); + if (readTimeout.toMillis() <= 0) { + throw new UncheckedTimeoutException("Timed out after " + timeBudget.originalTimeout().get()); + } return readTimeout; } public Duration getServerTimeoutOrThrow() { - // readTimeout = networkOverhead + serverTimeout - Duration serverTimeout = getReadTimeoutOrThrow().minus(DOWNSTREAM_OVERHEAD_PER_CALL); + // readTimeout = DOWNSTREAM_OVERHEAD + serverTimeout + Duration serverTimeout = getReadTimeoutOrThrow().minus(DOWNSTREAM_OVERHEAD); + if (serverTimeout.toMillis() < MIN_SERVER_TIMEOUT.toMillis()) { - throw new UncheckedTimeoutException("Server would be given too little time to complete: " + - serverTimeout + ". Original timeout was " + timeBudget.originalTimeout().get()); + throw new UncheckedTimeoutException("Timed out after " + timeBudget.originalTimeout().get()); } return serverTimeout; } - private static Duration min(Duration a, Duration b) { - return a.compareTo(b) < 0 ? a : b; - } - - private void verifyLargerThan(Duration timeLeft, Duration availableDuration) { - if (availableDuration.toMillis() <= 0) { - throw new UncheckedTimeoutException("Too little time left (" + timeLeft + - ") to call content cluster '" + clusterName + - "', original timeout was " + timeBudget.originalTimeout().get()); - } - } - - private void verifyPositive(Duration timeLeft, Duration duration) { verifyLargerThan(timeLeft, duration); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java new file mode 100644 index 00000000000..042eca0040e --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateErrorResponse.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.orchestrator.controller; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Error response from cluster controller. + * + * @author hakonhall + */ +public class ClusterControllerStateErrorResponse { + @JsonProperty("message") + public final String message; + + @JsonCreator + public ClusterControllerStateErrorResponse(@JsonProperty("message") String message) { + this.message = message; + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index 3e387012d2c..9900c8de752 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -12,10 +12,13 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateErrorResponse; import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; import java.io.IOException; import java.util.List; import java.util.Objects; @@ -75,6 +78,17 @@ public class StorageNodeImpl implements StorageNode { HostedVespaPolicy.CLUSTER_CONTROLLER_AVAILABLE_CONSTRAINT, "Failed to communicate with cluster controllers " + clusterControllers + ": " + e, e); + } catch (WebApplicationException e) { + Response webResponse = e.getResponse(); + // Response may contain detail message + ClusterControllerStateErrorResponse errorResponse = webResponse.readEntity(ClusterControllerStateErrorResponse.class); + String detail = errorResponse.message == null ? "" : ": " + errorResponse.message; + throw new HostStateChangeDeniedException( + hostName(), + HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT, + "Failure from cluster controllers " + clusterControllers + " when setting node " + nodeIndex + + " in cluster " + clusterId + " to state " + wantedNodeState + detail, + e); } catch (UncheckedTimeoutException e) { throw new HostStateChangeDeniedException( hostName(), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java index b12cd5aa7be..35dda403aed 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java @@ -8,7 +8,6 @@ import org.junit.Test; import java.time.Duration; -import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -31,7 +30,7 @@ public class ClusterControllerClientTest { OrchestratorContext context = mock(OrchestratorContext.class); ClusterControllerClientTimeouts timeouts = mock(ClusterControllerClientTimeouts.class); - when(context.getClusterControllerTimeouts(any())).thenReturn(timeouts); + when(context.getClusterControllerTimeouts()).thenReturn(timeouts); when(timeouts.getServerTimeoutOrThrow()).thenReturn(Duration.ofSeconds(1)); clusterControllerClient.setNodeState(context, STORAGE_NODE_INDEX, wantedState); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java index ee81a89d76c..63b8e498f16 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java @@ -10,30 +10,18 @@ import java.time.Duration; import java.util.Optional; import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.CONNECT_TIMEOUT; -import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD; -import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.IN_PROCESS_OVERHEAD_PER_CALL; +import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.DOWNSTREAM_OVERHEAD; import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.MIN_SERVER_TIMEOUT; -import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.DOWNSTREAM_OVERHEAD_PER_CALL; -import static com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts.NUM_CALLS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public class ClusterControllerClientTimeoutsTest { - // The minimum time left that allows for a single RPC to CC. - private static final Duration MINIMUM_TIME_LEFT = IN_PROCESS_OVERHEAD_PER_CALL - .plus(CONNECT_TIMEOUT) - .plus(DOWNSTREAM_OVERHEAD_PER_CALL) + // The minimum time that allows for a single RPC to CC. + private static final Duration MINIMUM_TIME_LEFT = CONNECT_TIMEOUT + .plus(DOWNSTREAM_OVERHEAD) .plus(MIN_SERVER_TIMEOUT); static { - assertEquals(Duration.ofMillis(410), MINIMUM_TIME_LEFT); - } - - // The minimum time left (= original time) that allows for NUM_CALLS RPCs to CC. - private static final Duration MINIMUM_ORIGINAL_TIMEOUT = MINIMUM_TIME_LEFT - .multipliedBy(NUM_CALLS) - .plus(IN_PROCESS_OVERHEAD); - static { - assertEquals(Duration.ofMillis(920), MINIMUM_ORIGINAL_TIMEOUT); + assertEquals(Duration.ofMillis(500), MINIMUM_TIME_LEFT); } private final ManualClock clock = new ManualClock(); @@ -45,7 +33,7 @@ public class ClusterControllerClientTimeoutsTest { private void makeTimeouts(Duration originalTimeout) { this.originalTimeout = originalTimeout; this.timeBudget = TimeBudget.from(clock, clock.instant(), Optional.of(originalTimeout)); - this.timeouts = new ClusterControllerClientTimeouts("clustername", timeBudget); + this.timeouts = new ClusterControllerClientTimeouts(timeBudget); } @Before @@ -54,47 +42,22 @@ public class ClusterControllerClientTimeoutsTest { } @Test - public void makes2RequestsWithMaxProcessingTime() { - assertStandardTimeouts(); - - Duration maxProcessingTime = IN_PROCESS_OVERHEAD_PER_CALL - .plus(CONNECT_TIMEOUT) - .plus(timeouts.getReadTimeoutOrThrow()); - assertEquals(1450, maxProcessingTime.toMillis()); - clock.advance(maxProcessingTime); - - assertStandardTimeouts(); - - clock.advance(maxProcessingTime); - - try { - timeouts.getServerTimeoutOrThrow(); - fail(); - } catch (UncheckedTimeoutException e) { - assertEquals( - "Too little time left (PT0.1S) to call content cluster 'clustername', original timeout was PT3S", - e.getMessage()); - } - } - - @Test - public void makesAtLeast3RequestsWithShortProcessingTime() { - assertStandardTimeouts(); - - Duration shortProcessingTime = Duration.ofMillis(200); - clock.advance(shortProcessingTime); + public void makesManyRequestsWithShortProcessingTime() { + assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2900), timeouts.getReadTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2600), timeouts.getServerTimeoutOrThrow()); - assertStandardTimeouts(); + clock.advance(Duration.ofMillis(100)); - clock.advance(shortProcessingTime); + assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2800), timeouts.getReadTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2500), timeouts.getServerTimeoutOrThrow()); - assertStandardTimeouts(); - } + clock.advance(Duration.ofMillis(100)); - private void assertStandardTimeouts() { assertEquals(Duration.ofMillis(100), timeouts.getConnectTimeoutOrThrow()); - assertEquals(Duration.ofMillis(1250), timeouts.getReadTimeoutOrThrow()); - assertEquals(Duration.ofMillis(1050), timeouts.getServerTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2700), timeouts.getReadTimeoutOrThrow()); + assertEquals(Duration.ofMillis(2400), timeouts.getServerTimeoutOrThrow()); } @Test @@ -105,9 +68,7 @@ public class ClusterControllerClientTimeoutsTest { timeouts.getServerTimeoutOrThrow(); fail(); } catch (UncheckedTimeoutException e) { - assertEquals( - "Exceeded the timeout PT3S against content cluster 'clustername' by PT1S", - e.getMessage()); + assertEquals("Timed out after PT3S", e.getMessage()); } } @@ -118,9 +79,7 @@ public class ClusterControllerClientTimeoutsTest { timeouts.getServerTimeoutOrThrow(); fail(); } catch (UncheckedTimeoutException e) { - assertEquals( - "Server would be given too little time to complete: PT0.009S. Original timeout was PT3S", - e.getMessage()); + assertEquals("Timed out after PT3S", e.getMessage()); } } @@ -129,23 +88,4 @@ public class ClusterControllerClientTimeoutsTest { clock.advance(originalTimeout.minus(MINIMUM_TIME_LEFT)); timeouts.getServerTimeoutOrThrow(); } - - @Test - public void justTooLittleInitialTime() { - makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT.minus(Duration.ofMillis(1))); - try { - timeouts.getServerTimeoutOrThrow(); - fail(); - } catch (UncheckedTimeoutException e) { - assertEquals( - "Server would be given too little time to complete: PT0.0095S. Original timeout was PT0.919S", - e.getMessage()); - } - } - - @Test - public void justEnoughInitialTime() { - makeTimeouts(MINIMUM_ORIGINAL_TIMEOUT); - timeouts.getServerTimeoutOrThrow(); - } }
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java index 3b0b1a43085..f47b43fa27b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java @@ -44,7 +44,7 @@ public class RetryingClusterControllerClientFactoryTest { ArgumentCaptor<ClusterControllerStateRequest> requestCaptor = ArgumentCaptor.forClass(ClusterControllerStateRequest.class); - verify(api, times(1)).setNodeState(eq(clusterName), eq(storageNode), eq(4.55f), requestCaptor.capture()); + verify(api, times(1)).setNodeState(eq(clusterName), eq(storageNode), eq(9.6f), requestCaptor.capture()); ClusterControllerStateRequest request = requestCaptor.getValue(); assertEquals(ClusterControllerStateRequest.Condition.SAFE, request.condition); Map<String, Object> expectedState = new HashMap<>(); |