diff options
Diffstat (limited to 'vespa-feed-client/src/main/java/ai/vespa/feed')
6 files changed, 34 insertions, 51 deletions
diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 424481b6ef2..c271ac356e9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -18,7 +18,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.LongSupplier; import java.util.function.Supplier; import static ai.vespa.feed.client.FeedClientBuilder.Compression.auto; @@ -59,7 +58,9 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { URI proxy; Duration connectionTtl = Duration.ZERO; - public FeedClientBuilderImpl() { } + + public FeedClientBuilderImpl() { + } FeedClientBuilderImpl(List<URI> endpoints) { this(); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/GracePeriodCircuitBreaker.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/GracePeriodCircuitBreaker.java index cec7106403e..1ea2089c0eb 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/GracePeriodCircuitBreaker.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/GracePeriodCircuitBreaker.java @@ -12,6 +12,7 @@ import java.util.function.LongSupplier; import java.util.logging.Logger; import static java.util.Objects.requireNonNull; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; @@ -23,22 +24,22 @@ import static java.util.logging.Level.WARNING; public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { private static final Logger log = Logger.getLogger(GracePeriodCircuitBreaker.class.getName()); + private static final long NEVER = 1L << 60; + private final AtomicLong failingSinceMillis = new AtomicLong(NEVER); private final AtomicBoolean halfOpen = new AtomicBoolean(false); private final AtomicBoolean open = new AtomicBoolean(false); - private final LongSupplier nanoClock; - private final long never; - private final AtomicLong failingSinceNanos; + private final LongSupplier clock; private final AtomicReference<String> detail = new AtomicReference<>(); - private final long graceNanos; - private final long doomNanos; + private final long graceMillis; + private final long doomMillis; /** * Creates a new circuit breaker with the given grace periods. * @param grace the period of consecutive failures before state changes to half-open. */ public GracePeriodCircuitBreaker(Duration grace) { - this(System::nanoTime, grace, null); + this(System::currentTimeMillis, grace, null); } /** @@ -47,25 +48,23 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { * @param doom the period of consecutive failures before shutting down. */ public GracePeriodCircuitBreaker(Duration grace, Duration doom) { - this(System::nanoTime, grace, doom); + this(System::currentTimeMillis, grace, doom); if (doom.isNegative()) throw new IllegalArgumentException("Doom delay must be non-negative"); } - GracePeriodCircuitBreaker(LongSupplier nanoClock, Duration grace, Duration doom) { + GracePeriodCircuitBreaker(LongSupplier clock, Duration grace, Duration doom) { if (grace.isNegative()) throw new IllegalArgumentException("Grace delay must be non-negative"); - this.nanoClock = requireNonNull(nanoClock); - this.never = nanoClock.getAsLong() + (1L << 60); - this.graceNanos = grace.toNanos(); - this.doomNanos = doom == null ? -1 : doom.toNanos(); - this.failingSinceNanos = new AtomicLong(never); + this.clock = requireNonNull(clock); + this.graceMillis = grace.toMillis(); + this.doomMillis = doom == null ? -1 : doom.toMillis(); } @Override public void success() { - failingSinceNanos.set(never); + failingSinceMillis.set(NEVER); if ( ! open.get() && halfOpen.compareAndSet(true, false)) log.log(INFO, "Circuit breaker is now closed, after a request was successful"); } @@ -81,21 +80,21 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { } private void failure(String detail) { - if (failingSinceNanos.compareAndSet(never, nanoClock.getAsLong())) + if (failingSinceMillis.compareAndSet(NEVER, clock.getAsLong())) this.detail.set(detail); } @Override public State state() { - long failingNanos = nanoClock.getAsLong() - failingSinceNanos.get(); - if (failingNanos > graceNanos && halfOpen.compareAndSet(false, true)) + long failingMillis = clock.getAsLong() - failingSinceMillis.get(); + if (failingMillis > graceMillis && halfOpen.compareAndSet(false, true)) log.log(INFO, "Circuit breaker is now half-open, as no requests have succeeded for the " + - "last " + failingNanos / 1_000_000 + "ms. The server will be pinged to see if it recovers" + - (doomNanos >= 0 ? ", but this client will give up if no successes are observed within " + doomNanos / 1_000_000 + "ms" : "") + + "last " + failingMillis + "ms. The server will be pinged to see if it recovers" + + (doomMillis >= 0 ? ", but this client will give up if no successes are observed within " + doomMillis + "ms" : "") + ". First failure was '" + detail.get() + "'."); - if (doomNanos >= 0 && failingNanos > doomNanos && open.compareAndSet(false, true)) - log.log(WARNING, "Circuit breaker is now open, after " + doomNanos / 1_000_000 + "ms of failing request, " + + if (doomMillis >= 0 && failingMillis > doomMillis && open.compareAndSet(false, true)) + log.log(WARNING, "Circuit breaker is now open, after " + doomMillis + "ms of failing request, " + "and this client will give up and abort its remaining feed operations."); return open.get() ? State.OPEN : halfOpen.get() ? State.HALF_OPEN : State.CLOSED; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java index 5eb611160cc..9dd11113c0b 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java @@ -23,6 +23,7 @@ import java.time.Duration; import java.time.Instant; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -30,7 +31,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.LongSupplier; import java.util.function.Supplier; import static ai.vespa.feed.client.OperationParameters.empty; @@ -45,7 +45,6 @@ import static java.util.Objects.requireNonNull; */ class HttpFeedClient implements FeedClient { - private static final Duration maxTimeout = Duration.ofMinutes(15); private static final JsonFactory jsonParserFactory = new JsonFactoryBuilder() .streamReadConstraints(StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()) .build(); @@ -112,8 +111,7 @@ class HttpFeedClient implements FeedClient { getPath(documentId) + getQuery(params, speedTest), requestHeaders, operationJson == null ? null : operationJson.getBytes(UTF_8), // TODO: make it bytes all the way? - params.timeout().orElse(maxTimeout), - System::nanoTime); + params.timeout().orElse(null)); CompletableFuture<Result> promise = new CompletableFuture<>(); requestStrategy.enqueue(documentId, request) @@ -138,8 +136,7 @@ class HttpFeedClient implements FeedClient { getPath(DocumentId.of("feeder", "handshake", "dummy")) + getQuery(empty(), true), requestHeaders, null, - Duration.ofSeconds(15), - System::nanoTime); + Duration.ofSeconds(15)); CompletableFuture<HttpResponse> future = new CompletableFuture<>(); cluster.dispatch(request, future); HttpResponse response = future.get(20, TimeUnit.SECONDS); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java index fdd35b74c35..6de3f034f22 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java @@ -3,7 +3,6 @@ package ai.vespa.feed.client.impl; import java.time.Duration; import java.util.Map; -import java.util.function.LongSupplier; import java.util.function.Supplier; class HttpRequest { @@ -13,17 +12,13 @@ class HttpRequest { private final Map<String, Supplier<String>> headers; private final byte[] body; private final Duration timeout; - private final long deadlineNanos; - private final LongSupplier nanoClock; - public HttpRequest(String method, String path, Map<String, Supplier<String>> headers, byte[] body, Duration timeout, LongSupplier nanoClock) { + public HttpRequest(String method, String path, Map<String, Supplier<String>> headers, byte[] body, Duration timeout) { this.method = method; this.path = path; this.headers = headers; this.body = body; - this.deadlineNanos = nanoClock.getAsLong() + timeout.toNanos(); this.timeout = timeout; - this.nanoClock = nanoClock; } public String method() { @@ -42,10 +37,6 @@ class HttpRequest { return body; } - public Duration timeLeft() { - return Duration.ofNanos(deadlineNanos - nanoClock.getAsLong()); - } - public Duration timeout() { return timeout; } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java index f699651634a..8f0327a1738 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java @@ -31,7 +31,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.FINE; import static java.util.logging.Level.FINER; import static java.util.logging.Level.FINEST; -import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; /** @@ -97,7 +96,7 @@ class HttpRequestStrategy implements RequestStrategy { } } catch (Throwable t) { - log.log(SEVERE, "Dispatch thread threw; shutting down", t); + log.log(WARNING, "Dispatch thread threw; shutting down", t); } destroy(); } @@ -120,7 +119,7 @@ class HttpRequestStrategy implements RequestStrategy { } private boolean retry(HttpRequest request, int attempt) { - if (attempt > strategy.retries() || request.timeLeft().toMillis() <= 0) + if (attempt > strategy.retries()) return false; switch (request.method().toUpperCase()) { @@ -138,6 +137,7 @@ class HttpRequestStrategy implements RequestStrategy { private boolean retry(HttpRequest request, Throwable thrown, int attempt) { breaker.failure(thrown); if ( (thrown instanceof IOException) // General IO problems. + // Thrown by HTTP2Session.StreamsState.reserveSlot, likely on GOAWAY from server || (thrown instanceof IllegalStateException && thrown.getMessage().equals("session closed")) ) { @@ -149,7 +149,7 @@ class HttpRequestStrategy implements RequestStrategy { return false; } - /** Retries throttled requests (429), adjusting the target inflight count, and server unavailable (503). */ + /** Retries throttled requests (429), adjusting the target inflight count, and server errors (500, 502, 503, 504). */ private boolean retry(HttpRequest request, HttpResponse response, int attempt) { if (response.code() / 100 == 2 || response.code() == 404 || response.code() == 412) { logResponse(FINEST, response, request, attempt); @@ -272,8 +272,7 @@ class HttpRequestStrategy implements RequestStrategy { } /** Handles the result of one attempt at the given operation, retrying if necessary. */ - private void handleAttempt(CompletableFuture<HttpResponse> vessel, HttpRequest request, - RetriableFuture<HttpResponse> result, int attempt) { + private void handleAttempt(CompletableFuture<HttpResponse> vessel, HttpRequest request, RetriableFuture<HttpResponse> result, int attempt) { vessel.whenCompleteAsync((response, thrown) -> { result.set(response, thrown); // Retry the operation if it failed with a transient error ... diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java index 18369f29f0b..df010a167f6 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java @@ -45,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -82,11 +81,8 @@ class JettyCluster implements Cluster { Endpoint endpoint = findLeastBusyEndpoint(endpoints); try { endpoint.inflight.incrementAndGet(); - long reqTimeoutMillis = req.timeLeft().toMillis(); - if (reqTimeoutMillis <= 0) { - vessel.completeExceptionally(new TimeoutException("operation timed out after '" + req.timeout() + "'")); - return; - } + long reqTimeoutMillis = req.timeout() != null + ? req.timeout().toMillis() * 11 / 10 + 1000 : IDLE_TIMEOUT.toMillis(); Request jettyReq = client.newRequest(URI.create(endpoint.uri + req.path())) .version(HttpVersion.HTTP_2) .method(HttpMethod.fromString(req.method())) |