Revert "Remove the http-utils module (now embedded in vespajlib)"

This reverts commit 7963cb974fdc63ae529b0c0895ea4ccc339b6a53.

18 files changed, 1166 insertions, 0 deletions

diff --git a/http-utils/OWNERS b/http-utils/OWNERS
new file mode 100644
index 00000000000..569bf1cc3a1
--- /dev/null
+++ b/http-utils/OWNERS
@@ -0,0 +1 @@
+bjorncs
diff --git a/http-utils/README.md b/http-utils/README.md
new file mode 100644
index 00000000000..714b7581ad4
--- /dev/null
+++ b/http-utils/README.md
@@ -0,0 +1,2 @@
+<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# Http utilities for Java
diff --git a/http-utils/pom.xml b/http-utils/pom.xml
new file mode 100644
index 00000000000..2a8ec1b9bb9
--- /dev/null
+++ b/http-utils/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>parent</artifactId>
+    <version>7-SNAPSHOT</version>
+    <relativePath>../parent/pom.xml</relativePath>
+  </parent>
+  <artifactId>http-utils</artifactId>
+  <packaging>jar</packaging>
+  <version>7-SNAPSHOT</version>
+
+  <properties>
+    <!-- vespa-http-client targets jdk8 and uses this library -->
+    <!-- TODO remove once vespa-http-client no longer builds against jdk8 -->
+    <maven.compiler.release>8</maven.compiler.release>
+  </properties>
+
+  <dependencies>
+    <!-- provided -->
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>security-utils</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <!-- Apache client artifacts are provided by the jdisc container and are therefore scoped as such -->
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <!-- test scope -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <jdkToolchain>
+            <version>${java.version}</version>
+          </jdkToolchain>
+          <source>${java.version}</source>
+          <target>${java.version}</target>
+          <showDeprecation>true</showDeprecation>
+          <compilerArgs>
+            <arg>-Xlint:all</arg>
+            <arg>-Xlint:-serial</arg>
+            <arg>-Werror</arg>
+          </compilerArgs>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/VespaHttpClientBuilder.java
new file mode 100644
index 00000000000..53bf7b866af
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/VespaHttpClientBuilder.java
@@ -0,0 +1,145 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4;
+
+import com.yahoo.security.tls.MixedMode;
+import com.yahoo.security.tls.TlsContext;
+import com.yahoo.security.tls.TransportSecurityUtils;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.UnsupportedSchemeException;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.routing.HttpRoutePlanner;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.impl.conn.DefaultSchemePortResolver;
+import org.apache.http.protocol.HttpContext;
+
+import javax.net.ssl.SSLParameters;
+import java.net.InetAddress;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Http client builder for internal Vespa communications over http/https.
+ *
+ * Notes:
+ *  - hostname verification is not enabled - CN/SAN verification is assumed to be handled by the underlying x509 trust manager.
+ *  - custom connection managers must be configured through {@link #createBuilder(ConnectionManagerFactory)}. Do not call {@link HttpClientBuilder#setConnectionManager(HttpClientConnectionManager)}.
+ *
+ * @author bjorncs
+ */
+public class VespaHttpClientBuilder {
+
+    private static final Logger log = Logger.getLogger(VespaHttpClientBuilder.class.getName());
+
+    public interface ConnectionManagerFactory {
+        HttpClientConnectionManager create(Registry<ConnectionSocketFactory> socketFactoryRegistry);
+    }
+
+    private VespaHttpClientBuilder() {}
+
+    /**
+     * Create a client builder with default connection manager.
+     */
+    public static HttpClientBuilder create() {
+        return createBuilder(null);
+    }
+
+    /**
+     * Create a client builder with a user specified connection manager.
+     */
+    public static HttpClientBuilder create(ConnectionManagerFactory connectionManagerFactory) {
+        return createBuilder(connectionManagerFactory);
+    }
+
+    /**
+     * Creates a client builder with a {@link BasicHttpClientConnectionManager} configured.
+     * This connection manager uses a single connection for all requests. See Javadoc for details.
+     */
+    public static HttpClientBuilder createWithBasicConnectionManager() {
+        return createBuilder(BasicHttpClientConnectionManager::new);
+    }
+
+    private static HttpClientBuilder createBuilder(ConnectionManagerFactory connectionManagerFactory) {
+        HttpClientBuilder builder = HttpClientBuilder.create();
+        addSslSocketFactory(builder, connectionManagerFactory);
+        addHttpsRewritingRoutePlanner(builder);
+        return builder;
+    }
+
+    private static void addSslSocketFactory(HttpClientBuilder builder, ConnectionManagerFactory connectionManagerFactory)  {
+        TransportSecurityUtils.getSystemTlsContext()
+                .ifPresent(tlsContext -> {
+                    log.log(Level.FINE, "Adding ssl socket factory to client");
+                    SSLConnectionSocketFactory socketFactory = createSslSocketFactory(tlsContext);
+                    if (connectionManagerFactory != null) {
+                        builder.setConnectionManager(connectionManagerFactory.create(createRegistry(socketFactory)));
+                    } else {
+                        builder.setSSLSocketFactory(socketFactory);
+                    }
+                    // Workaround that allows re-using https connections, see https://stackoverflow.com/a/42112034/1615280 for details.
+                    // Proper solution would be to add a request interceptor that adds a x500 principal as user token,
+                    // but certificate subject CN is not accessible through the TlsContext currently.
+                    builder.setUserTokenHandler(context -> null);
+                });
+    }
+
+    private static void addHttpsRewritingRoutePlanner(HttpClientBuilder builder) {
+        if (TransportSecurityUtils.isTransportSecurityEnabled()
+                && TransportSecurityUtils.getInsecureMixedMode() != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) {
+            builder.setRoutePlanner(new HttpToHttpsRoutePlanner());
+        }
+    }
+
+    private static SSLConnectionSocketFactory createSslSocketFactory(TlsContext tlsContext) {
+        SSLParameters parameters = tlsContext.parameters();
+        return new SSLConnectionSocketFactory(tlsContext.context(), parameters.getProtocols(), parameters.getCipherSuites(), new NoopHostnameVerifier());
+    }
+
+    private static Registry<ConnectionSocketFactory> createRegistry(SSLConnectionSocketFactory sslSocketFactory) {
+        return RegistryBuilder.<ConnectionSocketFactory>create()
+                .register("https", sslSocketFactory)
+                .register("http", PlainConnectionSocketFactory.getSocketFactory())
+                .build();
+    }
+
+
+    /**
+     * Reroutes requests using 'http' to 'https'.
+     * Implementation inspired by {@link org.apache.http.impl.conn.DefaultRoutePlanner}, but without proxy support.
+     */
+    static class HttpToHttpsRoutePlanner implements HttpRoutePlanner {
+
+        @Override
+        public HttpRoute determineRoute(HttpHost host, HttpRequest request, HttpContext context) throws HttpException {
+            HttpClientContext clientContext = HttpClientContext.adapt(context);
+            RequestConfig config = clientContext.getRequestConfig();
+            InetAddress local = config.getLocalAddress();
+
+            HttpHost target = resolveTarget(host);
+            boolean secure = target.getSchemeName().equalsIgnoreCase("https");
+            return new HttpRoute(target, local, secure);
+        }
+
+        private HttpHost resolveTarget(HttpHost host) throws HttpException {
+            try {
+                String originalScheme = host.getSchemeName();
+                String scheme = originalScheme.equalsIgnoreCase("http") ? "https" : originalScheme;
+                int port = DefaultSchemePortResolver.INSTANCE.resolve(host);
+                return new HttpHost(host.getHostName(), port, scheme);
+            } catch (UnsupportedSchemeException e) {
+                throw new HttpException(e.getMessage(), e);
+            }
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelaySupplier.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelaySupplier.java
new file mode 100644
index 00000000000..b202966c412
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelaySupplier.java
@@ -0,0 +1,44 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import java.time.Duration;
+
+/**
+ * An abstraction that calculates the next delay based on the current retry count.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+interface DelaySupplier {
+    Duration getDelay(int executionCount);
+
+    class Fixed implements DelaySupplier {
+        private final Duration delay;
+
+        Fixed(Duration delay) {
+            this.delay = delay;
+        }
+
+        @Override
+        public Duration getDelay(int executionCount) { return delay; }
+    }
+
+    class Exponential implements DelaySupplier {
+        private final Duration startDelay;
+        private final Duration maxDelay;
+
+        Exponential(Duration startDelay, Duration maxDelay) {
+            this.startDelay = startDelay;
+            this.maxDelay = maxDelay;
+        }
+
+        @Override
+        public Duration getDelay(int executionCount) {
+            Duration nextDelay = startDelay;
+            for (int i = 1; i < executionCount; ++i) {
+                nextDelay = nextDelay.multipliedBy(2);
+            }
+            return maxDelay.compareTo(nextDelay) > 0 ? nextDelay : maxDelay;
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandler.java
new file mode 100644
index 00000000000..3ba92c08e30
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandler.java
@@ -0,0 +1,126 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.annotation.Contract;
+import org.apache.http.annotation.ThreadingBehavior;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.protocol.HttpContext;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
+/**
+ * A {@link HttpRequestRetryHandler} that supports delayed retries.
+ *
+ * @author bjorncs
+ */
+@Contract(threading = ThreadingBehavior.IMMUTABLE)
+public class DelayedConnectionLevelRetryHandler implements HttpRequestRetryHandler {
+
+    private static final Logger log = Logger.getLogger(HttpRequestRetryHandler.class.getName());
+
+    private final DelaySupplier delaySupplier;
+    private final int maxRetries;
+    private final RetryPredicate<IOException> predicate;
+    private final RetryConsumer<IOException> retryConsumer;
+    private final RetryFailedConsumer<IOException> retryFailedConsumer;
+    private final Sleeper sleeper;
+
+    private DelayedConnectionLevelRetryHandler(
+            DelaySupplier delaySupplier,
+            int maxRetries,
+            RetryPredicate<IOException> predicate,
+            RetryConsumer<IOException> retryConsumer,
+            RetryFailedConsumer<IOException> retryFailedConsumer,
+            Sleeper sleeper) {
+        this.delaySupplier = delaySupplier;
+        this.maxRetries = maxRetries;
+        this.predicate = predicate;
+        this.retryConsumer = retryConsumer;
+        this.retryFailedConsumer = retryFailedConsumer;
+        this.sleeper = sleeper;
+    }
+
+    @Override
+    public boolean retryRequest(IOException exception, int executionCount, HttpContext ctx) {
+        log.fine(() -> String.format("retryRequest(exception='%s', executionCount='%d', ctx='%s'",
+                                     exception.getClass().getName(), executionCount, ctx));
+        HttpClientContext clientCtx = HttpClientContext.adapt(ctx);
+        if (!predicate.test(exception, clientCtx)) {
+            log.fine(() -> String.format("Not retrying for '%s'", ctx));
+            return false;
+        }
+        if (executionCount > maxRetries) {
+            log.fine(() -> String.format("Max retries exceeded for '%s'", ctx));
+            retryFailedConsumer.onRetryFailed(exception, executionCount, clientCtx);
+            return false;
+        }
+        Duration delay = delaySupplier.getDelay(executionCount);
+        log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx));
+        retryConsumer.onRetry(exception, delay, executionCount, clientCtx);
+        sleeper.sleep(delay);
+        return true;
+    }
+
+    public static class Builder {
+
+        private final DelaySupplier delaySupplier;
+        private final int maxRetries;
+        private RetryPredicate<IOException> predicate = (ioException, ctx) -> true;
+        private RetryConsumer<IOException> retryConsumer = (exception, delay, count, ctx) -> {};
+        private RetryFailedConsumer<IOException> retryFailedConsumer = (exception, count, ctx) -> {};
+        private Sleeper sleeper = new Sleeper.Default();
+
+        private Builder(DelaySupplier delaySupplier, int maxRetries) {
+            this.delaySupplier = delaySupplier;
+            this.maxRetries = maxRetries;
+        }
+
+        public static Builder withFixedDelay(Duration delay, int maxRetries) {
+            return new Builder(new DelaySupplier.Fixed(delay), maxRetries);
+        }
+
+        public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) {
+            return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries);
+        }
+
+        public Builder retryForExceptions(List<Class<? extends IOException>> exceptionTypes) {
+            this.predicate = (ioException, ctx) -> exceptionTypes.stream().anyMatch(type -> type.isInstance(ioException));
+            return this;
+        }
+
+        public Builder retryForExceptions(Predicate<IOException> predicate) {
+            this.predicate = (ioException, ctx) -> predicate.test(ioException);
+            return this;
+        }
+
+        public Builder retryFor(RetryPredicate<IOException> predicate) {
+            this.predicate = predicate;
+            return this;
+        }
+
+        public Builder onRetry(RetryConsumer<IOException> consumer) {
+            this.retryConsumer = consumer;
+            return this;
+        }
+
+        public Builder onRetryFailed(RetryFailedConsumer<IOException> consumer) {
+            this.retryFailedConsumer = consumer;
+            return this;
+        }
+
+        // For unit testing
+        Builder withSleeper(Sleeper sleeper) {
+            this.sleeper = sleeper;
+            return this;
+        }
+
+        public DelayedConnectionLevelRetryHandler build() {
+            return new DelayedConnectionLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer, sleeper);
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandler.java
new file mode 100644
index 00000000000..d4ceb44a3ab
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandler.java
@@ -0,0 +1,125 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.annotation.Contract;
+import org.apache.http.annotation.ThreadingBehavior;
+import org.apache.http.client.ServiceUnavailableRetryStrategy;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.protocol.HttpContext;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
+/**
+ * A {@link ServiceUnavailableRetryStrategy} that supports delayed retries on any response types.
+ *
+ * @author bjorncs
+ */
+@Contract(threading = ThreadingBehavior.IMMUTABLE)
+public class DelayedResponseLevelRetryHandler implements ServiceUnavailableRetryStrategy {
+
+    private static final Logger log = Logger.getLogger(DelayedResponseLevelRetryHandler.class.getName());
+
+    private final DelaySupplier delaySupplier;
+    private final int maxRetries;
+    private final RetryPredicate<HttpResponse> predicate;
+    private final RetryConsumer<HttpResponse> retryConsumer;
+    private final RetryFailedConsumer<HttpResponse> retryFailedConsumer;
+    private final ThreadLocal<Long> retryInterval = ThreadLocal.withInitial(() -> 0L);
+
+    private DelayedResponseLevelRetryHandler(
+            DelaySupplier delaySupplier,
+            int maxRetries,
+            RetryPredicate<HttpResponse> predicate,
+            RetryConsumer<HttpResponse> retryConsumer,
+            RetryFailedConsumer<HttpResponse> retryFailedConsumer) {
+
+        this.delaySupplier = delaySupplier;
+        this.maxRetries = maxRetries;
+        this.predicate = predicate;
+        this.retryConsumer = retryConsumer;
+        this.retryFailedConsumer = retryFailedConsumer;
+    }
+
+    @Override
+    public boolean retryRequest(HttpResponse response, int executionCount, HttpContext ctx) {
+        log.fine(() -> String.format("retryRequest(responseCode='%s', executionCount='%d', ctx='%s'",
+                                     response.getStatusLine().getStatusCode(), executionCount, ctx));
+        HttpClientContext clientCtx = HttpClientContext.adapt(ctx);
+        if (!predicate.test(response, clientCtx)) {
+            log.fine(() -> String.format("Not retrying for '%s'", ctx));
+            return false;
+        }
+        if (executionCount > maxRetries) {
+            log.fine(() -> String.format("Max retries exceeded for '%s'", ctx));
+            retryFailedConsumer.onRetryFailed(response, executionCount, clientCtx);
+            return false;
+        }
+        Duration delay = delaySupplier.getDelay(executionCount);
+        log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx));
+        retryInterval.set(delay.toMillis());
+        retryConsumer.onRetry(response, delay, executionCount, clientCtx);
+        return true;
+    }
+
+    @Override
+    public long getRetryInterval() {
+        // Calls to getRetryInterval are always guarded by a call to retryRequest (using the same thread).
+        // A thread local allows this retry handler to be thread safe and support dynamic retry intervals
+        return retryInterval.get();
+    }
+
+    public static class Builder {
+
+        private final DelaySupplier delaySupplier;
+        private final int maxRetries;
+        private RetryPredicate<HttpResponse> predicate = (response, ctx) -> true;
+        private RetryConsumer<HttpResponse> retryConsumer = (response, delay, count, ctx) -> {};
+        private RetryFailedConsumer<HttpResponse> retryFailedConsumer = (response, count, ctx) -> {};
+
+        private Builder(DelaySupplier delaySupplier, int maxRetries) {
+            this.delaySupplier = delaySupplier;
+            this.maxRetries = maxRetries;
+        }
+
+        public static Builder withFixedDelay(Duration delay, int maxRetries) {
+            return new Builder(new DelaySupplier.Fixed(delay), maxRetries);
+        }
+
+        public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) {
+            return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries);
+        }
+
+        public Builder retryForStatusCodes(List<Integer> statusCodes) {
+            this.predicate = (response, ctx) -> statusCodes.contains(response.getStatusLine().getStatusCode());
+            return this;
+        }
+
+        public Builder retryForResponses(Predicate<HttpResponse> predicate) {
+            this.predicate = (response, ctx) -> predicate.test(response);
+            return this;
+        }
+
+        public Builder retryFor(RetryPredicate<HttpResponse> predicate) {
+            this.predicate = predicate;
+            return this;
+        }
+
+        public Builder onRetry(RetryConsumer<HttpResponse> consumer) {
+            this.retryConsumer = consumer;
+            return this;
+        }
+
+        public Builder onRetryFailed(RetryFailedConsumer<HttpResponse> consumer) {
+            this.retryFailedConsumer = consumer;
+            return this;
+        }
+
+        public DelayedResponseLevelRetryHandler build() {
+            return new DelayedResponseLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer);
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryConsumer.java
new file mode 100644
index 00000000000..c168f7d50c9
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryConsumer.java
@@ -0,0 +1,16 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+import java.time.Duration;
+
+/**
+ * Invoked before performing a delay and retry.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+public interface RetryConsumer<T> {
+    void onRetry(T data, Duration delay, int executionCount, HttpClientContext context);
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryFailedConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryFailedConsumer.java
new file mode 100644
index 00000000000..801c8a5af2f
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryFailedConsumer.java
@@ -0,0 +1,14 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+/**
+ * Invoked after the last retry has failed.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+public interface RetryFailedConsumer<T> {
+    void onRetryFailed(T response, int executionCount, HttpClientContext context);
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryPredicate.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryPredicate.java
new file mode 100644
index 00000000000..45c5ef0d623
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryPredicate.java
@@ -0,0 +1,13 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+import java.util.function.BiPredicate;
+
+/**
+ * A predicate that determines whether an operation should be retried.
+ *
+ * @author bjorncs
+ */
+public interface RetryPredicate<T> extends BiPredicate<T, HttpClientContext> {}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/Sleeper.java
new file mode 100644
index 00000000000..f593561888d
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/Sleeper.java
@@ -0,0 +1,26 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import java.time.Duration;
+
+/**
+ * An abstraction used for mocking {@link Thread#sleep(long)} in unit tests.
+ *
+ * @author bjorncs
+ */
+public interface Sleeper {
+    void sleep(Duration duration);
+
+    class Default implements Sleeper {
+        @Override
+        public void sleep(Duration duration) {
+            try {
+                Thread.sleep(duration.toMillis());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
+
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java
new file mode 100644
index 00000000000..92cc35fc354
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc5;
+
+import org.apache.hc.client5.http.HttpRoute;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.client5.http.routing.HttpRoutePlanner;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.protocol.HttpContext;
+
+/**
+ * {@link HttpRoutePlanner} that changes assumes requests specify the HTTP scheme,
+ * and then changes this to HTTPS, keeping the other host parameters.
+ *
+ * @author jonmv
+ */
+class HttpToHttpsRoutePlanner implements HttpRoutePlanner {
+
+    @Override
+    public HttpRoute determineRoute(HttpHost target, HttpContext context) throws HttpException {
+        if ( ! target.getSchemeName().equals("http") && ! target.getSchemeName().equals("https"))
+            throw new IllegalArgumentException("Scheme must be 'http' or 'https' when using HttpToHttpsRoutePlanner, was '" + target.getSchemeName() + "'");
+
+        if (target.getPort() == -1)
+            throw new IllegalArgumentException("Port must be set when using HttpToHttpsRoutePlanner");
+
+        if (HttpClientContext.adapt(context).getRequestConfig().getProxy() != null)
+            throw new IllegalArgumentException("Proxies are not supported with HttpToHttpsRoutePlanner");
+
+        return new HttpRoute(new HttpHost("https", target.getAddress(), target.getHostName(), target.getPort()));
+    }
+
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaAsyncHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaAsyncHttpClientBuilder.java
new file mode 100644
index 00000000000..50af29f92aa
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaAsyncHttpClientBuilder.java
@@ -0,0 +1,71 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc5;
+
+import com.yahoo.security.tls.MixedMode;
+import com.yahoo.security.tls.TlsContext;
+import com.yahoo.security.tls.TransportSecurityUtils;
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.nio.AsyncClientConnectionManager;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLParameters;
+
+/**
+ * Async http client builder for internal Vespa communications over http/https.
+ * Configures Vespa mTLS and handles TLS mixed mode automatically.
+ * Client should only be used for requests to Vespa services.
+ *
+ * Caveats:
+ * - custom connection manager must be configured through {@link #create(AsyncConnectionManagerFactory)}.
+ *
+ * @author bjorncs
+ */
+public class VespaAsyncHttpClientBuilder {
+
+    public interface AsyncConnectionManagerFactory {
+        AsyncClientConnectionManager create(TlsStrategy tlsStrategy);
+    }
+
+    public static HttpAsyncClientBuilder create() {
+        return create(
+                tlsStrategy -> PoolingAsyncClientConnectionManagerBuilder.create()
+                        .setTlsStrategy(tlsStrategy)
+                        .build());
+    }
+
+    public static HttpAsyncClientBuilder create(AsyncConnectionManagerFactory factory) {
+        return create(factory, new NoopHostnameVerifier());
+    }
+
+    public static HttpAsyncClientBuilder create(AsyncConnectionManagerFactory factory, HostnameVerifier hostnameVerifier) {
+        HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create();
+        TlsContext vespaTlsContext = TransportSecurityUtils.getSystemTlsContext().orElse(null);
+        TlsStrategy tlsStrategy;
+        if (vespaTlsContext != null) {
+            SSLParameters vespaTlsParameters = vespaTlsContext.parameters();
+            tlsStrategy = ClientTlsStrategyBuilder.create()
+                    .setHostnameVerifier(hostnameVerifier)
+                    .setSslContext(vespaTlsContext.context())
+                    .setTlsVersions(vespaTlsParameters.getProtocols())
+                    .setCiphers(vespaTlsParameters.getCipherSuites())
+                    .build();
+            if (TransportSecurityUtils.getInsecureMixedMode() != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) {
+                clientBuilder.setRoutePlanner(new HttpToHttpsRoutePlanner());
+            }
+        } else {
+            tlsStrategy = ClientTlsStrategyBuilder.create().build();
+        }
+        clientBuilder.disableConnectionState(); // Share connections between subsequent requests
+        clientBuilder.disableCookieManagement();
+        clientBuilder.disableAuthCaching();
+        clientBuilder.disableRedirectHandling();
+        clientBuilder.setConnectionManager(factory.create(tlsStrategy));
+        clientBuilder.setConnectionManagerShared(false);
+        return clientBuilder;
+    }
+
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java
new file mode 100644
index 00000000000..e01d278ff38
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java
@@ -0,0 +1,93 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc5;
+
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.io.HttpClientConnectionManager;
+import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
+import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
+import org.apache.hc.core5.http.config.Registry;
+import org.apache.hc.core5.http.config.RegistryBuilder;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLParameters;
+
+import static com.yahoo.security.tls.MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER;
+import static com.yahoo.security.tls.TransportSecurityUtils.getInsecureMixedMode;
+import static com.yahoo.security.tls.TransportSecurityUtils.getSystemTlsContext;
+import static com.yahoo.security.tls.TransportSecurityUtils.isTransportSecurityEnabled;
+
+/**
+ * Sync HTTP client builder <em>for internal Vespa communications over http/https.</em>
+ *
+ * Configures Vespa mTLS and handles TLS mixed mode automatically.
+ * Custom connection managers must be configured through {@link #create(HttpClientConnectionManagerFactory)}.
+ *
+ * @author jonmv
+ */
+public class VespaHttpClientBuilder {
+
+    public interface HttpClientConnectionManagerFactory {
+        HttpClientConnectionManager create(Registry<ConnectionSocketFactory> socketFactories);
+    }
+
+    public static HttpClientBuilder create() {
+        return create(PoolingHttpClientConnectionManager::new);
+    }
+
+    public static HttpClientBuilder create(HttpClientConnectionManagerFactory connectionManagerFactory) {
+        return create(connectionManagerFactory, new NoopHostnameVerifier());
+    }
+
+    public static HttpClientBuilder create(HttpClientConnectionManagerFactory connectionManagerFactory,
+                                           HostnameVerifier hostnameVerifier) {
+        return create(connectionManagerFactory, hostnameVerifier, true);
+    }
+
+    public static HttpClientBuilder create(HttpClientConnectionManagerFactory connectionManagerFactory,
+                                           HostnameVerifier hostnameVerifier,
+                                           boolean rewriteHttpToHttps) {
+        HttpClientBuilder builder = HttpClientBuilder.create();
+        addSslSocketFactory(builder, connectionManagerFactory, hostnameVerifier);
+        if (rewriteHttpToHttps)
+            addHttpsRewritingRoutePlanner(builder);
+
+        builder.disableConnectionState(); // Share connections between subsequent requests.
+        builder.disableCookieManagement();
+        builder.disableAuthCaching();
+        builder.disableRedirectHandling();
+
+        return builder;
+    }
+
+    private static void addSslSocketFactory(HttpClientBuilder builder, HttpClientConnectionManagerFactory connectionManagerFactory,
+                                            HostnameVerifier hostnameVerifier) {
+        getSystemTlsContext().ifPresent(tlsContext -> {
+            SSLParameters parameters = tlsContext.parameters();
+            SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(tlsContext.context(),
+                                                                                      parameters.getProtocols(),
+                                                                                      parameters.getCipherSuites(),
+                                                                                      hostnameVerifier);
+            builder.setConnectionManager(connectionManagerFactory.create(createRegistry(socketFactory)));
+            // Workaround that allows re-using https connections, see https://stackoverflow.com/a/42112034/1615280 for details.
+            // Proper solution would be to add a request interceptor that adds a x500 principal as user token,
+            // but certificate subject CN is not accessible through the TlsContext currently.
+            builder.setUserTokenHandler((route, context) -> null);
+        });
+    }
+
+    private static Registry<ConnectionSocketFactory> createRegistry(SSLConnectionSocketFactory sslSocketFactory) {
+        return RegistryBuilder.<ConnectionSocketFactory>create()
+                .register("https", sslSocketFactory)
+                .register("http", PlainConnectionSocketFactory.getSocketFactory())
+                .build();
+    }
+
+    private static void addHttpsRewritingRoutePlanner(HttpClientBuilder builder) {
+        if (isTransportSecurityEnabled() && getInsecureMixedMode() != PLAINTEXT_CLIENT_MIXED_SERVER)
+            builder.setRoutePlanner(new HttpToHttpsRoutePlanner());
+    }
+
+}
diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc4/VespaHttpClientBuilderTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/VespaHttpClientBuilderTest.java
new file mode 100644
index 00000000000..58aa70b69b1
--- /dev/null
+++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/VespaHttpClientBuilderTest.java
@@ -0,0 +1,42 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.routing.HttpRoutePlanner;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author bjorncs
+ */
+public class VespaHttpClientBuilderTest {
+
+    @Test
+    public void route_planner_modifies_scheme_of_requests() throws HttpException {
+        verifyProcessedUriMatchesExpectedOutput("http://dummyhostname:8080", "https://dummyhostname:8080");
+    }
+
+    @Test
+    public void route_planer_handles_implicit_http_port() throws HttpException {
+        verifyProcessedUriMatchesExpectedOutput("http://dummyhostname", "https://dummyhostname:80");
+    }
+
+    @Test
+    public void route_planer_handles_https_port() throws HttpException {
+        verifyProcessedUriMatchesExpectedOutput("http://dummyhostname:443", "https://dummyhostname:443");
+    }
+
+    private static void verifyProcessedUriMatchesExpectedOutput(String inputHostString, String expectedHostString) throws HttpException {
+        HttpRoutePlanner routePlanner = new VespaHttpClientBuilder.HttpToHttpsRoutePlanner();
+        HttpRoute newRoute = routePlanner.determineRoute(HttpHost.create(inputHostString), mock(HttpRequest.class), new HttpClientContext());
+        HttpHost target = newRoute.getTargetHost();
+        assertEquals(expectedHostString, target.toURI());
+    }
+
+}
\ No newline at end of file
diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java
new file mode 100644
index 00000000000..7330a91d75c
--- /dev/null
+++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java
@@ -0,0 +1,134 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.junit.Test;
+
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.time.Duration;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author bjorncs
+ */
+public class DelayedConnectionLevelRetryHandlerTest {
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void retry_consumers_are_invoked() {
+        RetryConsumer<IOException> retryConsumer = (RetryConsumer<IOException>) mock(RetryConsumer.class);
+        RetryFailedConsumer<IOException> retryFailedConsumer = (RetryFailedConsumer<IOException>) mock(RetryFailedConsumer.class);
+
+        Duration delay = Duration.ofSeconds(10);
+        int maxRetries = 5;
+
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .withSleeper(mock(Sleeper.class))
+                .onRetry(retryConsumer)
+                .onRetryFailed(retryFailedConsumer)
+                .build();
+
+        IOException exception = new IOException();
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(exception, i, ctx);
+        }
+
+        verify(retryFailedConsumer).onRetryFailed(exception, lastExecutionCount, ctx);
+        for (int i = 1; i < lastExecutionCount; i++) {
+            verify(retryConsumer).onRetry(exception, delay, i, ctx);
+        }
+    }
+
+    @Test
+    public void retry_with_fixed_delay_sleeps_for_expected_duration() {
+        Sleeper sleeper = mock(Sleeper.class);
+
+        Duration delay = Duration.ofSeconds(2);
+        int maxRetries = 2;
+
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .withSleeper(sleeper)
+                .build();
+
+        IOException exception = new IOException();
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(exception, i, ctx);
+        }
+
+        verify(sleeper, times(2)).sleep(delay);
+    }
+
+    @Test
+    public void retry_with_fixed_backoff_sleeps_for_expected_durations() {
+        Sleeper sleeper = mock(Sleeper.class);
+
+        Duration startDelay = Duration.ofMillis(500);
+        Duration maxDelay = Duration.ofSeconds(5);
+        int maxRetries = 10;
+
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
+                .withExponentialBackoff(startDelay, maxDelay, maxRetries)
+                .withSleeper(sleeper)
+                .build();
+
+        IOException exception = new IOException();
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(exception, i, ctx);
+        }
+
+        verify(sleeper).sleep(startDelay);
+        verify(sleeper).sleep(Duration.ofSeconds(1));
+        verify(sleeper).sleep(Duration.ofSeconds(2));
+        verify(sleeper).sleep(Duration.ofSeconds(4));
+        verify(sleeper, times(6)).sleep(Duration.ofSeconds(5));
+    }
+
+    @Test
+    public void retries_for_listed_exceptions_until_max_retries_exceeded() {
+        int maxRetries = 2;
+
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), maxRetries)
+                .retryForExceptions(Arrays.asList(SSLException.class, ConnectException.class))
+                .withSleeper(mock(Sleeper.class))
+                .build();
+
+        SSLException sslException = new SSLException("ssl error");
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i < lastExecutionCount; i++) {
+            assertTrue(handler.retryRequest(sslException, i, ctx));
+        }
+        assertFalse(handler.retryRequest(sslException, lastExecutionCount, ctx));
+    }
+
+    @Test
+    public void does_not_retry_for_non_listed_exception() {
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), 2)
+                .retryForExceptions(Arrays.asList(SSLException.class, ConnectException.class))
+                .withSleeper(mock(Sleeper.class))
+                .build();
+
+        IOException ioException = new IOException();
+        HttpClientContext ctx = new HttpClientContext();
+        assertFalse(handler.retryRequest(ioException, 1, ctx));
+    }
+
+}
\ No newline at end of file
diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java
new file mode 100644
index 00000000000..514eae56fe8
--- /dev/null
+++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java
@@ -0,0 +1,131 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc4.retry;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.message.BasicStatusLine;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author bjorncs
+ */
+public class DelayedResponseLevelRetryHandlerTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void retry_consumers_are_invoked() {
+        RetryConsumer<HttpResponse> retryConsumer = mock(RetryConsumer.class);
+        RetryFailedConsumer<HttpResponse> retryFailedConsumer = mock(RetryFailedConsumer.class);
+
+        Duration delay = Duration.ofSeconds(10);
+        int maxRetries = 5;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .onRetry(retryConsumer)
+                .onRetryFailed(retryFailedConsumer)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+        }
+
+        verify(retryFailedConsumer).onRetryFailed(response, lastExecutionCount, ctx);
+        for (int i = 1; i < lastExecutionCount; i++) {
+            verify(retryConsumer).onRetry(response, delay, i, ctx);
+        }
+    }
+
+    @Test
+    public void retry_with_fixed_delay_sleeps_for_expected_duration() {
+        Duration delay = Duration.ofSeconds(2);
+        int maxRetries = 2;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+            assertEquals(delay.toMillis(), handler.getRetryInterval());
+        }
+    }
+
+    @Test
+    public void retry_with_fixed_backoff_sleeps_for_expected_durations() {
+        Duration startDelay = Duration.ofMillis(500);
+        Duration maxDelay = Duration.ofSeconds(5);
+        int maxRetries = 10;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withExponentialBackoff(startDelay, maxDelay, maxRetries)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        List<Duration> expectedIntervals =
+                Arrays.asList(
+                        startDelay, Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4),
+                        Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5),
+                        Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5));
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+            assertEquals(expectedIntervals.get(i-1).toMillis(), handler.getRetryInterval());
+        }
+    }
+
+    @Test
+    public void retries_for_listed_exceptions_until_max_retries_exceeded() {
+        int maxRetries = 2;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), maxRetries)
+                .retryForStatusCodes(Arrays.asList(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY))
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i < lastExecutionCount; i++) {
+            assertTrue(handler.retryRequest(response, i, ctx));
+        }
+        assertFalse(handler.retryRequest(response, lastExecutionCount, ctx));
+    }
+
+    @Test
+    public void does_not_retry_for_non_listed_exception() {
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), 2)
+                .retryForStatusCodes(Arrays.asList(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY))
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_OK);
+        HttpClientContext ctx = new HttpClientContext();
+        assertFalse(handler.retryRequest(response, 1, ctx));
+    }
+
+    private static HttpResponse createResponse(int statusCode) {
+        return new BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, statusCode, "reason phrase"));
+    }
+
+}
diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java
new file mode 100644
index 00000000000..58dc25fdf1a
--- /dev/null
+++ b/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java
@@ -0,0 +1,59 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.hc5;
+
+import org.apache.hc.client5.http.HttpRoute;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpHost;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class HttpToHttpsRoutePlannerTest {
+
+    final HttpToHttpsRoutePlanner planner = new HttpToHttpsRoutePlanner();
+
+    @Test
+    public void verifySchemeMustBeHttp() throws HttpException {
+        try {
+            planner.determineRoute(new HttpHost("https", "host", 1), new HttpClientContext());
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Scheme must be 'http' when using HttpToHttpsRoutePlanner", e.getMessage());
+        }
+    }
+
+    @Test
+    public void verifyPortMustBeSet() throws HttpException {
+        try {
+            planner.determineRoute(new HttpHost("http", "host", -1), new HttpClientContext());
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Port must be set when using HttpToHttpsRoutePlanner", e.getMessage());
+        }
+    }
+
+
+    @Test
+    public void verifyProxyIsDisallowed() throws HttpException {
+        HttpClientContext context = new HttpClientContext();
+        context.setRequestConfig(RequestConfig.custom().setProxy(new HttpHost("proxy")).build());
+        try {
+            planner.determineRoute(new HttpHost("http", "host", 1), context);
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Proxies are not supported with HttpToHttpsRoutePlanner", e.getMessage());
+        }
+    }
+
+    @Test
+    public void verifySchemeIsRewritten() throws HttpException {
+        assertEquals(new HttpRoute(new HttpHost("https", "host", 1)),
+                     planner.determineRoute(new HttpHost("http", "host", 1), new HttpClientContext()));
+    }
+
+}