Split out the general part of the configserver-client

9 files changed, 875 insertions, 0 deletions

diff --git a/http-client/OWNERS b/http-client/OWNERS
new file mode 100644
index 00000000000..d0a102ecbf4
--- /dev/null
+++ b/http-client/OWNERS
@@ -0,0 +1 @@
+jonmv
diff --git a/http-client/README.md b/http-client/README.md
new file mode 100644
index 00000000000..a5912ecf2ef
--- /dev/null
+++ b/http-client/README.md
@@ -0,0 +1,2 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# HTTP client wrapping on Apache http client 5
diff --git a/http-client/pom.xml b/http-client/pom.xml
new file mode 100644
index 00000000000..a452353cb8a
--- /dev/null
+++ b/http-client/pom.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>parent</artifactId>
+        <groupId>com.yahoo.vespa</groupId>
+        <version>7-SNAPSHOT</version>
+	<relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>http-client</artifactId>
+    <description>HTTP client wrapper around an Apache http client 5</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>http-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-apache-http-client-bundle</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespajlib</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>yolean</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-standalone</artifactId>
+            <version>2.27.2</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
+
diff --git a/http-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java b/http-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java
new file mode 100644
index 00000000000..ec952c5c1ad
--- /dev/null
+++ b/http-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java
@@ -0,0 +1,272 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import ai.vespa.http.HttpURL;
+import ai.vespa.http.HttpURL.Path;
+import ai.vespa.http.HttpURL.Query;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.Method;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.HttpEntities;
+import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.logging.Logger;
+
+import static java.util.Objects.requireNonNull;
+import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.WARNING;
+
+/**
+ * @author jonmv
+ */
+public abstract class AbstractConfigServerClient implements ConfigServerClient {
+
+    private static final Logger log = Logger.getLogger(AbstractConfigServerClient.class.getName());
+
+    public static ConfigServerClient wrapping(CloseableHttpClient client) {
+        return new AbstractConfigServerClient() {
+            @Override
+            protected ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException {
+                return client.execute(request, context);
+            }
+            @Override
+            public void close() throws IOException {
+                client.close();
+            }
+        };
+    }
+
+    /** Executes the request with the given context. The caller must close the response. */
+    protected abstract ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException;
+
+    /** Executes the given request with response/error handling and retries. */
+    private <T> T execute(RequestBuilder builder,
+                          BiFunction<ClassicHttpResponse, ClassicHttpRequest, T> handler,
+                          ExceptionHandler catcher) {
+        HttpClientContext context = HttpClientContext.create();
+        context.setRequestConfig(builder.config);
+
+        Throwable thrown = null;
+        for (URI host : builder.hosts) {
+            ClassicHttpRequest request = ClassicRequestBuilder.create(builder.method.name())
+                                                              .setUri(HttpURL.from(host)
+                                                                             .appendPath(builder.path)
+                                                                             .appendQuery(builder.query)
+                                                                             .asURI())
+                                                              .build();
+            request.setEntity(builder.entity);
+            try {
+                try {
+                    return handler.apply(execute(request, context), request);
+                }
+                catch (IOException e) {
+                    catcher.handle(e, request);
+                    throw RetryException.wrap(e, request);
+                }
+            }
+            catch (RetryException e) {
+                if (thrown == null)
+                    thrown = e.getCause();
+                else
+                    thrown.addSuppressed(e.getCause());
+
+                if (builder.entity != null && ! builder.entity.isRepeatable()) {
+                    log.log(WARNING, "Cannot retry " + request + " as entity is not repeatable");
+                    break;
+                }
+                log.log(FINE, e.getCause(), () -> request + " failed; will retry");
+            }
+        }
+        if (thrown != null) {
+            if (thrown instanceof IOException)
+                throw new UncheckedIOException((IOException) thrown);
+            else if (thrown instanceof RuntimeException)
+                throw (RuntimeException) thrown;
+            else
+                throw new IllegalStateException("Illegal retry cause: " + thrown.getClass(), thrown);
+        }
+
+        throw new IllegalStateException("No hosts to perform the request against");
+    }
+
+    @Override
+    public ConfigServerClient.RequestBuilder send(HostStrategy hosts, Method method) {
+        return new RequestBuilder(hosts, method);
+    }
+
+    /** Builder for a request against a given set of hosts. */
+    class RequestBuilder implements ConfigServerClient.RequestBuilder {
+
+        private final Method method;
+        private final HostStrategy hosts;
+        private HttpURL.Path path = Path.empty();
+        private HttpURL.Query query = Query.empty();
+        private HttpEntity entity;
+        private RequestConfig config = ConfigServerClient.defaultRequestConfig;
+        private ResponseVerifier verifier = ConfigServerClient.throwOnError;
+        private ExceptionHandler catcher = ConfigServerClient.retryAll;
+
+        private RequestBuilder(HostStrategy hosts, Method method) {
+            if ( ! hosts.iterator().hasNext())
+                throw new IllegalArgumentException("Host strategy cannot be empty");
+
+            this.hosts = hosts;
+            this.method = requireNonNull(method);
+        }
+
+        @Override
+        public RequestBuilder at(Path subPath) {
+            path = path.append(subPath);
+            return this;
+        }
+
+        @Override
+        public ConfigServerClient.RequestBuilder body(byte[] json) {
+            return body(HttpEntities.create(json, ContentType.APPLICATION_JSON));
+        }
+
+        @Override
+        public RequestBuilder body(HttpEntity entity) {
+            this.entity = requireNonNull(entity);
+            return this;
+        }
+
+        @Override
+        public ConfigServerClient.RequestBuilder emptyParameters(List<String> keys) {
+            for (String key : keys)
+                query = query.add(key);
+
+            return this;
+        }
+
+        @Override
+        public RequestBuilder parameters(List<String> pairs) {
+            if (pairs.size() % 2 != 0)
+                throw new IllegalArgumentException("Must supply parameter key/values in pairs");
+
+            for (int i = 0; i < pairs.size(); ) {
+                String key = pairs.get(i++), value = pairs.get(i++);
+                if (value != null)
+                    query = query.add(key, value);
+            }
+
+            return this;
+        }
+
+        @Override
+        public ConfigServerClient.RequestBuilder parameters(Query query) {
+            this.query = this.query.add(query.entries());
+            return this;
+        }
+
+        @Override
+        public RequestBuilder timeout(Duration timeout) {
+            return config(RequestConfig.copy(config)
+                                       .setResponseTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS)
+                                       .build());
+        }
+
+        @Override
+        public RequestBuilder config(RequestConfig config) {
+            this.config = requireNonNull(config);
+            return this;
+        }
+
+        @Override
+        public RequestBuilder catching(ExceptionHandler catcher) {
+            this.catcher = requireNonNull(catcher);
+            return this;
+        }
+
+        @Override
+        public RequestBuilder throwing(ResponseVerifier verifier) {
+            this.verifier = requireNonNull(verifier);
+            return this;
+        }
+
+        @Override
+        public String read() {
+            return handle((response, __) -> {
+                try (response) {
+                    return response.getEntity() == null ? "" : EntityUtils.toString(response.getEntity());
+                }
+                catch (ParseException e) {
+                    throw new IllegalStateException(e); // This isn't actually thrown by apache >_<
+                }
+            });
+        }
+
+        @Override
+        public <T> T read(Function<byte[], T> mapper) {
+            return handle((response, __) -> {
+                try (response) {
+                    return mapper.apply(response.getEntity() == null ? new byte[0] : EntityUtils.toByteArray(response.getEntity()));
+                }
+            });
+        }
+
+        @Override
+        public void discard() throws UncheckedIOException, ResponseException {
+            handle((response, __) -> {
+                try (response) {
+                    return null;
+                }
+            });
+        }
+
+        @Override
+        public HttpInputStream stream() throws UncheckedIOException, ResponseException {
+            return handle((response, __) -> new HttpInputStream(response));
+        }
+
+        @Override
+        public <T> T handle(ResponseHandler<T> handler) {
+            return execute(this,
+                           (response, request) -> {
+                               try {
+                                   verifier.verify(response, request); // This throws on unacceptable responses.
+                                   return handler.handle(response, request);
+                               }
+                               catch (IOException | RuntimeException | Error e) {
+                                   try {
+                                       response.close();
+                                   }
+                                   catch (IOException f) {
+                                       e.addSuppressed(f);
+                                   }
+                                   if (e instanceof IOException) {
+                                       catcher.handle((IOException) e, request);
+                                       throw RetryException.wrap((IOException) e, request);
+                                   }
+                                   else
+                                       sneakyThrow(e); // e is a runtime exception or an error, so this is fine.
+                                   throw new AssertionError("Should not happen");
+                               }
+                           },
+                           catcher);
+        }
+
+    }
+
+
+    @SuppressWarnings("unchecked")
+    private static <T extends Throwable> void sneakyThrow(Throwable t) throws T {
+        throw (T) t;
+    }
+
+}
\ No newline at end of file
diff --git a/http-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java b/http-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java
new file mode 100644
index 00000000000..47e062766c6
--- /dev/null
+++ b/http-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java
@@ -0,0 +1,278 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import ai.vespa.http.HttpURL;
+import ai.vespa.http.HttpURL.Path;
+import ai.vespa.http.HttpURL.Query;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.Method;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.util.Timeout;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toUnmodifiableList;
+
+/**
+ * @author jonmv
+ */
+public interface ConfigServerClient extends Closeable {
+
+    RequestConfig defaultRequestConfig = RequestConfig.custom()
+                                                      .setConnectionRequestTimeout(Timeout.ofSeconds(5))
+                                                      .setConnectTimeout(Timeout.ofSeconds(5))
+                                                      .setRedirectsEnabled(false)
+                                                      .build();
+
+    /** Does nothing, letting the client wrap with a {@link RetryException} and re-throw. */
+    ExceptionHandler retryAll = (exception, request) -> { };
+
+    /** Throws a a {@link RetryException} if {@code statusCode == 503}, or a {@link ResponseException} unless {@code 200 <= statusCode < 300}. */
+    ResponseVerifier throwOnError = new DefaultResponseVerifier() { };
+
+    /** Reads the response body, throwing an {@link UncheckedIOException} if this fails, or {@code null} if there is none. */
+    static byte[] getBytes(ClassicHttpResponse response) {
+        try {
+            return response.getEntity() == null ? null : EntityUtils.toByteArray(response.getEntity());
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /** Creates a builder for sending the given method, using the specified host strategy. */
+    RequestBuilder send(HostStrategy hosts, Method method);
+
+    /** Builder for a request against a given set of hosts, using this config server client. */
+    interface RequestBuilder {
+
+        /** Appends to the request path, with no trailing slash. */
+        default RequestBuilder at(String... pathSegments) { return at(List.of(pathSegments)); }
+
+        /** Appends to the request path, with no trailing slash. */
+        default RequestBuilder at(List<String> pathSegments) { return at(Path.empty().append(pathSegments).withoutTrailingSlash()); }
+
+        /** Appends to the request path. */
+        RequestBuilder at(HttpURL.Path path);
+
+        /** Sets the request body as UTF-8 application/json. */
+        RequestBuilder body(byte[] json);
+
+        /** Sets the request body. */
+        RequestBuilder body(HttpEntity entity);
+
+        /** Sets query parameters without a value, like {@code ?debug&recursive}. */
+        default RequestBuilder emptyParameters(String... keys) {
+            return emptyParameters(Arrays.asList(keys));
+        }
+
+        /** Sets query parameters without a value, like {@code ?debug&recursive}. */
+        RequestBuilder emptyParameters(List<String> keys);
+
+        /** Sets the parameter key/values for the request. Number of arguments must be even. Null values are omitted. */
+        default RequestBuilder parameters(String... pairs) {
+            return parameters(Arrays.asList(pairs));
+        }
+
+        /** Sets the parameter key/values for the request. Number of arguments must be even. Pairs with {@code null} values are omitted. */
+        RequestBuilder parameters(List<String> pairs);
+
+        /** Appends all parameters from the given query. */
+        RequestBuilder parameters(Query query);
+
+        /** Overrides the default socket read timeout of the request. {@code Duration.ZERO} gives infinite timeout. */
+        RequestBuilder timeout(Duration timeout);
+
+        /** Overrides the default request config of the request. */
+        RequestBuilder config(RequestConfig config);
+
+        /**
+         * Sets the catch clause for {@link IOException}s during execution of this.
+         * The default is to wrap the IOException in a {@link RetryException} and rethrow this;
+         * this makes the client retry the request, as long as there are remaining entries in the {@link HostStrategy}.
+         * If the catcher returns normally, the exception is wrapped and retried, as per the default.
+         */
+        RequestBuilder catching(ExceptionHandler catcher);
+
+        /**
+         * Sets the (error) response handler for this request. The default is {@link #throwOnError}.
+         * When the handler returns normally, the response is treated as a success, and passed on to a response mapper.
+         */
+        RequestBuilder throwing(ResponseVerifier handler);
+
+        /** Reads the response as a {@link String}, or throws if unsuccessful. */
+        String read();
+
+        /** Reads and maps the response, or throws if unsuccessful. */
+        <T> T read(Function<byte[], T> mapper);
+
+        /** Discards the response, but throws if unsuccessful. */
+        void discard();
+
+        /** Returns the raw response input stream, or throws if unsuccessful. The caller must close the returned stream. */
+        HttpInputStream stream();
+
+        /** Uses the response and request, if successful, to generate a mapped response. */
+        <T> T handle(ResponseHandler<T> handler);
+
+    }
+
+
+    class HttpInputStream extends ForwardingInputStream {
+
+        private final ClassicHttpResponse response;
+
+        protected HttpInputStream(ClassicHttpResponse response) throws IOException {
+            super(response.getEntity() != null ? response.getEntity().getContent()
+                                               : InputStream.nullInputStream());
+            this.response = response;
+        }
+
+        public int statusCode() { return response.getCode(); }
+
+        public String contentType() { return response.getEntity().getContentType(); }
+
+        @Override
+        public void close() throws IOException {
+            super.close();
+            response.close();
+        }
+
+    }
+
+
+    /** Reads a successful response and request to compute a result. */
+    @FunctionalInterface
+    interface ResponseHandler<T> {
+
+        /** Called with successful responses, as per {@link ResponseVerifier}. The caller must close the response. */
+        T handle(ClassicHttpResponse response, ClassicHttpRequest request) throws IOException;
+
+    }
+
+
+    /** Verifies a response, throwing on error responses, possibly indicating retries. */
+    @FunctionalInterface
+    interface ResponseVerifier {
+
+        /** Whether this status code means the response is an error response. */
+        default boolean isError(int statusCode) {
+            return statusCode < HttpStatus.SC_OK || HttpStatus.SC_REDIRECTION <= statusCode;
+        }
+
+        /** Whether this status code means we should retry. Has no effect if this is not also an error. */
+        default boolean shouldRetry(int statusCode) {
+            return statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE;
+        }
+
+        /** Verifies the given response, consuming it and throwing if it is an error; or leaving it otherwise. */
+        default void verify(ClassicHttpResponse response, ClassicHttpRequest request) throws IOException {
+            if (isError(response.getCode())) {
+                try (response) {
+                    byte[] body = response.getEntity() == null ? new byte[0] : EntityUtils.toByteArray(response.getEntity());
+                    RuntimeException exception = toException(response.getCode(), body, request);
+                    throw shouldRetry(response.getCode()) ? new RetryException(exception) : exception;
+                }
+            }
+        }
+
+        /** Throws the appropriate exception, for the given status code and body. */
+        RuntimeException toException(int statusCode, byte[] body, ClassicHttpRequest request);
+
+    }
+
+
+    interface DefaultResponseVerifier extends ResponseVerifier {
+
+        @Override
+        default RuntimeException toException(int statusCode, byte[] body, ClassicHttpRequest request) {
+            return new ResponseException(request + " failed with status " + statusCode + " and body '" + new String(body, UTF_8) + "'");
+        }
+
+    }
+
+
+    @FunctionalInterface
+    interface ExceptionHandler {
+
+        /**
+         * Called with any IO exception that might occur when attempting to send the request.
+         * To retry, wrap the exception with a {@link RetryException} and re-throw, or exit normally.
+         * Any other thrown exception will propagate out to the caller.
+         */
+        void handle(IOException exception, ClassicHttpRequest request);
+
+    }
+
+
+    /** What host(s) to try for a request, in what order. A host may be specified multiple times, for retries.  */
+    @FunctionalInterface
+    interface HostStrategy extends Iterable<URI> {
+
+        /** Attempts each request once against each listed host. */
+        static HostStrategy ordered(List<URI> hosts) {
+            return List.copyOf(hosts)::iterator;
+        }
+
+        /** Attempts each request once against each listed host, in random order. */
+        static HostStrategy shuffling(List<URI> hosts) {
+            return () -> {
+                List<URI> copy = new ArrayList<>(hosts);
+                Collections.shuffle(copy);
+                return copy.iterator();
+            };
+        }
+
+        /** Attempts each request against the host the specified number of times. */
+        static HostStrategy repeating(URI host, int count) {
+            return ordered(IntStream.range(0, count).mapToObj(__ -> host).collect(toUnmodifiableList()));
+        }
+
+    }
+
+
+    /** Exception wrapper that signals retries should be attempted. */
+    final class RetryException extends RuntimeException {
+
+        public RetryException(IOException cause) {
+            super(requireNonNull(cause));
+        }
+
+        public RetryException(RuntimeException cause) {
+            super(requireNonNull(cause));
+        }
+
+        static RetryException wrap(IOException exception, ClassicHttpRequest request) {
+            return new RetryException(new UncheckedIOException(request + " failed (" + exception.getClass().getSimpleName() + ")", exception));
+        }
+
+    }
+
+
+    /** An exception due to server error, a bad request, or similar, which resulted in a non-OK HTTP response. */
+    class ResponseException extends RuntimeException {
+
+        public ResponseException(String message) {
+            super(message);
+        }
+
+    }
+
+}
\ No newline at end of file
diff --git a/http-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java b/http-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java
new file mode 100644
index 00000000000..402c2689ca7
--- /dev/null
+++ b/http-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java
@@ -0,0 +1,55 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * @author jonmv
+ */
+public class ForwardingInputStream extends InputStream {
+
+    private final InputStream delegate;
+
+    public ForwardingInputStream(InputStream delegate) {
+        this.delegate = requireNonNull(delegate);
+    }
+
+    @Override
+    public int read() throws IOException {
+        return delegate.read();
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        return delegate.read(b, off, len);
+    }
+
+    @Override
+    public int available() throws IOException {
+        return delegate.available();
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+        delegate.mark(readlimit);
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+        delegate.reset();
+    }
+
+    @Override
+    public boolean markSupported() {
+        return delegate.markSupported();
+    }
+
+}
diff --git a/http-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java b/http-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java
new file mode 100644
index 00000000000..16aad80670b
--- /dev/null
+++ b/http-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java
@@ -0,0 +1,57 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.entity.HttpEntities;
+import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.function.Function;
+
+/**
+ * @author jonmv
+ */
+public class MockConfigServerClient extends AbstractConfigServerClient {
+
+    private final Deque<Expectation> expectations = new ArrayDeque<>();
+
+    @Override
+    protected ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException {
+        Expectation expectation = expectations.poll();
+        if (expectation == null)
+            throw new AssertionError("No further requests expected, but got " + request);
+
+        return expectation.handle(request);
+    }
+
+    @Override
+    public void close() {
+        if ( ! expectations.isEmpty())
+            throw new AssertionError(expectations.size() + " more requests were expected");
+    }
+
+    public void expect(Expectation expectation) {
+        expectations.add(expectation);
+    }
+
+    public void expect(int status, Function<ClassicHttpRequest, String> mapper) {
+        expect(request -> {
+            BasicClassicHttpResponse response = new BasicClassicHttpResponse(status);
+            response.setEntity(HttpEntities.create(mapper.apply(request), ContentType.APPLICATION_JSON));
+            return response;
+        });
+    }
+
+    @FunctionalInterface
+    public interface Expectation {
+
+        ClassicHttpResponse handle(ClassicHttpRequest request) throws IOException;
+
+    }
+
+}
diff --git a/http-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java b/http-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java
new file mode 100644
index 00000000000..1bb352568bc
--- /dev/null
+++ b/http-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java
@@ -0,0 +1,111 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import ai.vespa.hosted.client.ConfigServerClient.HostStrategy;
+import ai.vespa.hosted.client.ConfigServerClient.ResponseException;
+import com.github.tomakehurst.wiremock.http.Fault;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.Method;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author jonmv
+ */
+class HttpConfigServerClientTest {
+
+    @RegisterExtension
+    final WireMockExtension server = new WireMockExtension();
+
+    ConfigServerClient client;
+
+    @BeforeEach
+    void setup() {
+        client = AbstractConfigServerClient.wrapping(HttpClients.createMinimal());
+    }
+
+    @Test
+    void testRetries() {
+        // Two servers in list--two attempts on IOException.
+        server.stubFor(get("/root?query=foo"))
+              .setResponse(okJson("{}").withFault(Fault.RANDOM_DATA_THEN_CLOSE)
+                                       .build());
+        assertThrows(UncheckedIOException.class,
+                     () -> client.send(HostStrategy.ordered(List.of(URI.create("http://localhost:" + server.port()),
+                                                                    URI.create("http://localhost:" + server.port() + "/"))),
+                                       Method.GET)
+                                 .at("root")
+                                 .parameters("query", "foo")
+                                 .discard());
+        server.verify(2, getRequestedFor(urlEqualTo("/root?query=foo")));
+        server.verify(2, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // Two attempts on a different IOException.
+        server.stubFor(post("/prefix/root"))
+              .setResponse(okJson("{}").withFault(Fault.EMPTY_RESPONSE)
+                                       .build());
+        assertThrows(UncheckedIOException.class,
+                     () -> client.send(HostStrategy.shuffling(List.of(URI.create("http://localhost:" + server.port() + "/prefix"),
+                                                                      URI.create("http://localhost:" + server.port() + "/prefix/"))),
+                                       Method.POST)
+                                 .body("hello".getBytes(UTF_8))
+                                 .at("root")
+                                 .stream());
+        server.verify(2, postRequestedFor(urlEqualTo("/prefix/root")).withRequestBody(equalTo("hello")));
+        server.verify(2, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // Successful attempt returns.
+        server.stubFor(get("/root/boot/toot"))
+              .setResponse(okJson("{}").build());
+        assertEquals("{}",
+                     client.send(HostStrategy.repeating(URI.create("http://localhost:" + server.port()), 10),
+                                 Method.GET)
+                           .at("root", "boot")
+                           .at("toot")
+                           .read(String::new));
+        server.verify(1, getRequestedFor(urlEqualTo("/root/boot/toot")));
+        server.verify(1, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // ResponseException is not retried.
+        server.stubFor(get("/"))
+              .setResponse(aResponse().withStatus(409).withBody("hi").build());
+        ResponseException thrown = assertThrows(ResponseException.class,
+                                                () -> client.send(HostStrategy.repeating(URI.create("http://localhost:" + server.port()), 10),
+                                                                  Method.GET)
+                                                            .read(String::new));
+        assertEquals("GET http://localhost:" + server.port() + "/ failed with status 409 and body 'hi'", thrown.getMessage());
+        server.verify(1, getRequestedFor(urlEqualTo("/")));
+        server.verify(1, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+    }
+
+    @AfterEach
+    void teardown() throws IOException {
+        client.close();
+    }
+
+}
diff --git a/http-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java b/http-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java
new file mode 100644
index 00000000000..d95650727c0
--- /dev/null
+++ b/http-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java
@@ -0,0 +1,42 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.core.Options;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * Allows wiremock to be used as a JUnit 5 extension, like
+ * <pre>
+ *
+ *   &#64RegisterExtension
+ *   WireMockExtension mockServer1 = new WireMockExtension();
+ * </pre>
+ */
+public class WireMockExtension extends WireMockServer implements BeforeEachCallback, AfterEachCallback {
+
+    public WireMockExtension() {
+        this(WireMockConfiguration.options()
+                                  .dynamicPort()
+                                  .dynamicHttpsPort());
+    }
+
+    public WireMockExtension(Options options) {
+        super(options);
+    }
+
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+        start();
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+        stop();
+        resetAll();
+    }
+
+}