Add configserver-client module

10 files changed, 802 insertions, 0 deletions

diff --git a/configserver-client/OWNERS b/configserver-client/OWNERS
new file mode 100644
index 00000000000..d0a102ecbf4
--- /dev/null
+++ b/configserver-client/OWNERS
@@ -0,0 +1 @@
+jonmv
diff --git a/configserver-client/README.md b/configserver-client/README.md
new file mode 100644
index 00000000000..2b6d96ed79f
--- /dev/null
+++ b/configserver-client/README.md
@@ -0,0 +1,2 @@
+<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# HTTP client for configserver APIs, built on Apache http client 5
diff --git a/configserver-client/pom.xml b/configserver-client/pom.xml
new file mode 100644
index 00000000000..cee6f6c067f
--- /dev/null
+++ b/configserver-client/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2021 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>parent</artifactId>
+        <groupId>com.yahoo.vespa</groupId>
+        <version>7-SNAPSHOT</version>
+	<relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>configserver-client</artifactId>
+    <description>HTTP client for configserver APIs, built on Apache http client 5</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>http-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespa-athenz</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>security-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>config-provisioning</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespajlib</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>yolean</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>testutil</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-standalone</artifactId>
+            <version>2.27.2</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
+
diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java
new file mode 100644
index 00000000000..0ee9e320259
--- /dev/null
+++ b/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java
@@ -0,0 +1,263 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.SlimeUtils;
+import org.apache.hc.client5.http.classic.methods.ClassicHttpRequests;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.Method;
+import org.apache.hc.core5.http.io.entity.HttpEntities;
+import org.apache.hc.core5.net.URIBuilder;
+import org.apache.hc.core5.util.Timeout;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Stream;
+
+import static ai.vespa.hosted.client.ConfigServerClient.ConfigServerException.ErrorCode.INCOMPLETE_RESPONSE;
+import static java.util.Objects.requireNonNull;
+import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.WARNING;
+
+/**
+ * @author jonmv
+ */
+public abstract class AbstractConfigServerClient implements ConfigServerClient {
+
+    static final RequestConfig defaultRequestConfig = RequestConfig.custom()
+                                                                   .setConnectionRequestTimeout(Timeout.ofSeconds(5))
+                                                                   .setConnectTimeout(Timeout.ofSeconds(5))
+                                                                   .setRedirectsEnabled(false)
+                                                                   .build();
+
+    private static final Logger log = Logger.getLogger(AbstractConfigServerClient.class.getName());
+
+    /** Executes the request with the given context. The caller must close the response. */
+    protected abstract ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException;
+
+    /** Executes the given request with response/error handling and retries. */
+    private <T> T execute(RequestBuilder builder, BiFunction<ClassicHttpResponse, IOException, T> handler) {
+        HttpClientContext context = HttpClientContext.create();
+        context.setRequestConfig(builder.config);
+
+        Throwable thrown = null;
+        for (URI host : builder.hosts) {
+            ClassicHttpRequest request = ClassicHttpRequests.create(builder.method, concat(host, builder.uriBuilder));
+            request.setEntity(builder.entity);
+            try {
+                try {
+                    return handler.apply(execute(request, context), null);
+                }
+                catch (IOException e) {
+                    return handler.apply(null, e);
+                }
+            }
+            catch (RetryException e) {
+                if (thrown == null)
+                    thrown = e.getCause();
+                else
+                    thrown.addSuppressed(e.getCause());
+
+                if (builder.entity != null && ! builder.entity.isRepeatable()) {
+                    log.log(WARNING, "Cannot retry " + request + " as entity is not repeatable");
+                    break;
+                }
+                log.log(FINE, request + " failed; will retry", e.getCause());
+            }
+        }
+        if (thrown != null) {
+            if (thrown instanceof IOException)
+                throw new UncheckedIOException((IOException) thrown);
+            else if (thrown instanceof RuntimeException)
+                throw (RuntimeException) thrown;
+            else
+                throw new IllegalStateException("Illegal retry cause: " + thrown.getClass(), thrown);
+        }
+
+        throw new IllegalArgumentException("No hosts to perform the request against");
+    }
+
+    /** Append path to the given host, which may already contain a root path. */
+    static URI concat(URI host, URIBuilder pathAndQuery) {
+        URIBuilder builder = new URIBuilder(host);
+        List<String> pathSegments = new ArrayList<>(builder.getPathSegments());
+        if ( ! pathSegments.isEmpty() && pathSegments.get(pathSegments.size() - 1).isEmpty())
+            pathSegments.remove(pathSegments.size() - 1);
+        pathSegments.addAll(pathAndQuery.getPathSegments());
+        try {
+            return builder.setPathSegments(pathSegments)
+                    .setParameters(pathAndQuery.getQueryParams())
+                    .build();
+        }
+        catch (URISyntaxException e) {
+            throw new IllegalArgumentException("URISyntaxException should not be possible here", e);
+        }
+    }
+
+    @Override
+    public RequestBuilder send(HostStrategy hosts, Method method) {
+        return new RequestBuilder(hosts, method);
+    }
+
+    /** Builder for a request against a given set of hosts. */
+    class RequestBuilder implements ConfigServerClient.RequestBuilder {
+
+        private final Method method;
+        private final HostStrategy hosts;
+        private final URIBuilder uriBuilder = new URIBuilder();
+        private HttpEntity entity;
+        private RequestConfig config = defaultRequestConfig;
+
+        private RequestBuilder(HostStrategy hosts, Method method) {
+            if ( ! hosts.iterator().hasNext())
+                throw new IllegalArgumentException("Host strategy cannot be empty");
+
+            this.hosts = hosts;
+            this.method = requireNonNull(method);
+        }
+
+        @Override
+        public RequestBuilder at(String... pathSegments) {
+            uriBuilder.setPathSegments(requireNonNull(pathSegments));
+            return this;
+        }
+
+        @Override
+        public ConfigServerClient.RequestBuilder body(byte[] json) {
+            return body(HttpEntities.create(json, ContentType.APPLICATION_JSON));
+        }
+
+        @Override
+        public RequestBuilder body(HttpEntity entity) {
+            this.entity = requireNonNull(entity);
+            return this;
+        }
+
+        @Override
+        public RequestBuilder parameters(String... pairs) {
+            if (pairs.length % 2 != 0)
+                throw new IllegalArgumentException("Must supply parameter key/values in pairs");
+
+            for (int i = 0; i < pairs.length; )
+                uriBuilder.setParameter(pairs[i++], pairs[i++]);
+
+            return this;
+        }
+
+        @Override
+        public RequestBuilder timeout(Duration timeout) {
+            return config(RequestConfig.copy(defaultRequestConfig)
+                                       .setResponseTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS)
+                                       .build());
+        }
+
+        @Override
+        public RequestBuilder config(RequestConfig config) {
+            this.config = requireNonNull(config);
+
+            return this;
+        }
+
+        @Override
+        public <T> T handle(BiFunction<ClassicHttpResponse, IOException, T> handler) throws UncheckedIOException {
+            return execute(this, requireNonNull(handler));
+        }
+
+        @Override
+        public <T> T read(Function<byte[], T> mapper) throws UncheckedIOException, ConfigServerException {
+            return mapIfSuccess(input -> {
+                try (input) {
+                    return mapper.apply(input.readAllBytes());
+                }
+                catch (IOException e) {
+                    throw new RetryException(e);
+                }
+            });
+        }
+
+        @Override
+        public void discard() throws UncheckedIOException, ConfigServerException {
+            mapIfSuccess(input -> {
+                try (input) {
+                    return null;
+                }
+                catch (IOException e) {
+                    throw new RetryException(e);
+                }
+            });
+        }
+
+        @Override
+        public InputStream stream() throws UncheckedIOException, ConfigServerException {
+            return mapIfSuccess(input -> input);
+        }
+
+        /** Returns the mapped body, if successful, retrying any IOException. The caller must close the body stream. */
+        private <T> T mapIfSuccess(Function<InputStream, T> mapper) {
+            return handle((response, ioException) -> {
+                if (response != null) {
+                    try {
+                        InputStream body = response.getEntity() != null ? response.getEntity().getContent()
+                                                                        : InputStream.nullInputStream();
+                        if (response.getCode() >= HttpStatus.SC_REDIRECTION)
+                            throw readException(body.readAllBytes());
+
+                        return mapper.apply(new ForwardingInputStream(body) {
+                            @Override
+                            public void close() throws IOException {
+                                super.close();
+                                response.close();
+                            }
+                        });
+                    }
+                    catch (IOException | RuntimeException | Error e) {
+                        try {
+                            response.close();
+                        }
+                        catch (IOException f) {
+                            e.addSuppressed(f);
+                        }
+                        if (e instanceof IOException)
+                            ioException = (IOException) e;
+                        else
+                            sneakyThrow(e);
+                    }
+                }
+                throw new RetryException(ioException);
+            });
+        }
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T extends Throwable> void sneakyThrow(Throwable t) throws T {
+        throw (T) t;
+    }
+
+    private static ConfigServerException readException(byte[] serialised) {
+        Inspector root = SlimeUtils.jsonToSlime(serialised).get();
+        String codeName = root.field("error-code").asString();
+        ConfigServerException.ErrorCode code = Stream.of(ConfigServerException.ErrorCode.values())
+                                                     .filter(value -> value.name().equals(codeName))
+                                                     .findAny().orElse(INCOMPLETE_RESPONSE);
+        String message = root.field("message").valid() ? root.field("message").asString() : "(no message)";
+        return new ConfigServerException(code, message, "");
+    }
+
+}
\ No newline at end of file
diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java
new file mode 100644
index 00000000000..234dbe9ee06
--- /dev/null
+++ b/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java
@@ -0,0 +1,148 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.Method;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toUnmodifiableList;
+
+/**
+ * @author jonmv
+ */
+public interface ConfigServerClient extends AutoCloseable {
+
+    /** Creates a builder for sending the given method, using the specified host strategy. */
+    RequestBuilder send(HostStrategy hosts, Method method);
+
+    /** Builder for a request against a given set of hosts, using this config server client. */
+    interface RequestBuilder {
+
+        /** Sets the request path. */
+        RequestBuilder at(String... pathSegments);
+
+        /** Sets the request body as UTF-8 application/json. */
+        RequestBuilder body(byte[] json);
+
+        /** Sets the request body. */
+        RequestBuilder body(HttpEntity entity);
+
+        /** Sets the parameter key/values for the request. Number of arguments must be even. */
+        RequestBuilder parameters(String... pairs);
+
+        /** Overrides the default socket read timeout of the request. {@code Duration.ZERO} gives infinite timeout. */
+        RequestBuilder timeout(Duration timeout);
+
+        /** Overrides the default socket read timeout of the request. {@code null} allows infinite timeout. */
+        RequestBuilder config(RequestConfig config);
+
+        /**
+         * Sets custom retry/failure logic for this.
+         * <p>
+         * Exactly one of the arguments (response, exception) are non-null.
+         * Return a value to have that returned to the caller;
+         * throw a {@link RetryException} to have the request retried; or
+         * throw any other unchecked exception to have this propagate out to the caller.
+         * The caller must close the provided response, if any.
+         */
+        <T> T handle(BiFunction<ClassicHttpResponse, IOException, T> handler) throws UncheckedIOException;
+
+        /** Sets the response body mapper for this, for successful requests. */
+        <T> T read(Function<byte[], T> mapper) throws UncheckedIOException, ConfigServerException;
+
+        /** Discards the response, but throws if the response is unsuccessful. */
+        void discard() throws UncheckedIOException, ConfigServerException;
+
+        /** Returns the raw input stream of the response, if successful. The caller must close the returned stream. */
+        InputStream stream() throws UncheckedIOException, ConfigServerException;
+
+    }
+
+    /** Exception wrapper that signals retries should be attempted. */
+    final class RetryException extends RuntimeException {
+
+        public RetryException(IOException cause) {
+            super(requireNonNull(cause));
+        }
+
+        public RetryException(RuntimeException cause) {
+            super(requireNonNull(cause));
+        }
+
+    }
+
+    /** What host(s) to try for a request, in what order. A host may be specified multiple times, for retries.  */
+    @FunctionalInterface
+    interface HostStrategy extends Iterable<URI> {
+
+        /** Attempts each request once against each listed host. */
+        static HostStrategy ordered(List<URI> hosts) {
+            return List.copyOf(hosts)::iterator;
+        }
+
+        /** Attempts each request once against each listed host, in random order. */
+        static HostStrategy shuffling(List<URI> hosts) {
+            return () -> {
+                List<URI> copy = new ArrayList<>(hosts);
+                Collections.shuffle(copy);
+                return copy.iterator();
+            };
+        }
+
+        /** Attempts each request against the host the specified number of times. */
+        static HostStrategy repeating(URI host, int count) {
+            return ordered(IntStream.range(0, count).mapToObj(__ -> host).collect(toUnmodifiableList()));
+        }
+
+    }
+
+    /** An exception due to server error, a bad request, or similar. */
+    class ConfigServerException extends RuntimeException {
+
+        private final ErrorCode errorId;
+        private final String message;
+
+        public ConfigServerException(ErrorCode errorId, String message, String context) {
+            super(context + ": " + message);
+            this.errorId = errorId;
+            this.message = message;
+        }
+
+        public ErrorCode errorId() { return errorId; }
+
+        public String message() { return message; }
+
+        public enum ErrorCode {
+            APPLICATION_LOCK_FAILURE,
+            BAD_REQUEST,
+            ACTIVATION_CONFLICT,
+            INTERNAL_SERVER_ERROR,
+            INVALID_APPLICATION_PACKAGE,
+            METHOD_NOT_ALLOWED,
+            NOT_FOUND,
+            OUT_OF_CAPACITY,
+            REQUEST_TIMEOUT,
+            UNKNOWN_VESPA_VERSION,
+            PARENT_HOST_NOT_READY,
+            CERTIFICATE_NOT_READY,
+            LOAD_BALANCER_NOT_READY,
+            INCOMPLETE_RESPONSE
+        }
+
+    }
+
+}
\ No newline at end of file
diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java b/configserver-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java
new file mode 100644
index 00000000000..473dce8ea33
--- /dev/null
+++ b/configserver-client/src/main/java/ai/vespa/hosted/client/ForwardingInputStream.java
@@ -0,0 +1,55 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * @author jonmv
+ */
+public class ForwardingInputStream extends InputStream {
+
+    private final InputStream delegate;
+
+    public ForwardingInputStream(InputStream delegate) {
+        this.delegate = requireNonNull(delegate);
+    }
+
+    @Override
+    public int read() throws IOException {
+        return delegate.read();
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        return delegate.read(b, off, len);
+    }
+
+    @Override
+    public int available() throws IOException {
+        return delegate.available();
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+        delegate.mark(readlimit);
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+        delegate.reset();
+    }
+
+    @Override
+    public boolean markSupported() {
+        return delegate.markSupported();
+    }
+
+}
diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/HttpConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/HttpConfigServerClient.java
new file mode 100644
index 00000000000..c5b07eceaf5
--- /dev/null
+++ b/configserver-client/src/main/java/ai/vespa/hosted/client/HttpConfigServerClient.java
@@ -0,0 +1,61 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import ai.vespa.util.http.hc5.VespaHttpClientBuilder;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.io.SocketConfig;
+import org.apache.hc.core5.util.TimeValue;
+import org.apache.hc.core5.util.Timeout;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * @author jonmv
+ */
+public class HttpConfigServerClient extends AbstractConfigServerClient {
+
+    private final CloseableHttpClient client;
+
+    public HttpConfigServerClient(Collection<AthenzIdentity> serverIdentities, String userAgent) {
+        if (serverIdentities.isEmpty())
+            throw new IllegalArgumentException("At least one trusted server identity must be provided");
+
+        this.client = createClient(serverIdentities, userAgent);
+    }
+
+    @Override
+    public void close() throws IOException {
+        client.close();
+    }
+
+    @Override
+    protected ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException {
+        return client.execute(request, context);
+    }
+
+    private static CloseableHttpClient createClient(Collection<AthenzIdentity> serverIdentities, String userAgent) {
+        return VespaHttpClientBuilder.create(socketFactories -> {
+                                                 var manager = new PoolingHttpClientConnectionManager(socketFactories);
+                                                 manager.setMaxTotal(256);
+                                                 manager.setDefaultMaxPerRoute(8);
+                                                 manager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(Timeout.ofSeconds(5)).build());
+                                                 manager.setValidateAfterInactivity(TimeValue.ofSeconds(10));
+                                                 return manager;
+                                             },
+                                             new AthenzIdentityVerifier(Set.copyOf(serverIdentities)),
+                                             false)
+                                     .disableAutomaticRetries()
+                                     .setUserAgent(userAgent)
+                                     .setDefaultRequestConfig(defaultRequestConfig)
+                                     .build();
+    }
+
+}
diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java
new file mode 100644
index 00000000000..3bd93f7fac7
--- /dev/null
+++ b/configserver-client/src/main/java/ai/vespa/hosted/client/MockConfigServerClient.java
@@ -0,0 +1,57 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.entity.HttpEntities;
+import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.function.Function;
+
+/**
+ * @author jonmv
+ */
+public class MockConfigServerClient extends AbstractConfigServerClient {
+
+    private final Deque<Expectation> expectations = new ArrayDeque<>();
+
+    @Override
+    protected ClassicHttpResponse execute(ClassicHttpRequest request, HttpClientContext context) throws IOException {
+        Expectation expectation = expectations.poll();
+        if (expectation == null)
+            throw new AssertionError("No further requests expected, but got " + request);
+
+        return expectation.handle(request);
+    }
+
+    @Override
+    public void close() {
+        if ( ! expectations.isEmpty())
+            throw new AssertionError(expectations.size() + " more requests were expected");
+    }
+
+    public void expect(Expectation expectation) {
+        expectations.add(expectation);
+    }
+
+    public void expect(int status, Function<ClassicHttpRequest, String> mapper) {
+        expect(request -> {
+            BasicClassicHttpResponse response = new BasicClassicHttpResponse(status);
+            response.setEntity(HttpEntities.create(mapper.apply(request), ContentType.APPLICATION_JSON));
+            return response;
+        });
+    }
+
+    @FunctionalInterface
+    public interface Expectation {
+
+        ClassicHttpResponse handle(ClassicHttpRequest request) throws IOException;
+
+    }
+
+}
diff --git a/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java b/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java
new file mode 100644
index 00000000000..cbf38f46f6f
--- /dev/null
+++ b/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java
@@ -0,0 +1,98 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import ai.vespa.hosted.client.ConfigServerClient.ConfigServerException;
+import ai.vespa.hosted.client.ConfigServerClient.HostStrategy;
+import com.github.tomakehurst.wiremock.http.Fault;
+import com.yahoo.vespa.athenz.api.AthenzService;
+import org.apache.hc.core5.http.Method;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * @author jonmv
+ */
+class HttpConfigServerClientTest {
+
+    @RegisterExtension
+    final WireMockExtension server = new WireMockExtension();
+
+    final HttpConfigServerClient client = new HttpConfigServerClient(List.of(new AthenzService("mydomain", "yourservice")), "user");
+
+    @Test
+    void testRetries() {
+        // Two servers in list--two attempts on IOException.
+        server.stubFor(get("/root?query=foo"))
+              .setResponse(okJson("{}").withFault(Fault.RANDOM_DATA_THEN_CLOSE)
+                                       .build());
+        assertThrows(UncheckedIOException.class,
+                     () -> client.send(HostStrategy.ordered(List.of(URI.create("http://localhost:" + server.port()),
+                                                                    URI.create("http://localhost:" + server.port() + "/"))),
+                                       Method.GET)
+                                 .at("root")
+                                 .parameters("query", "foo")
+                                 .discard());
+        server.verify(2, getRequestedFor(urlEqualTo("/root?query=foo")));
+        server.verify(2, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // Two attempts on a different IOException.
+        server.stubFor(post("/prefix/%2Froot"))
+              .setResponse(okJson("{}").withFault(Fault.EMPTY_RESPONSE)
+                                       .build());
+        assertThrows(UncheckedIOException.class,
+                     () -> client.send(HostStrategy.shuffling(List.of(URI.create("http://localhost:" + server.port() + "/prefix"),
+                                                                      URI.create("http://localhost:" + server.port() + "/prefix/"))),
+                                       Method.POST)
+                                 .body("hello".getBytes(UTF_8))
+                                 .at("/root")
+                                 .stream());
+        server.verify(2, postRequestedFor(urlEqualTo("/prefix/%2Froot")).withRequestBody(equalTo("hello")));
+        server.verify(2, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // Successful attempt returns.
+        server.stubFor(get("/root/boot"))
+              .setResponse(okJson("{}").build());
+        assertEquals("{}",
+                     client.send(HostStrategy.repeating(URI.create("http://localhost:" + server.port()), 10),
+                                 Method.GET)
+                           .at("root", "boot")
+                           .read(String::new));
+        server.verify(1, getRequestedFor(urlEqualTo("/root/boot")));
+        server.verify(1, anyRequestedFor(anyUrl()));
+        server.resetRequests();
+
+        // ConfigServerException is not retried.
+        server.stubFor(get("/"))
+              .setResponse(aResponse().withStatus(409).withBody("{\"error-code\":\"ACTIVATION_CONFLICT\",\"message\":\"hi\"}").build());
+        ConfigServerException thrown = assertThrows(ConfigServerException.class,
+                                                    () -> client.send(HostStrategy.repeating(URI.create("http://localhost:" + server.port()), 10),
+                                                                      Method.GET)
+                                                                .read(String::new));
+        assertEquals(ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, thrown.errorId());
+        assertEquals("hi", thrown.message());
+        server.verify(1, getRequestedFor(urlEqualTo("/")));
+        server.verify(1, anyRequestedFor(anyUrl()));
+
+    }
+
+}
diff --git a/configserver-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java b/configserver-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java
new file mode 100644
index 00000000000..6b8a36abb6e
--- /dev/null
+++ b/configserver-client/src/test/java/ai/vespa/hosted/client/WireMockExtension.java
@@ -0,0 +1,42 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.hosted.client;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.core.Options;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * Allows wiremock to be used as a JUnit 5 extension, like
+ * <pre>
+ *
+ *   &#64RegisterExtension
+ *   WireMockExtension mockServer1 = new WireMockExtension();
+ * </pre>
+ */
+public class WireMockExtension extends WireMockServer implements BeforeEachCallback, AfterEachCallback {
+
+    public WireMockExtension() {
+        this(WireMockConfiguration.options()
+                                  .dynamicPort()
+                                  .dynamicHttpsPort());
+    }
+
+    public WireMockExtension(Options options) {
+        super(options);
+    }
+
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+        start();
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+        stop();
+        resetAll();
+    }
+
+}