Merge pull request #4600 from vespa-engine/freva/node-admin-http-requests-with-certificate

Freva/node admin http requests with certificate

7 files changed, 239 insertions, 69 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java
index 5d8fed67682..980a244484c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java
@@ -37,6 +37,8 @@ public class DockerAdminComponent implements AdminComponent {
     private final MetricReceiverWrapper metricReceiver;
     private final ClassLocking classLocking;
 
+    private ConfigServerHttpRequestExecutor requestExecutor;
+
     private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty();
 
     public DockerAdminComponent(NodeAdminConfig config,
@@ -55,10 +57,9 @@ public class DockerAdminComponent implements AdminComponent {
             return;
         }
 
-
         Environment environment = new Environment();
-        ConfigServerHttpRequestExecutor requestExecutor =
-                ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris());
+        requestExecutor = ConfigServerHttpRequestExecutor.create(
+                environment.getConfigServerUris(), environment.getKeyStoreOptions(), environment.getTrustStoreOptions());
         NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor);
         Orchestrator orchestrator = new OrchestratorImpl(requestExecutor);
 
@@ -123,6 +124,7 @@ public class DockerAdminComponent implements AdminComponent {
         }
 
         nodeAdminStateUpdater.get().stop();
+        requestExecutor.close();
         nodeAdminStateUpdater = Optional.empty();
         // TODO: Also stop docker
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
index 94ad94d9a65..7b7de0e7417 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
@@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.util;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.concurrent.ThreadFactoryFactory;
 import org.apache.http.HttpHeaders;
-import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
@@ -13,27 +13,27 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.ssl.SSLContextBuilder;
 
+import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
-import java.security.GeneralSecurityException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 
 /**
  * Retries request on config server a few times before giving up. Assumes that all requests should be sent with
@@ -41,62 +41,39 @@ import java.util.Optional;
  *
  * @author dybdahl
  */
-public class ConfigServerHttpRequestExecutor {
+public class ConfigServerHttpRequestExecutor implements AutoCloseable {
     private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerHttpRequestExecutor.class);
+    private static final Duration CLIENT_REFRESH_INTERVAL = Duration.ofHours(1);
 
     private final ObjectMapper mapper = new ObjectMapper();
-    private final CloseableHttpClient client;
+    private final ScheduledExecutorService clientRefresherScheduler =
+            Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("http-client-refresher"));
     private final List<URI> configServerHosts;
 
-    @Override
-    public void finalize() throws Throwable {
-        try {
-            client.close();
-        } catch (Exception e) {
-            NODE_ADMIN_LOGGER.warning("Ignoring exception thrown when closing client against " + configServerHosts, e);
-        }
-
-        super.finalize();
-    }
-
-    public static ConfigServerHttpRequestExecutor create(Collection<URI> configServerUris) {
-        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(getConnectionSocketFactoryRegistry());
-        cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough
-
-        // Have experienced hang in socket read, which may have been because of
-        // system defaults, therefore set explicit timeouts. Set arbitrarily to
-        // 15s > 10s used by Orchestrator lock timeout.
-        int timeoutMs = 15_000;
-        RequestConfig requestBuilder = RequestConfig.custom()
-                .setConnectTimeout(timeoutMs) // establishment of connection
-                .setConnectionRequestTimeout(timeoutMs) // connection from connection manager
-                .setSocketTimeout(timeoutMs) // waiting for data
-                .build();
-
-        return new ConfigServerHttpRequestExecutor(randomizeConfigServerUris(configServerUris),
-                                                   HttpClientBuilder.create()
-                                                           .setDefaultRequestConfig(requestBuilder)
-                                                           .disableAutomaticRetries()
-                                                           .setUserAgent("node-admin")
-                                                           .setConnectionManager(cm).build());
-    }
-
-    private static Registry<ConnectionSocketFactory> getConnectionSocketFactoryRegistry() {
-        try {
-            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
-                    new SSLContextBuilder().loadTrustMaterial(null, (arg0, arg1) -> true).build(),
-                    NoopHostnameVerifier.INSTANCE);
-
-            return RegistryBuilder.<ConnectionSocketFactory>create()
-                    .register("http", PlainConnectionSocketFactory.getSocketFactory())
-                    .register("https", sslSocketFactory)
-                    .build();
-        } catch (GeneralSecurityException e) {
-            throw new RuntimeException("Failed to create SSL context", e);
+    /**
+     * The 'client' will be periodically re-created by clientRefresherScheduler if we provide keyStoreOptions
+     * or trustStoreOptions. This is needed because the key/trust stores are updated outside of node-admin,
+     * but we want to use the most recent store.
+     *
+     * The 'client' reference must be volatile because it is set and read in different threads, and visibility
+     * of changes is only guaranteed for volatile variables.
+     */
+    private volatile SelfCloseableHttpClient client;
+
+    public static ConfigServerHttpRequestExecutor create(
+            Collection<URI> configServerUris, Optional<KeyStoreOptions> keyStoreOptions, Optional<KeyStoreOptions> trustStoreOptions) {
+        Supplier<SelfCloseableHttpClient> clientSupplier = () -> createHttpClient(keyStoreOptions, trustStoreOptions);
+        ConfigServerHttpRequestExecutor requestExecutor = new ConfigServerHttpRequestExecutor(
+                randomizeConfigServerUris(configServerUris), clientSupplier.get());
+
+        if (keyStoreOptions.isPresent() || trustStoreOptions.isPresent()) {
+            requestExecutor.clientRefresherScheduler.scheduleAtFixedRate(() -> requestExecutor.client = clientSupplier.get(),
+                    CLIENT_REFRESH_INTERVAL.toMillis(), CLIENT_REFRESH_INTERVAL.toMillis(), TimeUnit.MILLISECONDS);
         }
+        return requestExecutor;
     }
 
-    ConfigServerHttpRequestExecutor(List<URI> configServerHosts, CloseableHttpClient client) {
+    ConfigServerHttpRequestExecutor(List<URI> configServerHosts, SelfCloseableHttpClient client) {
         this.configServerHosts = configServerHosts;
         this.client = client;
     }
@@ -200,4 +177,51 @@ public class ConfigServerHttpRequestExecutor {
         return shuffledConfigServerHosts;
     }
 
+    private static SelfCloseableHttpClient createHttpClient(Optional<KeyStoreOptions> keyStoreOptions,
+                                                            Optional<KeyStoreOptions> trustStoreOptions) {
+        NODE_ADMIN_LOGGER.info("Creating new HTTP client");
+        try {
+            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
+                    makeSslContext(keyStoreOptions, trustStoreOptions), NoopHostnameVerifier.INSTANCE);
+            return new SelfCloseableHttpClient(sslSocketFactory);
+        } catch (Exception e) {
+            NODE_ADMIN_LOGGER.error("Failed to create HTTP client with custom SSL Context, proceeding with default");
+            return new SelfCloseableHttpClient();
+        }
+    }
+
+    private static SSLContext makeSslContext(Optional<KeyStoreOptions> keyStoreOptions, Optional<KeyStoreOptions> trustStoreOptions)
+            throws KeyManagementException, NoSuchAlgorithmException {
+        SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+        keyStoreOptions.ifPresent(options -> {
+            try {
+                sslContextBuilder.loadKeyMaterial(options.getKeyStore(), options.password);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        trustStoreOptions.ifPresent(options -> {
+            try {
+                sslContextBuilder.loadTrustMaterial(options.getKeyStore(), null);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        return sslContextBuilder.build();
+    }
+
+    @Override
+    public void close() {
+        do {
+            try {
+                clientRefresherScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+            } catch (InterruptedException e1) {
+                NODE_ADMIN_LOGGER.info("Interrupted while waiting for clientRefresherScheduler to shutdown");
+            }
+        } while (!clientRefresherScheduler.isTerminated());
+
+        client.close();
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
index 2b5e7990617..dcde950c4d3 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
@@ -17,6 +17,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Optional;
 import java.util.TimeZone;
 import java.util.stream.Collectors;
 
@@ -37,6 +38,8 @@ public class Environment {
     private static final String REGION = "REGION";
     private static final String LOGSTASH_NODES = "LOGSTASH_NODES";
     private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT";
+    private static final String KEY_STORE_PATH = "KEY_STORE_PATH";
+    private static final String TRUST_STORE_PATH = "TRUST_STORE_PATH";
 
     private final List<URI> configServerHosts;
     private final String environment;
@@ -46,6 +49,8 @@ public class Environment {
     private final PathResolver pathResolver;
     private final List<String> logstashNodes;
     private final String feedEndpoint;
+    private final Optional<KeyStoreOptions> keyStoreOptions;
+    private final Optional<KeyStoreOptions> trustStoreOptions;
 
     static {
         filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -59,7 +64,11 @@ public class Environment {
              new InetAddressResolver(),
              new PathResolver(),
              getLogstashNodesFromEnvironment(),
-             getEnvironmentVariable(COREDUMP_FEED_ENDPOINT));
+             getEnvironmentVariable(COREDUMP_FEED_ENDPOINT),
+
+            // TODO: Make key store password and type configurable
+             getKeyStoreOptionsFromEnvironment(KEY_STORE_PATH, new char[0], "PKCS12"),
+             getKeyStoreOptionsFromEnvironment(TRUST_STORE_PATH, "changeit".toCharArray(), "JKS"));
     }
 
     public Environment(List<URI> configServerHosts,
@@ -69,7 +78,9 @@ public class Environment {
                        InetAddressResolver inetAddressResolver,
                        PathResolver pathResolver,
                        List<String> logstashNodes,
-                       String feedEndpoint) {
+                       String feedEndpoint,
+                       Optional<KeyStoreOptions> keyStoreOptions,
+                       Optional<KeyStoreOptions> trustStoreOptions) {
         this.configServerHosts = configServerHosts;
         this.environment = environment;
         this.region = region;
@@ -78,6 +89,8 @@ public class Environment {
         this.pathResolver = pathResolver;
         this.logstashNodes = logstashNodes;
         this.feedEndpoint = feedEndpoint;
+        this.keyStoreOptions = keyStoreOptions;
+        this.trustStoreOptions = trustStoreOptions;
     }
 
     public List<URI> getConfigServerUris() { return configServerHosts; }
@@ -124,6 +137,12 @@ public class Environment {
         return Arrays.asList(logstashNodes.split("[,\\s]+"));
     }
 
+    private static Optional<KeyStoreOptions> getKeyStoreOptionsFromEnvironment(String pathToKeyStore, char[] password, String type) {
+        return Optional.ofNullable(System.getenv(pathToKeyStore))
+                .filter(path -> !Strings.isNullOrEmpty(path))
+                .map(path -> new KeyStoreOptions(Paths.get(path), password, type));
+    }
+
     public InetAddress getInetAddressForHost(String hostname) throws UnknownHostException {
         return inetAddressResolver.getInetAddressForHost(hostname);
     }
@@ -189,7 +208,16 @@ public class Environment {
     public List<String> getLogstashNodes() {
         return logstashNodes;
     }
-    
+
+    public Optional<KeyStoreOptions> getKeyStoreOptions() {
+        return keyStoreOptions;
+    }
+
+    public Optional<KeyStoreOptions> getTrustStoreOptions() {
+        return trustStoreOptions;
+    }
+
+
     public static class Builder {
         private List<URI> configServerHosts = Collections.emptyList();
         private String environment;
@@ -199,6 +227,8 @@ public class Environment {
         private PathResolver pathResolver;
         private List<String> logstashNodes = Collections.emptyList();
         private String feedEndpoint;
+        private KeyStoreOptions keyStoreOptions;
+        private KeyStoreOptions trustStoreOptions;
 
         public Builder configServerUris(String... hosts) {
             configServerHosts = Arrays.stream(hosts)
@@ -242,10 +272,21 @@ public class Environment {
             return this;
         }
 
+        public Builder keyStoreOptions(KeyStoreOptions keyStoreOptions) {
+            this.keyStoreOptions = keyStoreOptions;
+            return this;
+        }
+
+        public Builder trustStoreOptions(KeyStoreOptions trustStoreOptions) {
+            this.trustStoreOptions = trustStoreOptions;
+            return this;
+        }
+
 
         public Environment build() {
             return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver,
-                                   pathResolver, logstashNodes, feedEndpoint);
+                                   pathResolver, logstashNodes, feedEndpoint,
+                                   Optional.ofNullable(keyStoreOptions), Optional.ofNullable(trustStoreOptions));
         }
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java
new file mode 100644
index 00000000000..fbcaf701c6f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java
@@ -0,0 +1,30 @@
+package com.yahoo.vespa.hosted.node.admin.util;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+class KeyStoreOptions {
+    public final Path path;
+    public final char[] password;
+    public final String type;
+
+    public KeyStoreOptions(Path path, char[] password, String type) {
+        this.path = path;
+        this.password = password;
+        this.type = type;
+    }
+
+    public KeyStore getKeyStore() throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException {
+        try (FileInputStream fis = new FileInputStream(path.toFile())) {
+            KeyStore keyStore = KeyStore.getInstance(type);
+            keyStore.load(fis, password);
+
+            return keyStore;
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java
new file mode 100644
index 00000000000..ddb473d348c
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java
@@ -0,0 +1,76 @@
+package com.yahoo.vespa.hosted.node.admin.util;
+
+import com.yahoo.log.LogLevel;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+/**
+ * @author freva
+ */
+class SelfCloseableHttpClient implements AutoCloseable {
+
+    private static final Logger log = Logger.getLogger(SelfCloseableHttpClient.class.getName());
+
+    private final CloseableHttpClient httpClient;
+
+    SelfCloseableHttpClient() {
+        this(SSLConnectionSocketFactory.getSocketFactory());
+    }
+
+    SelfCloseableHttpClient(SSLConnectionSocketFactory sslConnectionSocketFactory) {
+        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
+                .register("http", PlainConnectionSocketFactory.getSocketFactory())
+                .register("https", sslConnectionSocketFactory)
+                .build();
+
+        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+        cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough
+
+        // Have experienced hang in socket read, which may have been because of
+        // system defaults, therefore set explicit timeouts. Set arbitrarily to
+        // 15s > 10s used by Orchestrator lock timeout.
+        int timeoutMs = 15_000;
+        RequestConfig requestBuilder = RequestConfig.custom()
+                .setConnectTimeout(timeoutMs) // establishment of connection
+                .setConnectionRequestTimeout(timeoutMs) // connection from connection manager
+                .setSocketTimeout(timeoutMs) // waiting for data
+                .build();
+
+        this.httpClient = HttpClientBuilder.create()
+                .setDefaultRequestConfig(requestBuilder)
+                .disableAutomaticRetries()
+                .setUserAgent("node-admin")
+                .setConnectionManager(cm).build();
+    }
+
+    public CloseableHttpResponse execute(HttpUriRequest request) throws IOException {
+        return httpClient.execute(request);
+    }
+
+    @Override
+    public void finalize() throws Throwable {
+        close();
+        super.finalize();
+    }
+
+    @Override
+    public void close() {
+        try {
+            httpClient.close();
+        } catch (Exception e) {
+            log.log(LogLevel.WARNING, "Ignoring exception thrown when closing http client", e);
+        }
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
index 45f5de3d1a5..c6e4447a606 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
@@ -57,7 +57,7 @@ public class NodeRepositoryImplTest {
     public void startContainer() throws Exception {
         final int port = findRandomOpenPort();
         requestExecutor = ConfigServerHttpRequestExecutor.create(
-                Collections.singleton(URI.create("http://127.0.0.1:" + port)));
+                Collections.singleton(URI.create("http://127.0.0.1:" + port)), Optional.empty(), Optional.empty());
         container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable);
     }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
index 799f8a72fd9..175d3a9a051 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
@@ -7,7 +7,6 @@ import org.apache.http.HttpVersion;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.entity.BasicHttpEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicStatusLine;
 import org.junit.Before;
 import org.junit.Test;
@@ -27,7 +26,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -56,7 +54,7 @@ public class ConfigServerHttpRequestExecutorTest {
 
     @Before
     public void initExecutor() throws IOException {
-        CloseableHttpClient httpMock = mock(CloseableHttpClient.class);
+        SelfCloseableHttpClient httpMock = mock(SelfCloseableHttpClient.class);
         when(httpMock.execute(any())).thenAnswer(invocationOnMock -> {
             HttpGet get = (HttpGet) invocationOnMock.getArguments()[0];
             mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append("  ");
@@ -74,7 +72,6 @@ public class ConfigServerHttpRequestExecutorTest {
 
             return response;
         });
-        doNothing().when(httpMock).close();
         executor = new ConfigServerHttpRequestExecutor(configServers, httpMock);
     }