Node-Admin: Use HTTPS against config servers

10 files changed, 133 insertions, 138 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index d3af55cdff5..d2863468ee7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -18,11 +18,13 @@ import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
 import java.io.IOException;
 import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
@@ -99,7 +101,9 @@ public class DockerOperationsImpl implements DockerOperations {
             InetAddress nodeInetAddress = environment.getInetAddressForHost(nodeSpec.hostname);
             final boolean isIPv6 = nodeInetAddress instanceof Inet6Address;
 
-            String configServers = String.join(",", environment.getConfigServerHosts());
+            String configServers = environment.getConfigServerUris().stream()
+                    .map(URI::getHost)
+                    .collect(Collectors.joining(","));
             Docker.CreateContainerCommand command = docker.createContainerCommand(
                     nodeSpec.wantedDockerImage.get(),
                     ContainerResources.from(nodeSpec.minCpuCores, nodeSpec.minMainMemoryAvailableGb),
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 763291bf86b..f41d7deb04b 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker;
 
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -310,8 +311,8 @@ public class StorageMaintainer {
      * @throws RuntimeException if exit code != 0
      */
     public String getHardwareDivergence() {
-        String configServers = environment.getConfigServerHosts().stream()
-                .map(configServer -> "http://" +  configServer + ":" + 4080)
+        String configServers = environment.getConfigServerUris().stream()
+                .map(URI::getHost)
                 .collect(Collectors.joining(","));
         return executeMaintainer("com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier", configServers);
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
index 8283c90e43d..b5c439e209a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
@@ -31,11 +31,9 @@ public class NodeRepositoryImpl implements NodeRepository {
     private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(NodeRepositoryImpl.class);
 
     private final ConfigServerHttpRequestExecutor requestExecutor;
-    private final int port;
 
-    public NodeRepositoryImpl(ConfigServerHttpRequestExecutor requestExecutor, int port) {
+    public NodeRepositoryImpl(ConfigServerHttpRequestExecutor requestExecutor) {
         this.requestExecutor = requestExecutor;
-        this.port = port;
     }
 
     @Override
@@ -43,7 +41,6 @@ public class NodeRepositoryImpl implements NodeRepository {
         try {
             final GetNodesResponse nodesForHost = requestExecutor.get(
                     "/nodes/v2/node/?parentHost=" + baseHostName + "&recursive=true",
-                    port,
                     GetNodesResponse.class);
 
             if (nodesForHost.nodes == null) {
@@ -71,7 +68,6 @@ public class NodeRepositoryImpl implements NodeRepository {
     public Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName) {
         try {
             GetNodesResponse.Node nodeResponse = requestExecutor.get("/nodes/v2/node/" + hostName,
-                                                                     port,
                                                                      GetNodesResponse.Node.class);
             if (nodeResponse == null) {
                 return Optional.empty();
@@ -86,7 +82,7 @@ public class NodeRepositoryImpl implements NodeRepository {
     public List<ContainerAclSpec> getContainerAclSpecs(String hostName) {
         try {
             final String path = String.format("/nodes/v2/acl/%s?children=true", hostName);
-            final GetAclResponse response = requestExecutor.get(path, port, GetAclResponse.class);
+            final GetAclResponse response = requestExecutor.get(path, GetAclResponse.class);
             return response.trustedNodes.stream()
                     .map(node -> new ContainerAclSpec(
                             node.hostname, node.ipAddress, ContainerName.fromHostname(node.trustedBy)))
@@ -96,7 +92,7 @@ public class NodeRepositoryImpl implements NodeRepository {
         }
     }
 
-    public static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node)
+    private static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node)
             throws IllegalArgumentException, NullPointerException {
         Objects.requireNonNull(node.nodeState, "Unknown node state");
         Node.State nodeState = Node.State.valueOf(node.nodeState);
@@ -145,7 +141,6 @@ public class NodeRepositoryImpl implements NodeRepository {
     public void updateNodeAttributes(final String hostName, final NodeAttributes nodeAttributes) {
         UpdateNodeAttributesResponse response = requestExecutor.patch(
                 "/nodes/v2/node/" + hostName,
-                port,
                 new UpdateNodeAttributesRequestBody(nodeAttributes),
                 UpdateNodeAttributesResponse.class);
 
@@ -170,7 +165,6 @@ public class NodeRepositoryImpl implements NodeRepository {
     private void markNodeToState(String hostName, String state) {
         NodeMessageResponse response = requestExecutor.put(
                 "/nodes/v2/state/" + state + "/" + hostName,
-                port,
                 Optional.empty(), /* body */
                 NodeMessageResponse.class);
         NODE_ADMIN_LOGGER.info(response.message);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
index d1e996f8e93..7093f3f12e7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
@@ -26,11 +26,9 @@ public class OrchestratorImpl implements Orchestrator {
             = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX;
 
     private final ConfigServerHttpRequestExecutor requestExecutor;
-    private final int port;
 
-    public OrchestratorImpl(ConfigServerHttpRequestExecutor requestExecutor, int port) {
+    public OrchestratorImpl(ConfigServerHttpRequestExecutor requestExecutor) {
         this.requestExecutor = requestExecutor;
-        this.port = port;
     }
 
     @Override
@@ -38,7 +36,6 @@ public class OrchestratorImpl implements Orchestrator {
         UpdateHostResponse response;
         try {
             response = requestExecutor.put(getSuspendPath(hostName),
-                    port,
                     Optional.empty(), /* body */
                     UpdateHostResponse.class);
         } catch (HttpException.NotFoundException n) {
@@ -61,7 +58,6 @@ public class OrchestratorImpl implements Orchestrator {
         try {
             batchOperationResult = requestExecutor.put(
                     ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API,
-                    port,
                     Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)),
                     BatchOperationResult.class);
         } catch (HttpException e) {
@@ -81,7 +77,7 @@ public class OrchestratorImpl implements Orchestrator {
         UpdateHostResponse response;
         try {
             String path = getSuspendPath(hostName);
-            response = requestExecutor.delete(path, port, UpdateHostResponse.class);
+            response = requestExecutor.delete(path, UpdateHostResponse.class);
         } catch (HttpException.NotFoundException n) {
             throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found");
         } catch (HttpException e) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java
index 109dbab924c..a343e431b5a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java
@@ -29,8 +29,6 @@ import java.time.Clock;
 import java.time.Duration;
 import java.util.function.Function;
 
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-
 /**
  * Set up node admin for production.
  *
@@ -38,7 +36,6 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults;
  */
 public class NodeAdminProvider implements Provider<NodeAdminStateUpdater> {
 
-    private static final int WEB_SERVICE_PORT = getDefaults().vespaWebServicePort();
     private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30);
     private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30);
 
@@ -51,9 +48,9 @@ public class NodeAdminProvider implements Provider<NodeAdminStateUpdater> {
         ProcessExecuter processExecuter = new ProcessExecuter();
         Environment environment = new Environment();
 
-        ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerHosts());
-        NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor, WEB_SERVICE_PORT);
-        Orchestrator orchestrator = new OrchestratorImpl(requestExecutor, WEB_SERVICE_PORT);
+        ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris());
+        NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor);
+        Orchestrator orchestrator = new OrchestratorImpl(requestExecutor);
         DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter);
 
         StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
index e825fa4ac8f..92bfb8b288c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java
@@ -12,18 +12,28 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
+
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.security.GeneralSecurityException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 
 /**
  * Retries request on config server a few times before giving up. Assumes that all requests should be sent with
@@ -33,11 +43,11 @@ import java.util.Set;
  */
 public class ConfigServerHttpRequestExecutor {
     private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerHttpRequestExecutor.class);
+    private static final int MAX_LOOPS = 2;
 
     private final ObjectMapper mapper = new ObjectMapper();
     private final CloseableHttpClient client;
-    private final List<String> configServerHosts;
-    private final static int MAX_LOOPS = 2;
+    private final List<URI> configServerHosts;
 
     @Override
     public void finalize() throws Throwable {
@@ -50,31 +60,45 @@ public class ConfigServerHttpRequestExecutor {
         super.finalize();
     }
 
-    public static ConfigServerHttpRequestExecutor create(Set<String> configServerHosts) {
-        if (configServerHosts.isEmpty()) {
-            throw new IllegalStateException("Environment setting for config servers missing or empty.");
-        }
+    public static ConfigServerHttpRequestExecutor create(Collection<URI> configServerUris) {
+        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(getConnectionSocketFactoryRegistry());
+        cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough
+
+        return new ConfigServerHttpRequestExecutor(randomizeConfigServerUris(configServerUris),
+                                                   HttpClientBuilder.create()
+                                                           .disableAutomaticRetries()
+                                                           .setUserAgent("node-admin")
+                                                           .setConnectionManager(cm).build());
+    }
 
-        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
-        // Increase max total connections to 200, which should be enough
-        cm.setMaxTotal(200);
-        return new ConfigServerHttpRequestExecutor(configServerHosts,
-                                                   HttpClientBuilder.create().disableAutomaticRetries().setConnectionManager(cm).build());
+    private static Registry<ConnectionSocketFactory> getConnectionSocketFactoryRegistry() {
+        try {
+            SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
+                    new SSLContextBuilder().loadTrustMaterial(null, (arg0, arg1) -> true).build(),
+                    NoopHostnameVerifier.INSTANCE);
+
+            return RegistryBuilder.<ConnectionSocketFactory>create()
+                    .register("http", PlainConnectionSocketFactory.getSocketFactory())
+                    .register("https", sslSocketFactory)
+                    .build();
+        } catch (GeneralSecurityException e) {
+            throw new RuntimeException("Failed to create SSL context", e);
+        }
     }
 
-    ConfigServerHttpRequestExecutor(Set<String> configServerHosts, CloseableHttpClient client) {
-        this.configServerHosts = randomizeConfigServerHosts(configServerHosts);
+    ConfigServerHttpRequestExecutor(List<URI> configServerHosts, CloseableHttpClient client) {
+        this.configServerHosts = configServerHosts;
         this.client = client;
     }
 
     public interface CreateRequest {
-        HttpUriRequest createRequest(String configserver) throws JsonProcessingException, UnsupportedEncodingException;
+        HttpUriRequest createRequest(URI configServerUri) throws JsonProcessingException, UnsupportedEncodingException;
     }
 
     private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType) {
         Exception lastException = null;
         for (int loopRetry = 0; loopRetry < MAX_LOOPS; loopRetry++) {
-            for (String configServer : configServerHosts) {
+            for (URI configServer : configServerHosts) {
                 final CloseableHttpResponse response;
                 try {
                     response = client.execute(requestFactory.createRequest(configServer));
@@ -118,9 +142,9 @@ public class ConfigServerHttpRequestExecutor {
                 + configServerHosts + ") failed, last as follows:", lastException);
     }
 
-    public <T> T put(String path, int port, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) {
+    public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) {
         return tryAllConfigServers(configServer -> {
-            HttpPut put = new HttpPut("http://" + configServer + ":" + port + path);
+            HttpPut put = new HttpPut(configServer.resolve(path));
             setContentTypeToApplicationJson(put);
             if (bodyJsonPojo.isPresent()) {
                 put.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo.get())));
@@ -129,28 +153,28 @@ public class ConfigServerHttpRequestExecutor {
         }, wantedReturnType);
     }
 
-    public <T> T patch(String path, int port, Object bodyJsonPojo, Class<T> wantedReturnType) {
+    public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) {
         return tryAllConfigServers(configServer -> {
-            HttpPatch patch = new HttpPatch("http://" + configServer + ":" + port + path);
+            HttpPatch patch = new HttpPatch(configServer.resolve(path));
             setContentTypeToApplicationJson(patch);
             patch.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo)));
             return patch;
         }, wantedReturnType);
     }
 
-    public <T> T delete(String path, int port, Class<T> wantedReturnType) {
+    public <T> T delete(String path, Class<T> wantedReturnType) {
         return tryAllConfigServers(configServer ->
-                new HttpDelete("http://" + configServer + ":" + port + path), wantedReturnType);
+                new HttpDelete(configServer.resolve(path)), wantedReturnType);
     }
 
-    public <T> T get(String path, int port, Class<T> wantedReturnType) {
+    public <T> T get(String path, Class<T> wantedReturnType) {
         return tryAllConfigServers(configServer ->
-                new HttpGet("http://" + configServer + ":" + port + path), wantedReturnType);
+                new HttpGet(configServer.resolve(path)), wantedReturnType);
     }
 
-    public <T> T post(String path, int port, Object bodyJsonPojo, Class<T> wantedReturnType) {
+    public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) {
         return tryAllConfigServers(configServer -> {
-            HttpPost post = new HttpPost("http://" + configServer + ":" + port + path);
+            HttpPost post = new HttpPost(configServer.resolve(path));
             setContentTypeToApplicationJson(post);
             post.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo)));
             return post;
@@ -161,9 +185,9 @@ public class ConfigServerHttpRequestExecutor {
         request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
     }
 
-    // Shuffle config server hosts to balance load
-    private List<String> randomizeConfigServerHosts(Set<String> configServerHosts) {
-        List<String> shuffledConfigServerHosts = new ArrayList<>(configServerHosts);
+    // Shuffle config server URIs to balance load
+    private static List<URI> randomizeConfigServerUris(Collection<URI> configServerUris) {
+        List<URI> shuffledConfigServerHosts = new ArrayList<>(configServerUris);
         Collections.shuffle(shuffledConfigServerHosts);
         return shuffledConfigServerHosts;
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
index 65a4d84b4c3..2b5e7990617 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
@@ -6,6 +6,7 @@ import com.yahoo.net.HostName;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -15,9 +16,7 @@ import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.TimeZone;
 import java.util.stream.Collectors;
 
@@ -31,13 +30,15 @@ public class Environment {
     private static final DateFormat filenameFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
     public static final String APPLICATION_STORAGE_CLEANUP_PATH_PREFIX = "cleanup_";
 
-    private static final String ENV_CONFIGSERVERS = "services__addr_configserver";
+    private static final String ENV_CONFIGSERVER_SCHEME = "CONFIG_SERVER_SCHEME";
+    private static final String ENV_CONFIGSERVER_HOSTS = "CONFIG_SERVER_ADDRESS";
+    private static final String ENV_CONFIGSERVER_PORT = "CONFIG_SERVER_PORT";
     private static final String ENVIRONMENT = "ENVIRONMENT";
     private static final String REGION = "REGION";
     private static final String LOGSTASH_NODES = "LOGSTASH_NODES";
     private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT";
 
-    private final Set<String> configServerHosts;
+    private final List<URI> configServerHosts;
     private final String environment;
     private final String region;
     private final String parentHostHostname;
@@ -51,7 +52,7 @@ public class Environment {
     }
 
     public Environment() {
-        this(getConfigServerHostsFromEnvironment(),
+        this(getConfigServerUrlsFromEnvironment(),
              getEnvironmentVariable(ENVIRONMENT),
              getEnvironmentVariable(REGION),
              HostName.getLocalhost(),
@@ -61,7 +62,7 @@ public class Environment {
              getEnvironmentVariable(COREDUMP_FEED_ENDPOINT));
     }
 
-    public Environment(Set<String> configServerHosts,
+    public Environment(List<URI> configServerHosts,
                        String environment,
                        String region,
                        String parentHostHostname,
@@ -79,7 +80,7 @@ public class Environment {
         this.feedEndpoint = feedEndpoint;
     }
 
-    public Set<String> getConfigServerHosts() { return configServerHosts; }
+    public List<URI> getConfigServerUris() { return configServerHosts; }
 
     public String getEnvironment() {
         return environment;
@@ -95,7 +96,9 @@ public class Environment {
 
     private static String getEnvironmentVariable(String name) {
         final String value = System.getenv(name);
-        if (value == null) throw new IllegalStateException(String.format("Environment variable %s not set", name));
+        if (Strings.isNullOrEmpty(value)) {
+            throw new IllegalStateException(String.format("Environment variable %s not set", name));
+        }
         return value;
     }
 
@@ -103,14 +106,14 @@ public class Environment {
         return getEnvironment() + "." + getRegion();
     }
 
-    private static Set<String> getConfigServerHostsFromEnvironment() {
-        String configServerHosts = System.getenv(ENV_CONFIGSERVERS);
-        if (configServerHosts == null) {
-            return Collections.emptySet();
-        }
+    private static List<URI> getConfigServerUrlsFromEnvironment() {
+        String scheme = getEnvironmentVariable(ENV_CONFIGSERVER_SCHEME);
+        String configServerHosts = getEnvironmentVariable(ENV_CONFIGSERVER_HOSTS);
+        String port = getEnvironmentVariable(ENV_CONFIGSERVER_PORT);
 
-        final List<String> hostNameStrings = Arrays.asList(configServerHosts.split("[,\\s]+"));
-        return new HashSet<>(hostNameStrings);
+        return Arrays.stream(configServerHosts.split("[,\\s]+"))
+                .map(hostname -> URI.create(scheme + "://" + hostname + ":" + port))
+                .collect(Collectors.toList());
     }
 
     private static List<String> getLogstashNodesFromEnvironment() {
@@ -188,7 +191,7 @@ public class Environment {
     }
     
     public static class Builder {
-        private Set<String> configServerHosts = Collections.emptySet();
+        private List<URI> configServerHosts = Collections.emptyList();
         private String environment;
         private String region;
         private String parentHostHostname;
@@ -197,8 +200,10 @@ public class Environment {
         private List<String> logstashNodes = Collections.emptyList();
         private String feedEndpoint;
 
-        public Builder configServerHosts(String... hosts) {
-            configServerHosts = Arrays.stream(hosts).collect(Collectors.toSet());
+        public Builder configServerUris(String... hosts) {
+            configServerHosts = Arrays.stream(hosts)
+                    .map(URI::create)
+                    .collect(Collectors.toList());
             return this;
         }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
index 7456b07d2f4..45f5de3d1a5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
@@ -17,6 +17,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 import java.net.ServerSocket;
+import java.net.URI;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
@@ -35,9 +36,7 @@ import static org.junit.Assert.fail;
  */
 public class NodeRepositoryImplTest {
     private JDisc container;
-    private int port;
-    private final ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(
-            Collections.singleton("127.0.0.1"));
+    private ConfigServerHttpRequestExecutor requestExecutor;
 
 
     private int findRandomOpenPort() throws IOException {
@@ -56,14 +55,15 @@ public class NodeRepositoryImplTest {
      */
     @Before
     public void startContainer() throws Exception {
-        port = findRandomOpenPort();
-        System.err.println("PORT IS " + port);
+        final int port = findRandomOpenPort();
+        requestExecutor = ConfigServerHttpRequestExecutor.create(
+                Collections.singleton(URI.create("http://127.0.0.1:" + port)));
         container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable);
     }
 
     private void waitForJdiscContainerToServe() throws InterruptedException {
         Instant start = Instant.now();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         while (Instant.now().minusSeconds(120).isBefore(start)) {
             try {
                 nodeRepositoryApi.getContainersToRun("foobar");
@@ -85,7 +85,7 @@ public class NodeRepositoryImplTest {
     @Test
     public void testGetContainersToRunApi() throws IOException, InterruptedException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         String dockerHostHostname = "dockerhost1.yahoo.com";
 
         final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(dockerHostHostname);
@@ -104,7 +104,7 @@ public class NodeRepositoryImplTest {
     @Test
     public void testGetContainer() throws InterruptedException, IOException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         String hostname = "host4.yahoo.com";
         Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname);
         assertThat(nodeSpec.isPresent(), is(true));
@@ -114,7 +114,7 @@ public class NodeRepositoryImplTest {
     @Test
     public void testGetContainerForNonExistingNode() throws InterruptedException, IOException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         String hostname = "host-that-does-not-exist";
         Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname);
         assertFalse(nodeSpec.isPresent());
@@ -123,7 +123,7 @@ public class NodeRepositoryImplTest {
     @Test
     public void testUpdateNodeAttributes() throws InterruptedException, IOException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         String hostname = "host4.yahoo.com";
         nodeRepositoryApi.updateNodeAttributes(
                 hostname,
@@ -136,7 +136,7 @@ public class NodeRepositoryImplTest {
     @Test(expected = RuntimeException.class)
     public void testUpdateNodeAttributesWithBadValue() throws InterruptedException, IOException {
         waitForJdiscContainerToServe();
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         String hostname = "host4.yahoo.com";
         nodeRepositoryApi.updateNodeAttributes(
                 hostname,
@@ -148,7 +148,7 @@ public class NodeRepositoryImplTest {
 
     @Test
     public void testMarkAsReady() throws InterruptedException, IOException {
-        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port);
+        NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor);
         waitForJdiscContainerToServe();
 
         nodeRepositoryApi.markAsDirty("host5.yahoo.com");
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
index 0a11ddd1e62..779a0a6a376 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
@@ -24,15 +24,12 @@ public class OrchestratorImplTest {
     private static final String hostName = "host123.yahoo.com";
 
     private final ConfigServerHttpRequestExecutor requestExecutor = mock(ConfigServerHttpRequestExecutor.class);
-    private final int port = 1234;
-    private final OrchestratorImpl orchestrator = new OrchestratorImpl(requestExecutor, port);
-
+    private final OrchestratorImpl orchestrator = new OrchestratorImpl(requestExecutor);
 
     @Test
     public void testSuspendCall() {
         when(requestExecutor.put(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended",
-                port,
                 Optional.empty(),
                 UpdateHostResponse.class
         )).thenReturn(new UpdateHostResponse(hostName, null));
@@ -44,7 +41,6 @@ public class OrchestratorImplTest {
     public void testSuspendCallWithFailureReason() {
         when(requestExecutor.put(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended",
-                port,
                 Optional.empty(),
                 UpdateHostResponse.class
         )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail")));
@@ -56,7 +52,6 @@ public class OrchestratorImplTest {
     public void testSuspendCallWithNotFound() {
         when(requestExecutor.put(
                 any(String.class),
-                any(Integer.class),
                 any(),
                 any()
         )).thenThrow(new HttpException.NotFoundException("Not Found"));
@@ -68,7 +63,6 @@ public class OrchestratorImplTest {
     public void testSuspendCallWithSomeOtherException() {
         when(requestExecutor.put(
                 any(String.class),
-                any(Integer.class),
                 any(),
                 any()
         )).thenThrow(new RuntimeException("Some parameter was wrong"));
@@ -81,7 +75,6 @@ public class OrchestratorImplTest {
     public void testResumeCall() {
         when(requestExecutor.delete(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended",
-                port,
                 UpdateHostResponse.class
         )).thenReturn(new UpdateHostResponse(hostName, null));
 
@@ -92,7 +85,6 @@ public class OrchestratorImplTest {
     public void testResumeCallWithFailureReason() {
         when(requestExecutor.delete(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended",
-                port,
                 UpdateHostResponse.class
         )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail")));
 
@@ -103,7 +95,6 @@ public class OrchestratorImplTest {
     public void testResumeCallWithNotFound() {
         when(requestExecutor.delete(
                 any(String.class),
-                any(Integer.class),
                 any()
         )).thenThrow(new HttpException.NotFoundException("Not Found"));
 
@@ -114,7 +105,6 @@ public class OrchestratorImplTest {
     public void testResumeCallWithSomeOtherException() {
         when(requestExecutor.put(
                 any(String.class),
-                any(Integer.class),
                 any(),
                 any()
         )).thenThrow(new RuntimeException("Some parameter was wrong"));
@@ -130,7 +120,6 @@ public class OrchestratorImplTest {
 
         when(requestExecutor.put(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API,
-                port,
                 Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)),
                 BatchOperationResult.class
         )).thenReturn(BatchOperationResult.successResult());
@@ -146,7 +135,6 @@ public class OrchestratorImplTest {
 
         when(requestExecutor.put(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API,
-                port,
                 Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)),
                 BatchOperationResult.class
         )).thenReturn(new BatchOperationResult(failureReason));
@@ -162,7 +150,6 @@ public class OrchestratorImplTest {
 
         when(requestExecutor.put(
                 OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API,
-                port,
                 Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)),
                 BatchOperationResult.class
         )).thenThrow(new RuntimeException(exceptionMessage));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
index d431f175b1c..990d047b546 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java
@@ -3,19 +3,21 @@ package com.yahoo.vespa.hosted.node.admin.util;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.yahoo.collections.ArraySet;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
+import org.apache.http.HttpVersion;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.entity.BasicHttpEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicStatusLine;
 import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
@@ -38,11 +40,14 @@ public class ConfigServerHttpRequestExecutorTest {
     @JsonIgnoreProperties(ignoreUnknown = true)
     public static class TestPojo {
         @JsonProperty("foo")
-        public String foo;
+        String foo;
         @JsonProperty("error-code")
-        public Integer errorCode;
+        Integer errorCode;
     }
 
+    private final String uri1 = "http://host1:666";
+    private final String uri2 = "http://host2:666";
+    private final List<URI> configServers = Arrays.asList(URI.create(uri1), URI.create(uri2));
     private final StringBuilder mockLog = new StringBuilder();
     private int mockReturnCode = 200;
 
@@ -51,16 +56,18 @@ public class ConfigServerHttpRequestExecutorTest {
         when(httpMock.execute(any())).thenAnswer(invocationOnMock -> {
             HttpGet get = (HttpGet) invocationOnMock.getArguments()[0];
             mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append("  ");
-            CloseableHttpResponse response = mock(CloseableHttpResponse.class);
-            StatusLine statusLine = mock(StatusLine.class);
-            when(statusLine.getStatusCode()).thenReturn(mockReturnCode);
-            when(response.getStatusLine()).thenReturn(statusLine);
             if (mockReturnCode == 100000) throw new RuntimeException("FAIL");
-            HttpEntity entity = mock(HttpEntity.class);
-            when(response.getEntity()).thenReturn(entity);
+
+            BasicStatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, mockReturnCode, null);
+            BasicHttpEntity entity = new BasicHttpEntity();
             String returnMessage = "{\"foo\":\"bar\", \"no\":3, \"error-code\": " + mockReturnCode + "}";
             InputStream stream = new ByteArrayInputStream(returnMessage.getBytes(StandardCharsets.UTF_8));
-            when(entity.getContent()).thenReturn(stream);
+            entity.setContent(stream);
+
+            CloseableHttpResponse response = mock(CloseableHttpResponse.class);
+            when(response.getEntity()).thenReturn(entity);
+            when(response.getStatusLine()).thenReturn(statusLine);
+
             return response;
         });
         doNothing().when(httpMock).close();
@@ -69,75 +76,61 @@ public class ConfigServerHttpRequestExecutorTest {
 
     @Test
     public void testBasicParsingSingleServer() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         ConfigServerHttpRequestExecutor executor = new ConfigServerHttpRequestExecutor(configServers, createClientMock());
-        TestPojo answer = executor.get("/path", 666, TestPojo.class);
+        TestPojo answer = executor.get("/path", TestPojo.class);
         assertThat(answer.foo, is("bar"));
         assertLogStringContainsGETForAHost();
     }
 
     @Test(expected = HttpException.class)
     public void testBasicFailure() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Server is returning 400, no retries.
         mockReturnCode = 400;
         ConfigServerHttpRequestExecutor executor = new ConfigServerHttpRequestExecutor(configServers, createClientMock());
 
-        TestPojo testPojo = executor.get("/path", 666, TestPojo.class);
+        TestPojo testPojo = executor.get("/path", TestPojo.class);
         assertEquals(testPojo.errorCode.intValue(), mockReturnCode);
         assertLogStringContainsGETForAHost();
     }
 
     @Test
     public void testBasicSuccessWithNoRetries() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Server is returning 201, no retries.
         mockReturnCode = 201;
         ConfigServerHttpRequestExecutor executor = new ConfigServerHttpRequestExecutor(configServers, createClientMock());
 
-        TestPojo testPojo = executor.get("/path", 666, TestPojo.class);
+        TestPojo testPojo = executor.get("/path", TestPojo.class);
         assertEquals(testPojo.errorCode.intValue(), mockReturnCode);
         assertLogStringContainsGETForAHost();
     }
 
     @Test
     public void testRetries() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Client is throwing exception, should be retries.
         mockReturnCode = 100000;
         ConfigServerHttpRequestExecutor executor =
                 new ConfigServerHttpRequestExecutor(configServers, createClientMock());
         try {
-            executor.get("/path", 666, TestPojo.class);
+            executor.get("/path", TestPojo.class);
             fail("Expected failure");
         } catch (Exception e) {
             // ignore
         }
 
         String[] log = mockLog.toString().split("  ");
+        System.out.println(Arrays.toString(log));
         assertThat(log, arrayContainingInAnyOrder("GET http://host1:666/path", "GET http://host2:666/path",
                                          "GET http://host1:666/path", "GET http://host2:666/path"));
     }
 
     @Test
     public void testRetriesOnBadHttpResponseCode() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Client is throwing exception, should be retries.
         mockReturnCode = 503;
         ConfigServerHttpRequestExecutor executor =
                 new ConfigServerHttpRequestExecutor(configServers, createClientMock());
         try {
-            executor.get("/path", 666, TestPojo.class);
+            executor.get("/path", TestPojo.class);
             fail("Expected failure");
         } catch (Exception e) {
             // ignore
@@ -151,14 +144,11 @@ public class ConfigServerHttpRequestExecutorTest {
 
     @Test
     public void testNotFound() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Server is returning 404, special exception is thrown.
         mockReturnCode = 404;
         ConfigServerHttpRequestExecutor executor = new ConfigServerHttpRequestExecutor(configServers, createClientMock());
         try {
-            executor.get("/path", 666, TestPojo.class);
+            executor.get("/path", TestPojo.class);
             fail("Expected exception");
         } catch (HttpException.NotFoundException e) {
             // ignore
@@ -168,14 +158,11 @@ public class ConfigServerHttpRequestExecutorTest {
 
     @Test
     public void testConflict() throws Exception {
-        Set<String> configServers = new ArraySet<>(2);
-        configServers.add("host1");
-        configServers.add("host2");
         // Server is returning 409, no exception is thrown.
         mockReturnCode = 409;
         ConfigServerHttpRequestExecutor executor =
                 new ConfigServerHttpRequestExecutor(configServers, createClientMock());
-        executor.get("/path", 666, TestPojo.class);
+        executor.get("/path", TestPojo.class);
         assertLogStringContainsGETForAHost();
     }