Merge pull request #11183 from vespa-engine/freva/configserver-v1

Create /configserver/v1

14 files changed, 464 insertions, 130 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
index e3c048e865a..4fa7a40d38a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java
@@ -2,9 +2,10 @@
 package com.yahoo.vespa.hosted.controller.proxy;
 
 import com.google.inject.Inject;
-import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.http.HttpRequest.Method;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
@@ -17,14 +18,17 @@ import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 
 import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.net.URI;
 import java.time.Duration;
 import java.util.ArrayList;
@@ -33,7 +37,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import static com.yahoo.yolean.Exceptions.uncheck;
 
@@ -43,33 +49,37 @@ import static com.yahoo.yolean.Exceptions.uncheck;
  * @author bjorncs
  */
 @SuppressWarnings("unused") // Injected
-public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
+public class ConfigServerRestExecutorImpl extends AbstractComponent implements ConfigServerRestExecutor {
 
     private static final Logger log = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName());
 
     private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10);
     private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type");
 
-    private final ZoneRegistry zoneRegistry;
-    private final ServiceIdentityProvider sslContextProvider;
+    private final CloseableHttpClient client;
 
     @Inject
     public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) {
-        this.zoneRegistry = zoneRegistry;
-        this.sslContextProvider = sslContextProvider;
+        RequestConfig config = RequestConfig.custom()
+                .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis())
+                .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis())
+                .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build();
+
+        this.client = createHttpClient(config, sslContextProvider,
+                new ControllerOrConfigserverHostnameVerifier(zoneRegistry));
     }
 
     @Override
     public ProxyResponse handle(ProxyRequest proxyRequest) throws ProxyException {
-        HostnameVerifier hostnameVerifier = createHostnameVerifier(proxyRequest.getZoneId());
-        List<URI> allServers = getConfigserverEndpoints(proxyRequest.getZoneId());
+        // Make a local copy of the list as we want to manipulate it in case of ping problems.
+        List<URI> allServers = new ArrayList<>(proxyRequest.getTargets());
 
         StringBuilder errorBuilder = new StringBuilder();
-        if (queueFirstServerIfDown(allServers, hostnameVerifier)) {
+        if (queueFirstServerIfDown(allServers)) {
             errorBuilder.append("Change ordering due to failed ping.");
         }
         for (URI uri : allServers) {
-            Optional<ProxyResponse> proxyResponse = proxyCall(uri, proxyRequest, hostnameVerifier, errorBuilder);
+            Optional<ProxyResponse> proxyResponse = proxyCall(uri, proxyRequest, errorBuilder);
             if (proxyResponse.isPresent()) {
                 return proxyResponse.get();
             }
@@ -79,32 +89,14 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
                 + errorBuilder.toString()));
     }
 
-    private List<URI> getConfigserverEndpoints(ZoneId zoneId) {
-        // TODO: Use config server VIP for all zones that have one
-        // Make a local copy of the list as we want to manipulate it in case of ping problems.
-        if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) {
-            return List.of(zoneRegistry.getConfigServerVipUri(zoneId));
-        } else {
-            return new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId));
-        }
-    }
-
-    private Optional<ProxyResponse> proxyCall(
-            URI uri, ProxyRequest proxyRequest, HostnameVerifier hostnameVerifier, StringBuilder errorBuilder)
+    private Optional<ProxyResponse> proxyCall(URI uri, ProxyRequest proxyRequest, StringBuilder errorBuilder)
             throws ProxyException {
         final HttpRequestBase requestBase = createHttpBaseRequest(
                 proxyRequest.getMethod(), proxyRequest.createConfigServerRequestUri(uri), proxyRequest.getData());
         // Empty list of headers to copy for now, add headers when needed, or rewrite logic.
         copyHeaders(proxyRequest.getHeaders(), requestBase);
 
-        RequestConfig config = RequestConfig.custom()
-                .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis())
-                .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis())
-                .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build();
-        try (
-                CloseableHttpClient client = createHttpClient(config, sslContextProvider, hostnameVerifier);
-                CloseableHttpResponse response = client.execute(requestBase)
-        ) {
+        try (CloseableHttpResponse response = client.execute(requestBase)) {
             String content = getContent(response);
             int status = response.getStatusLine().getStatusCode();
             if (status / 100 == 5) {
@@ -182,7 +174,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
      * if it is not responding, we try the other servers first. False positive/negatives are not critical,
      * but will increase latency to some extent.
      */
-    private boolean queueFirstServerIfDown(List<URI> allServers, HostnameVerifier hostnameVerifier) {
+    private boolean queueFirstServerIfDown(List<URI> allServers) {
         if (allServers.size() < 2) {
             return false;
         }
@@ -194,10 +186,8 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
                 .setConnectTimeout(timeout)
                 .setConnectionRequestTimeout(timeout)
                 .setSocketTimeout(timeout).build();
-        try (
-                CloseableHttpClient client = createHttpClient(config, sslContextProvider, hostnameVerifier);
-                CloseableHttpResponse response = client.execute(httpget)
-        ) {
+        httpget.setConfig(config);
+        try (CloseableHttpResponse response = client.execute(httpget)) {
             if (response.getStatusLine().getStatusCode() == 200) {
                 return false;
             }
@@ -210,8 +200,13 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
         return true;
     }
 
-    private HostnameVerifier createHostnameVerifier(ZoneId zoneId) {
-        return new AthenzIdentityVerifier(Set.of(zoneRegistry.getConfigServerHttpsIdentity(zoneId)));
+    @Override
+    public void deconstruct() {
+        try {
+            client.close();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
     private static CloseableHttpClient createHttpClient(RequestConfig config,
@@ -222,7 +217,31 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor {
                 .setSslcontext(sslContextProvider.getIdentitySslContext())
                 .setSSLHostnameVerifier(hostnameVerifier)
                 .setDefaultRequestConfig(config)
+                .setMaxConnPerRoute(10)
+                .setMaxConnTotal(500)
+                .setConnectionTimeToLive(1, TimeUnit.MINUTES)
                 .build();
     }
 
+    private static class ControllerOrConfigserverHostnameVerifier implements HostnameVerifier {
+
+        private final HostnameVerifier controllerVerifier = new DefaultHostnameVerifier();
+        private final HostnameVerifier configserverVerifier;
+
+        ControllerOrConfigserverHostnameVerifier(ZoneRegistry registry) {
+            this.configserverVerifier = createConfigserverVerifier(registry);
+        }
+
+        private static HostnameVerifier createConfigserverVerifier(ZoneRegistry registry) {
+            Set<AthenzIdentity> configserverIdentities = registry.zones().all().zones().stream()
+                    .map(zone -> registry.getConfigServerHttpsIdentity(zone.getId()))
+                    .collect(Collectors.toSet());
+            return new AthenzIdentityVerifier(configserverIdentities);
+        }
+
+        @Override
+        public boolean verify(String hostname, SSLSession session) {
+            return controllerVerifier.verify(hostname, session) || configserverVerifier.verify(hostname, session);
+        }
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java
index 100292a0bdc..f398683567b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.proxy;
 
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 
 import java.io.InputStream;
@@ -26,36 +25,36 @@ public class ProxyRequest {
     private final Map<String, List<String>> headers;
     private final InputStream requestData;
 
-    private final ZoneId zoneId;
-    private final String proxyPath;
+    private final List<URI> targets;
+    private final String targetPath;
 
     /**
      * The constructor calls exception if the request is invalid.
      *
      * @param request the request from the jdisc framework.
-     * @param zoneId the zone to proxy to.
-     * @param proxyPath the path to proxy to.
+     * @param targets list of targets this request should be proxied to (targets are tried once in order until a response is returned).
+     * @param targetPath the path to proxy to.
      * @throws ProxyException on errors
      */
-    public ProxyRequest(HttpRequest request, ZoneId zoneId, String proxyPath) throws ProxyException {
+    public ProxyRequest(HttpRequest request, List<URI> targets, String targetPath) throws ProxyException {
         this(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), request.getData(),
-             zoneId, proxyPath);
+             targets, targetPath);
     }
 
     ProxyRequest(Method method, URI requestUri, Map<String, List<String>> headers, InputStream body,
-                 ZoneId zoneId, String proxyPath) throws ProxyException {
+                 List<URI> targets, String targetPath) throws ProxyException {
         Objects.requireNonNull(requestUri, "Request must be non-null");
-        if (!requestUri.getPath().endsWith(proxyPath))
+        if (!requestUri.getPath().endsWith(targetPath))
             throw new ProxyException(ErrorResponse.badRequest(String.format(
-                    "Request path '%s' does not end with proxy path '%s'", requestUri.getPath(), proxyPath)));
+                    "Request path '%s' does not end with proxy path '%s'", requestUri.getPath(), targetPath)));
 
         this.method = Objects.requireNonNull(method);
         this.requestUri = Objects.requireNonNull(requestUri);
         this.headers = Objects.requireNonNull(headers);
         this.requestData = body;
 
-        this.zoneId = Objects.requireNonNull(zoneId);
-        this.proxyPath = proxyPath.startsWith("/") ? proxyPath : "/" + proxyPath;
+        this.targets = List.copyOf(targets);
+        this.targetPath = targetPath.startsWith("/") ? targetPath : "/" + targetPath;
     }
 
 
@@ -71,23 +70,23 @@ public class ProxyRequest {
         return requestData;
     }
 
-    public ZoneId getZoneId() {
-        return zoneId;
+    public List<URI> getTargets() {
+        return targets;
     }
 
     public URI createConfigServerRequestUri(URI baseURI) {
         try {
             return new URI(baseURI.getScheme(), baseURI.getUserInfo(), baseURI.getHost(),
-                    baseURI.getPort(), proxyPath, requestUri.getQuery(), requestUri.getFragment());
+                    baseURI.getPort(), targetPath, requestUri.getQuery(), requestUri.getFragment());
         } catch (URISyntaxException e) {
             throw new RuntimeException(e);
         }
     }
 
     public URI getControllerPrefixUri() {
-        String prefixPath = proxyPath.equals("/") && !requestUri.getPath().endsWith("/") ?
-                requestUri.getPath() + proxyPath :
-                requestUri.getPath().substring(0, requestUri.getPath().length() - proxyPath.length() + 1);
+        String prefixPath = targetPath.equals("/") && !requestUri.getPath().endsWith("/") ?
+                requestUri.getPath() + targetPath :
+                requestUri.getPath().substring(0, requestUri.getPath().length() - targetPath.length() + 1);
         try {
             return new URI(requestUri.getScheme(), requestUri.getUserInfo(), requestUri.getHost(),
                     requestUri.getPort(), prefixPath, null, null);
@@ -98,7 +97,7 @@ public class ProxyRequest {
 
     @Override
     public String toString() {
-        return "[zone: " + zoneId + " request: " + proxyPath + "]";
+        return "[targets: " + targets + " request: " + targetPath + "]";
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java
new file mode 100644
index 00000000000..99cc78a2614
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java
@@ -0,0 +1,127 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.configserver;
+
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.config.provision.zone.ZoneList;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.ErrorResponse;
+import com.yahoo.restapi.Path;
+import com.yahoo.restapi.SlimeJsonResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
+import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor;
+import com.yahoo.vespa.hosted.controller.proxy.ProxyException;
+import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
+import com.yahoo.yolean.Exceptions;
+
+import java.net.URI;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.stream.Stream;
+
+/**
+ * REST API for proxying operator APIs to config servers in a given zone.
+ *
+ * @author freva
+ */
+@SuppressWarnings("unused")
+public class ConfigServerApiHandler extends AuditLoggingRequestHandler {
+
+    private static final String OPTIONAL_PREFIX = "/api";
+    private static final ZoneId CONTROLLER_ZONE = ZoneId.from("prod", "controller");
+    private static final List<String> WHITELISTED_APIS = List.of("/flags/v1/", "/nodes/v2/", "/orchestrator/v1/");
+
+    private final ZoneRegistry zoneRegistry;
+    private final ConfigServerRestExecutor proxy;
+
+    public ConfigServerApiHandler(Context parentCtx, ZoneRegistry zoneRegistry,
+                                  ConfigServerRestExecutor proxy, Controller controller) {
+        super(parentCtx, controller.auditLogger());
+        this.zoneRegistry = zoneRegistry;
+        this.proxy = proxy;
+    }
+
+    @Override
+    public HttpResponse auditAndHandle(HttpRequest request) {
+        try {
+            switch (request.getMethod()) {
+                case GET:
+                    return get(request);
+                case POST:
+                case PUT:
+                case DELETE:
+                case PATCH:
+                    return proxy(request);
+                default:
+                    return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported");
+            }
+        } catch (IllegalArgumentException e) {
+            return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "', "
+                                   + Exceptions.toMessageString(e));
+            return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+        }
+    }
+
+    private HttpResponse get(HttpRequest request) {
+        Path path = new Path(request.getUri(), OPTIONAL_PREFIX);
+        if (path.matches("/configserver/v1")) {
+            return root(request);
+        }
+        return proxy(request);
+    }
+
+    private HttpResponse proxy(HttpRequest request) {
+        Path path = new Path(request.getUri(), OPTIONAL_PREFIX);
+        if ( ! path.matches("/configserver/v1/{environment}/{region}/{*}")) {
+            return ErrorResponse.notFoundError("Nothing at " + path);
+        }
+
+        ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region"));
+        if (! zoneRegistry.hasZone(zoneId) && ! CONTROLLER_ZONE.equals(zoneId)) {
+            throw new IllegalArgumentException("No such zone: " + zoneId.value());
+        }
+
+        String cfgPath = "/" + path.getRest();
+        if (WHITELISTED_APIS.stream().noneMatch(cfgPath::startsWith)) {
+            return ErrorResponse.forbidden("Cannot access '" + cfgPath +
+                    "' through /configserver/v1, following APIs are permitted: " + String.join(", ", WHITELISTED_APIS));
+        }
+
+        try {
+            return proxy.handle(new ProxyRequest(request, List.of(getEndpoint(zoneId)), cfgPath));
+        } catch (ProxyException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private HttpResponse root(HttpRequest request) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        ZoneList zoneList = zoneRegistry.zones().reachable();
+
+        Cursor zones = root.setArray("zones");
+        Stream.concat(Stream.of(CONTROLLER_ZONE), zoneRegistry.zones().reachable().ids().stream())
+                .forEach(zone -> {
+            Cursor object = zones.addObject();
+            object.setString("environment", zone.environment().value());
+            object.setString("region", zone.region().value());
+            object.setString("uri", request.getUri().resolve(
+                    "/configserver/v1/" + zone.environment().value() + "/" + zone.region().value()).toString());
+        });
+        return new SlimeJsonResponse(slime);
+    }
+
+    private HttpResponse notFound(Path path) {
+        return ErrorResponse.notFoundError("Nothing at " + path);
+    }
+
+    private URI getEndpoint(ZoneId zoneId) {
+        return CONTROLLER_ZONE.equals(zoneId) ? zoneRegistry.apiUrl() : zoneRegistry.getConfigServerVipUri(zoneId);
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java
new file mode 100644
index 00000000000..9949c2d17bf
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author freva
+ */
+package com.yahoo.vespa.hosted.controller.restapi.configserver;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
index be601511763..1a7002c5759 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
@@ -19,6 +19,8 @@ import com.yahoo.vespa.hosted.controller.proxy.ProxyException;
 import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
 import com.yahoo.yolean.Exceptions;
 
+import java.net.URI;
+import java.util.List;
 import java.util.logging.Level;
 
 /**
@@ -82,7 +84,7 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler {
             throw new IllegalArgumentException("No such zone: " + zoneId.value());
         }
         try {
-            return proxy.handle(new ProxyRequest(request, zoneId, path.getRest()));
+            return proxy.handle(new ProxyRequest(request, getConfigserverEndpoints(zoneId), path.getRest()));
         } catch (ProxyException e) {
             throw new RuntimeException(e);
         }
@@ -110,4 +112,13 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler {
     private HttpResponse notFound(Path path) {
         return ErrorResponse.notFoundError("Nothing at " + path);
     }
+
+    private List<URI> getConfigserverEndpoints(ZoneId zoneId) {
+        // TODO: Use config server VIP for all zones that have one
+        if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) {
+            return List.of(zoneRegistry.getConfigServerVipUri(zoneId));
+        } else {
+            return zoneRegistry.getConfigServerUris(zoneId);
+        }
+    }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 32bbf3ceb9b..f5158a1ffa2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
-import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
@@ -24,8 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 
 import java.net.URI;
 import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -38,7 +35,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>();
     private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>();
-    private List<ZoneApi> zones = new ArrayList<>();
+    private List<ZoneApi> zones = List.of();
     private SystemName system;
     private UpgradePolicy upgradePolicy = null;
     private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
@@ -136,7 +133,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     @Override
     public List<UpgradePolicy> osUpgradePolicies() {
-        return ImmutableList.copyOf(osUpgradePolicies.values());
+        return List.copyOf(osUpgradePolicies.values());
     }
 
     @Override
@@ -176,7 +173,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     @Override
     public List<URI> getConfigServerUris(ZoneId zoneId) {
-        return Collections.singletonList(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value())));
+        return List.of(
+                URI.create(String.format("https://cfg1.%s.test:4443/", zoneId.value())),
+                URI.create(String.format("https://cfg2.%s.test:4443/", zoneId.value())));
     }
 
     @Override
@@ -186,11 +185,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     @Override
     public List<URI> getConfigServerApiUris(ZoneId zoneId) {
-        List<URI> uris = new ArrayList<URI>();
-        uris.add(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value())));
-        uris.add(URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value())));
-
-        return uris;
+        return List.of(
+                URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value())),
+                URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value())));
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java
index b9d68c2a3da..d8373cb8928 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java
@@ -1,13 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.proxy;
 
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.http.HttpRequest;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import java.net.URI;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -23,7 +23,7 @@ public class ProxyRequestTest {
     @Test
     public void testEmpty() throws Exception {
         exception.expectMessage("Request must be non-null");
-        new ProxyRequest(HttpRequest.Method.GET, null, Map.of(), null, ZoneId.from("dev", "us-north-1"), "/zone/v2");
+        new ProxyRequest(HttpRequest.Method.GET, null, Map.of(), null, List.of(), "/zone/v2");
     }
 
     @Test
@@ -69,6 +69,6 @@ public class ProxyRequestTest {
 
     private static ProxyRequest testRequest(String url, String pathPrefix) throws ProxyException {
         return new ProxyRequest(
-                HttpRequest.Method.GET, URI.create(url), Map.of(), null, ZoneId.from("dev", "us-north-1"), pathPrefix);
+                HttpRequest.Method.GET, URI.create(url), Map.of(), null, List.of(), pathPrefix);
     }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java
index efe7e17c58e..0aac59321b5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java
@@ -1,13 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.proxy;
 
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.http.HttpRequest;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -20,7 +20,7 @@ public class ProxyResponseTest {
     @Test
     public void testRewriteUrl() throws Exception {
         ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("http://domain.tld/zone/v2/dev/us-north-1/configserver"),
-                Map.of(), null, ZoneId.from("dev", "us-north-1"), "configserver");
+                Map.of(), null, List.of(), "configserver");
         ProxyResponse proxyResponse = new ProxyResponse(
                 request,
                 "response link is http://configserver:1234/bla/bla/",
@@ -38,7 +38,7 @@ public class ProxyResponseTest {
     @Test
     public void testRewriteSecureUrl() throws Exception {
         ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("https://domain.tld/zone/v2/prod/eu-south-3/configserver"),
-                Map.of(), null, ZoneId.from("prod", "eu-south-3"), "configserver");
+                Map.of(), null, List.of(), "configserver");
         ProxyResponse proxyResponse = new ProxyResponse(
                 request,
                 "response link is http://configserver:1234/bla/bla/",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index fb0e92ab7f4..1a53920e8de 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -34,12 +34,16 @@ import static org.junit.Assert.assertEquals;
  */
 public class ControllerContainerTest {
 
+    private static final AthenzUser hostedOperator = AthenzUser.fromUserId("alice");
     private static final AthenzUser defaultUser = AthenzUser.fromUserId("bob");
 
     protected JDisc container;
 
     @Before
-    public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); }
+    public void startContainer() {
+        container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable);
+        addUserToHostedOperatorRole(hostedOperator);
+    }
 
     @After
     public void stopContainer() { container.close(); }
@@ -92,6 +96,12 @@ public class ControllerContainerTest {
                "    <binding>http://*/zone/v2</binding>\n" +
                "    <binding>http://*/zone/v2/*</binding>\n" +
                "  </handler>\n" +
+               "  <handler id='com.yahoo.vespa.hosted.controller.restapi.configserver.ConfigServerApiHandler'>\n" +
+               "    <binding>http://*/configserver/v1</binding>\n" +
+               "    <binding>http://*/configserver/v1/*</binding>\n" +
+               "    <binding>http://*/api/configserver/v1</binding>\n" +
+               "    <binding>http://*/api/configserver/v1/*</binding>\n" +
+               "  </handler>\n" +
                "  <handler id='com.yahoo.vespa.hosted.controller.restapi.flags.AuditedFlagsHandler'>\n" +
                "    <binding>http://*/flags/v1</binding>\n" +
                "    <binding>http://*/flags/v1/*</binding>\n" +
@@ -147,10 +157,18 @@ public class ControllerContainerTest {
         return addIdentityToRequest(new Request(uri), defaultUser);
     }
 
-    protected static Request authenticatedRequest(String uri, byte[] body, Request.Method method) {
+    protected static Request authenticatedRequest(String uri, String body, Request.Method method) {
         return addIdentityToRequest(new Request(uri, body, method), defaultUser);
     }
 
+    protected static Request operatorRequest(String uri) {
+        return addIdentityToRequest(new Request(uri), hostedOperator);
+    }
+
+    protected static Request operatorRequest(String uri, String body, Request.Method method) {
+        return addIdentityToRequest(new Request(uri, body, method), hostedOperator);
+    }
+
     protected static Request addIdentityToRequest(Request request, AthenzIdentity identity) {
         request.getHeaders().put(IDENTITY_HEADER_NAME, identity.getFullName());
         return request;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java
new file mode 100644
index 00000000000..27af81ef1f9
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java
@@ -0,0 +1,144 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.configserver;
+
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URI;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author freva
+ */
+public class ConfigServerApiHandlerTest extends ControllerContainerTest {
+
+    private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/";
+    private static final List<ZoneApi> zones = List.of(
+            ZoneApiMock.fromId("prod.us-north-1"),
+            ZoneApiMock.fromId("dev.aws-us-north-2"),
+            ZoneApiMock.fromId("test.us-north-3"),
+            ZoneApiMock.fromId("staging.us-north-4"));
+
+    private ContainerControllerTester tester;
+    private ConfigServerProxyMock proxy;
+
+    @Before
+    public void before() {
+        ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components()
+                .getComponent(ZoneRegistryMock.class.getName());
+        zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2"))
+                .setZones(zones);
+        this.tester = new ContainerControllerTester(container, responseFiles);
+        this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName());
+    }
+
+    @Test
+    public void test_requests() {
+        // GET /configserver/v1
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1"),
+                new File("root.json"));
+
+        // GET /configserver/v1/nodes/v2/node/?recursive=true
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node/?recursive=true"),
+                "ok");
+        assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "GET");
+
+        // POST /configserver/v1/dev/us-north-2/nodes/v2/command/restart?hostname=node1
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/dev/aws-us-north-2/nodes/v2/command/restart?hostname=node1",
+                "", Request.Method.POST),
+                "ok");
+
+        // PUT /configserver/v1/prod/us-north-1/nodes/v2/state/dirty/node1
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/state/dirty/node1",
+                "", Request.Method.PUT), "ok");
+        assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "PUT");
+
+        // DELETE /configserver/v1/prod/us-north-1/nodes/v2/node/node1
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/api/configserver/v1/prod/controller/nodes/v2/node/node1",
+                "", Request.Method.DELETE), "ok");
+        assertLastRequest("https://api.tld:4443/", "DELETE");
+
+        // PATCH /configserver/v1/prod/us-north-1/nodes/v2/node/node1
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/dev/aws-us-north-2/nodes/v2/node/node1",
+                "{\"currentRestartGeneration\": 1}",
+                Request.Method.PATCH), "ok");
+        assertLastRequest("https://cfg.dev.aws-us-north-2.test.vip:4443/", "PATCH");
+        assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get());
+
+        assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty());
+    }
+
+    @Test
+    public void test_allowed_apis() {
+        // GET /configserver/v1/prod/us-north-1
+        tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1"),
+                "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}",
+                403);
+
+        tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/application/v2/tenant/vespa"),
+                "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/application/v2/tenant/vespa' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}",
+                403);
+    }
+
+    @Test
+    public void test_invalid_requests() {
+        // POST /configserver/v1/prod/us-north-34/nodes/v2
+        tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-42/nodes/v2",
+                "", Request.Method.POST),
+                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such zone: prod.us-north-42\"}", 400);
+        assertFalse(proxy.lastReceived().isPresent());
+    }
+
+    @Test
+    public void non_operators_are_forbidden() {
+        // Read request
+        tester.containerTester().assertResponse(() -> authenticatedRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node"),
+                "{\n" +
+                "  \"code\" : 403,\n" +
+                "  \"message\" : \"Access denied\"\n" +
+                "}", 403);
+
+        // Write request
+        tester.containerTester().assertResponse(() -> authenticatedRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.POST),
+                "{\n" +
+                "  \"code\" : 403,\n" +
+                "  \"message\" : \"Access denied\"\n" +
+                "}", 403);
+    }
+
+    @Test
+    public void unauthenticated_request_are_unauthorized() {
+        {
+            // Read request
+            Request request = new Request("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.GET);
+            tester.containerTester().assertResponse(() -> request, "{\n  \"message\" : \"Not authenticated\"\n}", 401);
+        }
+
+        {
+            // Write request
+            Request request = new Request("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.POST);
+            tester.containerTester().assertResponse(() -> request, "{\n  \"message\" : \"Not authenticated\"\n}", 401);
+        }
+    }
+
+
+    private void assertLastRequest(String target, String method) {
+        ProxyRequest last = proxy.lastReceived().orElseThrow();
+        assertEquals(List.of(URI.create(target)), last.getTargets());
+        assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod());
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json
new file mode 100644
index 00000000000..5ccf75d2448
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json
@@ -0,0 +1,29 @@
+{
+  "zones": [
+    {
+      "environment": "prod",
+      "region": "controller",
+      "uri": "http://localhost:8080/configserver/v1/prod/controller"
+    },
+    {
+      "environment": "prod",
+      "region": "us-north-1",
+      "uri": "http://localhost:8080/configserver/v1/prod/us-north-1"
+    },
+    {
+      "environment": "dev",
+      "region": "aws-us-north-2",
+      "uri": "http://localhost:8080/configserver/v1/dev/aws-us-north-2"
+    },
+    {
+      "environment": "test",
+      "region": "us-north-3",
+      "uri": "http://localhost:8080/configserver/v1/test/us-north-3"
+    },
+    {
+      "environment": "staging",
+      "region": "us-north-4",
+      "uri": "http://localhost:8080/configserver/v1/staging/us-north-4"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 74d637499bd..13e82e5132e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.controller;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
@@ -27,48 +25,46 @@ import static org.junit.Assert.assertFalse;
 public class ControllerApiTest extends ControllerContainerTest {
 
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
-    private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator");
 
     private ContainerControllerTester tester;
 
     @Before
     public void before() {
-        addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR);
         tester = new ContainerControllerTester(container, responseFiles);
     }
 
     @Test
     public void testControllerApi() {
-        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", new byte[0], Request.Method.GET), new File("root.json"));
+        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", "", Request.Method.GET), new File("root.json"));
 
         // POST deactivates a maintenance job
-        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+        tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
                                             "", Request.Method.POST),
                        "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}", 200);
 
         // GET a list of all maintenance jobs
-        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", new byte[0], Request.Method.GET),
+        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", "", Request.Method.GET),
                               new File("maintenance.json"));
 
         // DELETE activates maintenance job
-        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+        tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
                                             "", Request.Method.DELETE),
                        "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}",
                               200);
 
         // DELETE fails to activate unknown maintenance job
-        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/foo",
+        tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/foo",
                                                     "", Request.Method.DELETE),
                               "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'foo'\"}",
                               404);
 
         // DELETE clears inactive flag for maintenance job that has been removed from the code base
         tester.controller().curator().writeInactiveJobs(Set.of("bar"));
-        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar",
+        tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar",
                                                     "", Request.Method.DELETE),
                               "{\"message\":\"Re-activated job 'bar'\"}",
                               200);
-        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar",
+        tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar",
                                                     "", Request.Method.DELETE),
                               "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'bar'\"}",
                               404);
@@ -79,55 +75,55 @@ public class ControllerApiTest extends ControllerContainerTest {
     @Test
     public void testUpgraderApi() {
         // Get current configuration
-        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", new byte[0], Request.Method.GET),
+        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", "", Request.Method.GET),
                               "{\"upgradesPerMinute\":100.0,\"confidenceOverrides\":[]}",
                               200);
 
         // Set invalid configuration
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":-1}", Request.Method.PATCH),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":-1}", Request.Method.PATCH),
                 "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0, got -1.0\"}",
                 400);
 
         // Ignores unrecognized field
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader","{\"foo\":\"bar\"}", Request.Method.PATCH),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"foo\":\"bar\"}", Request.Method.PATCH),
                 "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such modifiable field(s)\"}",
                 400);
 
         // Set upgrades per minute
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":42.0}", Request.Method.PATCH),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":42.0}", Request.Method.PATCH),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}",
                 200);
 
         // Set target major version
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":6}", Request.Method.PATCH),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":6}", Request.Method.PATCH),
                 "{\"upgradesPerMinute\":42.0,\"targetMajorVersion\":6,\"confidenceOverrides\":[]}",
                 200);
 
         // Clear target major version
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":null}", Request.Method.PATCH),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":null}", Request.Method.PATCH),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}",
                 200);
 
         // Override confidence
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"}]}",
                 200);
 
         // Override confidence for another version
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", "broken", Request.Method.POST),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", "broken", Request.Method.POST),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"},{\"6.43\":\"broken\"}]}",
                 200);
 
         // Remove first override
         tester.assertResponse(
-                hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE),
+                operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.43\":\"broken\"}]}",
                 200);
 
@@ -160,8 +156,4 @@ public class ControllerApiTest extends ControllerContainerTest {
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json"));
     }
 
-    private static Request hostedOperatorRequest(String uri, String body, Request.Method method) {
-        return addIdentityToRequest(new Request(uri, body, method), HOSTED_VESPA_OPERATOR);
-    }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
index 33ea538e9b6..40562ba493e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
@@ -1,15 +1,11 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.zone.v2;
 
-import com.yahoo.application.container.handler.Request;
 import com.yahoo.application.container.handler.Request.Method;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
@@ -24,17 +20,17 @@ import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author mpolden
  */
 public class ZoneApiTest extends ControllerContainerTest {
 
-    private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator");
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/";
     private static final List<ZoneApi> zones = List.of(
             ZoneApiMock.fromId("prod.us-north-1"),
-            ZoneApiMock.fromId("dev.us-north-2"),
+            ZoneApiMock.fromId("dev.aws-us-north-2"),
             ZoneApiMock.fromId("test.us-north-3"),
             ZoneApiMock.fromId("staging.us-north-4"));
 
@@ -49,7 +45,6 @@ public class ZoneApiTest extends ControllerContainerTest {
                     .setZones(zones);
         this.tester = new ContainerControllerTester(container, responseFiles);
         this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName());
-        addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR);
     }
 
     @Test
@@ -61,33 +56,34 @@ public class ZoneApiTest extends ControllerContainerTest {
         // GET /zone/v2/prod/us-north-1
         tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v2/prod/us-north-1"),
                                                 "ok");
-        assertLastRequest(ZoneId.from("prod", "us-north-1"), "GET");
+
+        assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "GET");
 
         // GET /zone/v2/nodes/v2/node/?recursive=true
         tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/?recursive=true"),
                                                 "ok");
-        assertLastRequest(ZoneId.from("prod", "us-north-1"), "GET");
+        assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "GET");
 
         // POST /zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1
-        tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1",
-                                                            new byte[0], Method.POST),
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/command/restart?hostname=node1",
+                                                            "", Method.POST),
                                                 "ok");
 
         // PUT /zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1
-        tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1",
-                                                            new byte[0], Method.PUT), "ok");
-        assertLastRequest(ZoneId.from("prod", "us-north-1"), "PUT");
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1",
+                                                            "", Method.PUT), "ok");
+        assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "PUT");
 
         // DELETE /zone/v2/prod/us-north-1/nodes/v2/node/node1
-        tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1",
-                                                            new byte[0], Method.DELETE), "ok");
-        assertLastRequest(ZoneId.from("prod", "us-north-1"), "DELETE");
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1",
+                                                            "", Method.DELETE), "ok");
+        assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "DELETE");
 
         // PATCH /zone/v2/prod/us-north-1/nodes/v2/node/node1
-        tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1",
-                                                            Utf8.toBytes("{\"currentRestartGeneration\": 1}"),
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/node/node1",
+                                                            "{\"currentRestartGeneration\": 1}",
                                                             Method.PATCH), "ok");
-        assertLastRequest(ZoneId.from("prod", "us-north-1"), "PATCH");
+        assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 1, "PATCH");
         assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get());
 
         assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty());
@@ -96,20 +92,17 @@ public class ZoneApiTest extends ControllerContainerTest {
     @Test
     public void test_invalid_requests() {
         // POST /zone/v2/prod/us-north-34/nodes/v2
-        tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2",
-                                                            new byte[0], Method.POST),
+        tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2",
+                                                            "", Method.POST),
                                                 new File("unknown-zone.json"), 400);
         assertFalse(proxy.lastReceived().isPresent());
     }
 
-    private void assertLastRequest(ZoneId zoneId, String method) {
+    private void assertLastRequest(ZoneId zoneId, int targets, String method) {
         ProxyRequest last = proxy.lastReceived().orElseThrow();
-        assertEquals(zoneId, last.getZoneId());
+        assertEquals(targets, last.getTargets().size());
+        assertTrue(last.getTargets().get(0).toString().contains(zoneId.value()));
         assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod());
     }
 
-    private static Request hostedOperatorRequest(String uri, byte[] body, Request.Method method) {
-        return addIdentityToRequest(new Request(uri, body, method), HOSTED_VESPA_OPERATOR);
-    }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json
index ab168854267..bd1bc40ba81 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json
@@ -1,7 +1,7 @@
 {
   "uris": [
     "http://localhost:8080/zone/v2/prod/us-north-1",
-    "http://localhost:8080/zone/v2/dev/us-north-2",
+    "http://localhost:8080/zone/v2/dev/aws-us-north-2",
     "http://localhost:8080/zone/v2/test/us-north-3",
     "http://localhost:8080/zone/v2/staging/us-north-4"
   ],
@@ -12,7 +12,7 @@
     },
     {
       "environment": "dev",
-      "region": "us-north-2"
+      "region": "aws-us-north-2"
     },
     {
       "environment": "test",