Adds Config Server application handler for Cluster Controller status page

18 files changed, 343 insertions, 40 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 3150b2c2cfb..03b5c75107e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -15,6 +15,7 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.application.TenantApplications;
 import com.yahoo.vespa.config.server.configchange.ConfigChangeActions;
@@ -57,6 +58,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Curator curator;
     private final LogServerLogGrabber logServerLogGrabber;
     private final ApplicationConvergenceChecker convergeChecker;
+    private final HttpProxy httpProxy;
     private final Clock clock;
     private final DeployLogger logger = new SilentDeployLogger();
 
@@ -64,12 +66,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  HostProvisionerProvider hostProvisionerProvider,
                                  Curator curator,
                                  LogServerLogGrabber logServerLogGrabber,
-                                 ApplicationConvergenceChecker applicationConvergenceChecker) {
+                                 ApplicationConvergenceChecker applicationConvergenceChecker,
+                                 HttpProxy httpProxy) {
         this.tenants = tenants;
         this.hostProvisioner = hostProvisionerProvider.getHostProvisioner();
         this.curator = curator;
         this.logServerLogGrabber = logServerLogGrabber;
         this.convergeChecker = applicationConvergenceChecker;
+        this.httpProxy = httpProxy;
         this.clock = Clock.systemUTC();
     }
 
@@ -159,6 +163,22 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return convergeChecker.serviceListToCheckForConfigConvergence(application, uri);
     }
 
+    public HttpResponse clusterControllerStatusPage(
+            Tenant tenant,
+            ApplicationId applicationId,
+            String hostName,
+            String pathSuffix) {
+        Application application = getApplication(tenant, applicationId);
+
+        // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise...
+        // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise
+        // change the hostname and port. Exposing other paths on the cluster controller should be fine.
+        // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components.
+        String relativePath = "clustercontroller-status/" + pathSuffix;
+
+        return httpProxy.get(application, hostName, "container-clustercontroller", relativePath);
+    }
+
     public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) {
         return getApplication(tenant, applicationId).getApplicationGeneration();
     }
@@ -307,5 +327,4 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         TenantApplications applicationRepo = tenant.getApplicationRepo();
         return getLocalSession(tenant, applicationRepo.getSessionIdForApplication(applicationId));
     }
-
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java
new file mode 100644
index 00000000000..83950d7ce03
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.application;
+
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.server.http.BadRequestException;
+import com.yahoo.vespa.config.server.http.HttpErrorResponse;
+import com.yahoo.vespa.config.server.http.HttpFetcher;
+import com.yahoo.vespa.config.server.http.InternalServerException;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class HttpProxy {
+    private static Logger logger = Logger.getLogger(HttpProxy.class.getName());
+
+    private final HttpFetcher fetcher;
+
+    public HttpProxy(HttpFetcher fetcher) {
+        this.fetcher = fetcher;
+    }
+
+    public HttpResponse get(Application application, String hostName, String serviceType, String relativePath) {
+        HostInfo host = application.getModel().getHosts().stream()
+                .filter(hostInfo -> hostInfo.getHostname().equals(hostName))
+                .findFirst()
+                .orElseThrow(() -> new BadRequestException("Failed to find host " + hostName));
+
+        ServiceInfo service = host.getServices().stream()
+                .filter(serviceInfo -> serviceType.equals(serviceInfo.getServiceType()))
+                .findFirst()
+                .orElseThrow(() -> new BadRequestException("Failed to find any service of type " + serviceType
+                        + " on host " + hostName));
+
+        // "http" and "state" seems to uniquely identify an interesting HTTP port on each service
+        PortInfo port = service.getPorts().stream()
+                .filter(portInfo -> !portInfo.getTags().stream().collect(Collectors.toSet()).containsAll(
+                        Stream.of("http", "state").collect(Collectors.toSet())))
+                .findFirst()
+                .orElseThrow(() -> new InternalServerException("Failed to find HTTP status port"));
+
+        return internalGet(host.getHostname(), port.getPort(), relativePath);
+    }
+
+    private HttpResponse internalGet(String hostname, int port, String relativePath) {
+        String urlString = "http://" + hostname + ":" + port + "/" + relativePath;
+        URL url;
+        try {
+            url = new URL(urlString);
+        } catch (MalformedURLException e) {
+            logger.log(LogLevel.WARNING, "Badly formed url: " + urlString, e);
+            return HttpErrorResponse.internalServerError("Failed to construct URL for backend");
+        }
+
+        HttpFetcher.Params params = new HttpFetcher.Params(2000); // 2_000 ms read timeout
+        return fetcher.get(params, url);
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
index b5886992f10..d1746b1ee24 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
@@ -38,7 +38,8 @@ public class HttpErrorResponse extends HttpResponse {
         INTERNAL_SERVER_ERROR,
         INVALID_APPLICATION_PACKAGE,
         UNKNOWN_VESPA_VERSION,
-        OUT_OF_CAPACITY
+        OUT_OF_CAPACITY,
+        REQUEST_TIMEOUT
     }
 
     public static HttpErrorResponse notFoundError(String msg) {
@@ -69,6 +70,10 @@ public class HttpErrorResponse extends HttpResponse {
         return new HttpErrorResponse(BAD_REQUEST, errorCodes.UNKNOWN_VESPA_VERSION.name(), message);
     }
 
+    public static HttpResponse requestTimeout(String message) {
+        return new HttpErrorResponse(REQUEST_TIMEOUT, errorCodes.REQUEST_TIMEOUT.name(), message);
+    }
+
     @Override
     public void render(OutputStream stream) throws IOException {
         new JsonFormat(true).encode(stream, slime);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java
new file mode 100644
index 00000000000..6cf065f3185
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http;
+
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.net.URL;
+
+public interface HttpFetcher {
+    class Params {
+        // See HttpUrlConnection::setReadTimeout. 0 means infinite (not recommended!).
+        public final int readTimeoutMs;
+
+        public Params(int readTimeoutMs) {
+            this.readTimeoutMs = readTimeoutMs;
+        }
+    }
+
+    // On failure to get or build HttpResponse for url, an exception is thrown to be handled by HttpHandler.
+    HttpResponse get(Params params, URL url);
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
index 77fd35f9c0a..d7600dccc6c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
@@ -58,6 +58,8 @@ public class HttpHandler extends LoggingRequestHandler {
             return HttpErrorResponse.internalServerError(getMessage(e, request));
         } catch (UnknownVespaVersionException e) {
             return HttpErrorResponse.unknownVespaVersion(getMessage(e, request));
+        } catch (RequestTimeoutException e) {
+            return HttpErrorResponse.requestTimeout(getMessage(e, request));
         } catch (Exception e) {
             e.printStackTrace();
             return HttpErrorResponse.internalServerError(getMessage(e, request));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java
new file mode 100644
index 00000000000..114c6c08bb6
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class ProxyResponse extends HttpResponse {
+    private final String contentType;
+    private final InputStream inputStream;
+
+    /**
+     *
+     * @param status
+     * @param contentType
+     * @param inputStream    Ownership is passed to ProxyResponse (responsible for closing it)
+     */
+    public ProxyResponse(int status, String contentType, InputStream inputStream) {
+        super(status);
+        this.contentType = contentType;
+        this.inputStream = inputStream;
+    }
+
+    @Override
+    public void render(OutputStream outputStream) throws IOException {
+        IOUtils.copy(inputStream, outputStream);
+        inputStream.close();
+    }
+
+    @Override
+    public String getContentType() {
+        return contentType;
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/RequestTimeoutException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/RequestTimeoutException.java
new file mode 100644
index 00000000000..9af5eaef3e3
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/RequestTimeoutException.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http;
+
+/**
+ * Will create an exception which will result in a Request Timeout, 408.
+ */
+public class RequestTimeoutException extends RuntimeException {
+    public RequestTimeoutException(String message) {
+        super(message);
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java
new file mode 100644
index 00000000000..5fe61d5dc8d
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.log.LogLevel;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.SocketTimeoutException;
+import java.net.URL;
+import java.util.logging.Logger;
+
+public class SimpleHttpFetcher implements HttpFetcher {
+    private static final Logger logger = Logger.getLogger(SimpleHttpFetcher.class.getName());
+
+    @Override
+    public HttpResponse get(Params params, URL url) {
+        try {
+            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection.setReadTimeout(params.readTimeoutMs);
+            int code = connection.getResponseCode();
+            String contentType = connection.getContentType();
+            try (InputStream inputStream = connection.getInputStream()) {
+                ProxyResponse response = new ProxyResponse(code, contentType, inputStream);
+                return response;
+            }
+        } catch (SocketTimeoutException e) {
+            String message = "Timed out after " + params.readTimeoutMs + " ms reading response from " + url;
+            logger.log(LogLevel.WARNING, message, e);
+            throw new RequestTimeoutException(message);
+        } catch (IOException e) {
+            String message = "Failed to get response from " + url;
+            logger.log(LogLevel.WARNING, message, e);
+            throw new InternalServerException(message);
+        }
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 2c11a7303e2..62df83e5916 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -65,6 +65,14 @@ public class ApplicationHandler extends HttpHandler {
         if (isServiceConvergeRequest(request)) {
             return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostFromRequest(request), request.getUri());
         }
+
+        if (isClusterControllerStatusRequest(request)) {
+            String hostName = getHostNameFromRequest(request);
+            String pathSuffix = getPathSuffix(request);
+            return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix);
+        }
+
+        // This matches if group count > 7!? It should claim a namespace... We will have to mangle the namespace above.
         if (isContentRequest(request)) {
             long sessionId = applicationRepository.getSessionIdForApplication(tenant, applicationId);
             String contentPath = ApplicationContentRequest.getContentPath(request);
@@ -85,6 +93,7 @@ public class ApplicationHandler extends HttpHandler {
         if (isServiceConvergeListRequest(request)) {
             return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri());
         }
+
         return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId));
     }
 
@@ -148,6 +157,7 @@ public class ApplicationHandler extends HttpHandler {
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*",
+                "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
                 "http://*/application/v2/tenant/*/application/*");
     }
@@ -162,6 +172,11 @@ public class ApplicationHandler extends HttpHandler {
                 request.getUri().getPath().contains("/serviceconverge/");
     }
 
+    private static boolean isClusterControllerStatusRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 9 &&
+                request.getUri().getPath().contains("/clustercontroller/");
+    }
+
     private static boolean isContentRequest(HttpRequest request) {
         return getBindingMatch(request).groupCount() > 7;
     }
@@ -171,6 +186,16 @@ public class ApplicationHandler extends HttpHandler {
         return bm.group(7);
     }
 
+    private static String getHostNameFromRequest(HttpRequest req) {
+        BindingMatch<?> bm = getBindingMatch(req);
+        return bm.group(7);
+    }
+
+    private static String getPathSuffix(HttpRequest req) {
+        BindingMatch<?> bm = getBindingMatch(req);
+        return bm.group(8);
+    }
+
     private static ApplicationId getApplicationIdFromRequest(HttpRequest req) {
         // Two bindings for this: with full app id or only application name
         BindingMatch<?> bm = getBindingMatch(req);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index 659d82379da..07fb3cd51c7 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -29,7 +29,9 @@ import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.TimeoutBudget;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
@@ -149,7 +151,8 @@ public class DeployTester {
                                                                                 HostProvisionerProvider.withProvisioner(createHostProvisioner()),
                                                                                 curator,
                                                                                 new LogServerLogGrabber(),
-                                                                                new ApplicationConvergenceChecker());
+                                                                                new ApplicationConvergenceChecker(),
+                                                                                new HttpProxy(new SimpleHttpFetcher()));
 
         return applicationRepository.deployFromLocalActive(id, Duration.ofSeconds(60));
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java
index 81ade07fd5a..6eb730d11dc 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java
@@ -11,8 +11,10 @@ import com.yahoo.jdisc.Response;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.http.ContentHandlerTestBase;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.Session;
 import com.yahoo.vespa.curator.mock.MockCurator;
@@ -62,7 +64,8 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase {
                                                                    HostProvisionerProvider.empty(),
                                                                    new MockCurator(),
                                                                    new LogServerLogGrabber(),
-                                                                   new ApplicationConvergenceChecker()));
+                                                                   new ApplicationConvergenceChecker(),
+                                                                   new HttpProxy(new SimpleHttpFetcher())));
         pathPrefix = createPath(idTenant1, Zone.defaultZone());
         baseUrl = baseServer + pathPrefix;
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index d7fd292c07e..f695c533b0f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LiteralResponse;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.jdisc.Response;
 import com.yahoo.path.Path;
@@ -19,25 +20,27 @@ import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.GlobalComponentRegistry;
 import com.yahoo.vespa.config.server.MockReloadHandler;
 import com.yahoo.vespa.config.server.SuperModelGenerationCounter;
+import com.yahoo.vespa.config.server.TestComponentRegistry;
+import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
+import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.application.TenantApplications;
 import com.yahoo.vespa.config.server.application.ZKTenantApplications;
+import com.yahoo.vespa.config.server.http.HandlerTest;
+import com.yahoo.vespa.config.server.http.HttpErrorResponse;
+import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
+import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.LocalSession;
+import com.yahoo.vespa.config.server.session.MockSessionZKClient;
+import com.yahoo.vespa.config.server.session.RemoteSession;
 import com.yahoo.vespa.config.server.session.SessionContext;
 import com.yahoo.vespa.config.server.session.SessionZooKeeperClient;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantBuilder;
 import com.yahoo.vespa.config.server.tenant.Tenants;
-import com.yahoo.vespa.config.server.TestComponentRegistry;
-import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
-import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
-import com.yahoo.vespa.config.server.http.HandlerTest;
-import com.yahoo.vespa.config.server.http.HttpErrorResponse;
-import com.yahoo.vespa.config.server.http.SessionHandlerTest;
-import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
-import com.yahoo.vespa.config.server.session.MockSessionZKClient;
-import com.yahoo.vespa.config.server.session.RemoteSession;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.model.VespaModelFactory;
 import org.junit.Assert;
@@ -54,10 +57,14 @@ import java.util.Collections;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * @author hmusum
@@ -74,6 +81,7 @@ public class ApplicationHandlerTest {
     private Tenants tenants;
     private SessionActiveHandlerTest.MockProvisioner provisioner;
     private MockStateApiFactory stateApiFactory = new MockStateApiFactory();
+    private final HttpProxy mockHttpProxy = mock(HttpProxy.class);
 
     @Before
     public void setup() throws Exception {
@@ -84,7 +92,10 @@ public class ApplicationHandlerTest {
         tenants = testBuilder.createTenants();
         provisioner = new SessionActiveHandlerTest.MockProvisioner();
         mockHandler = createMockApplicationHandler(
-                provisioner, new ApplicationConvergenceChecker(stateApiFactory), new LogServerLogGrabber());
+                provisioner,
+                new ApplicationConvergenceChecker(stateApiFactory),
+                mockHttpProxy,
+                new LogServerLogGrabber());
         listApplicationsHandler = new ListApplicationsHandler(
                 Runnable::run, AccessLog.voidAccessLog(), tenants, Zone.defaultZone());
     }
@@ -92,6 +103,7 @@ public class ApplicationHandlerTest {
     private ApplicationHandler createMockApplicationHandler(
             Provisioner provisioner,
             ApplicationConvergenceChecker convergeChecker,
+            HttpProxy httpProxy,
             LogServerLogGrabber logServerLogGrabber) {
         return new ApplicationHandler(
                 Runnable::run,
@@ -101,7 +113,8 @@ public class ApplicationHandlerTest {
                                           HostProvisionerProvider.withProvisioner(provisioner),
                                           new MockCurator(),
                                           logServerLogGrabber,
-                                          convergeChecker));
+                                          convergeChecker,
+                                          httpProxy));
     }
 
     private ApplicationHandler createApplicationHandler(Tenants tenants) {
@@ -113,7 +126,8 @@ public class ApplicationHandlerTest {
                                           HostProvisionerProvider.withProvisioner(provisioner),
                                           new MockCurator(),
                                           new LogServerLogGrabber(),
-                                          new ApplicationConvergenceChecker(stateApiFactory)));
+                                          new ApplicationConvergenceChecker(stateApiFactory),
+                                          new HttpProxy(new SimpleHttpFetcher())));
     }
 
     @Test
@@ -205,6 +219,21 @@ public class ApplicationHandlerTest {
     }
 
     @Test
+    public void testClusterControllerStatus() throws Exception {
+        long sessionId = 1;
+        ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
+        addMockApplication(tenants.getTenant(mytenantName), application, sessionId);
+        String host = "foo.yahoo.com";
+        String url = toUrlPath(application, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1";
+
+        when(mockHttpProxy.get(any(), eq(host), eq("container-clustercontroller"), eq("clustercontroller-status/v1/clusterName1")))
+                .thenReturn(new LiteralResponse(200, "<html>...</html>"));
+
+        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "<html>...</html>");
+    }
+
+    @Test
     public void testPutIsIllegal() throws IOException {
         assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.PUT);
     }
@@ -214,7 +243,10 @@ public class ApplicationHandlerTest {
     public void testFailingProvisioner() throws Exception {
         provisioner = new SessionActiveHandlerTest.FailingMockProvisioner();
         mockHandler = createMockApplicationHandler(
-                provisioner, new ApplicationConvergenceChecker(stateApiFactory), new LogServerLogGrabber());
+                provisioner,
+                new ApplicationConvergenceChecker(stateApiFactory),
+                new HttpProxy(new SimpleHttpFetcher()),
+                new LogServerLogGrabber());
         final ApplicationId applicationId = ApplicationId.defaultId();
         addMockApplication(tenants.getTenant(mytenantName), applicationId, 1);
         assertApplicationExists(mytenantName, applicationId, Zone.defaultZone());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
index dda5ecb897e..2d76e9aa2cc 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
@@ -1,23 +1,35 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.http.v2;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import com.yahoo.config.provision.*;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.http.HttpRequest;
+import com.yahoo.path.Path;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.transaction.NestedTransaction;
-import com.yahoo.vespa.config.server.*;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.PathProvider;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
+import com.yahoo.vespa.config.server.application.MemoryTenantApplications;
 import com.yahoo.vespa.config.server.http.HttpErrorResponse;
+import com.yahoo.vespa.config.server.http.SessionActiveHandlerTestBase;
+import com.yahoo.vespa.config.server.http.SessionHandler;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.LocalSessionRepo;
 import com.yahoo.vespa.config.server.session.RemoteSession;
@@ -25,20 +37,23 @@ import com.yahoo.vespa.config.server.session.RemoteSessionRepo;
 import com.yahoo.vespa.config.server.session.Session;
 import com.yahoo.vespa.config.server.session.SessionFactory;
 import com.yahoo.vespa.config.server.session.SessionZooKeeperClient;
-
-import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.Before;
 
-import static org.hamcrest.CoreMatchers.*;
-import static org.junit.Assert.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
 
-import com.yahoo.path.Path;
-import com.yahoo.vespa.config.server.application.MemoryTenantApplications;
-import com.yahoo.vespa.config.server.http.SessionActiveHandlerTestBase;
-import com.yahoo.vespa.config.server.http.SessionHandler;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class SessionActiveHandlerTest extends SessionActiveHandlerTestBase {
 
@@ -167,7 +182,8 @@ public class SessionActiveHandlerTest extends SessionActiveHandlerTestBase {
                                           HostProvisionerProvider.withProvisioner(hostProvisioner),
                                           curator,
                                           new LogServerLogGrabber(),
-                                          new ApplicationConvergenceChecker()));
+                                          new ApplicationConvergenceChecker(),
+                                          new HttpProxy(new SimpleHttpFetcher())));
     }
 
     public static class MockProvisioner implements Provisioner {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java
index 224cd5d28d1..7f5ab96d76a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java
@@ -11,9 +11,11 @@ import com.yahoo.jdisc.http.HttpRequest;
 import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.http.ContentHandlerTestBase;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import org.apache.commons.io.FileUtils;
@@ -175,6 +177,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase {
                                                                    HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()),
                                                                    new MockCurator(),
                                                                    new LogServerLogGrabber(),
-                                                                   new ApplicationConvergenceChecker()));
+                                                                   new ApplicationConvergenceChecker(),
+                                                                   new HttpProxy(new SimpleHttpFetcher())));
     }
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java
index 908f4481a95..107297743e1 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java
@@ -10,6 +10,7 @@ import com.yahoo.container.logging.AccessLog;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.application.MemoryTenantApplications;
 import com.yahoo.vespa.config.server.application.TenantApplications;
@@ -17,6 +18,7 @@ import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest;
 import com.yahoo.vespa.config.server.http.HandlerTest;
 import com.yahoo.vespa.config.server.http.HttpErrorResponse;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.*;
 import com.yahoo.vespa.config.server.tenant.Tenants;
@@ -249,7 +251,8 @@ public class SessionCreateHandlerTest extends SessionHandlerTest {
                                                                   HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()),
                                                                   new MockCurator(),
                                                                   new LogServerLogGrabber(),
-                                                                  new ApplicationConvergenceChecker()));
+                                                                  new ApplicationConvergenceChecker(),
+                                                                  new HttpProxy(new SimpleHttpFetcher())));
     }
 
     public HttpRequest post() throws FileNotFoundException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
index 632a1c66ac0..bee1229c093 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
@@ -22,6 +22,7 @@ import com.yahoo.vespa.config.server.PathProvider;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
+import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.host.HostRegistry;
 import com.yahoo.vespa.config.server.application.TenantApplications;
@@ -350,7 +351,8 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
                                                                    HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()),
                                                                    new MockCurator(),
                                                                    new LogServerLogGrabber(),
-                                                                   new ApplicationConvergenceChecker()));
+                                                                   new ApplicationConvergenceChecker(),
+                                                                   new HttpProxy(new SimpleHttpFetcher())));
     }
 
     private TestTenantBuilder addTenant(TenantName tenantName,
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java
index 1b089930482..20f904efb9b 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java
@@ -42,7 +42,7 @@ public abstract class HttpResponse {
     }
 
     /**
-     * Marshal this response to the network layer.
+     * Marshal this response to the network layer. The caller is responsible for flushing and closing outputStream.
      */
     public abstract void render(OutputStream outputStream) throws IOException;
 
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LiteralResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/LiteralResponse.java
new file mode 100644
index 00000000000..c923c0ce0a5
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/LiteralResponse.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.jdisc;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+public class LiteralResponse extends HttpResponse {
+    private final String body;
+
+    public LiteralResponse(int code, String body) {
+        super(code);
+        this.body = body;
+    }
+
+    @Override
+    public void render(OutputStream outputStream) throws IOException {
+        outputStream.write(body.getBytes(StandardCharsets.UTF_8));
+    }
+}