Add file distribution status API

7 files changed, 374 insertions, 10 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index a4dd943aa47..1181cbba7de 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -21,6 +21,7 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
+import com.yahoo.vespa.config.server.application.FileDistributionStatus;
 import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
 import com.yahoo.vespa.config.server.application.TenantApplications;
@@ -70,6 +71,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final DeployLogger logger = new SilentDeployLogger();
     private final ConfigserverConfig configserverConfig;
     private final Environment environment;
+    private final FileDistributionStatus fileDistributionStatus;
 
     @Inject
     public ApplicationRepository(Tenants tenants,
@@ -79,7 +81,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  HttpProxy httpProxy, 
                                  ConfigserverConfig configserverConfig) {
         this(tenants, hostProvisionerProvider.getHostProvisioner(), logServerLogGrabber,
-             applicationConvergenceChecker, httpProxy, configserverConfig, Clock.systemUTC());
+             applicationConvergenceChecker, httpProxy, configserverConfig, Clock.systemUTC(), new FileDistributionStatus());
     }
 
     // For testing
@@ -88,7 +90,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  Clock clock) {
         this(tenants, Optional.of(hostProvisioner), new LogServerLogGrabber(),
              new ApplicationConvergenceChecker(), new HttpProxy(new SimpleHttpFetcher()),
-             new ConfigserverConfig(new ConfigserverConfig.Builder()), clock);
+             new ConfigserverConfig(new ConfigserverConfig.Builder()), clock, new FileDistributionStatus());
     }
 
     private ApplicationRepository(Tenants tenants,
@@ -97,7 +99,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   ApplicationConvergenceChecker applicationConvergenceChecker,
                                   HttpProxy httpProxy,
                                   ConfigserverConfig configserverConfig,
-                                  Clock clock) {
+                                  Clock clock,
+                                  FileDistributionStatus fileDistributionStatus) {
         this.tenants = tenants;
         this.hostProvisioner = hostProvisioner;
         this.logServerLogGrabber = logServerLogGrabber;
@@ -106,6 +109,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.clock = clock;
         this.configserverConfig = configserverConfig;
         this.environment = Environment.from(configserverConfig.environment());
+        this.fileDistributionStatus = fileDistributionStatus;
     }
 
     /**
@@ -259,6 +263,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter));
     }
 
+    public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId) {
+        Application application = getApplication(tenant, applicationId);
+        return fileDistributionStatus.status(application);
+    }
+
     public Tenant verifyTenantAndApplication(ApplicationId applicationId) {
         TenantName tenantName = applicationId.tenant();
         if (!tenants.checkThatTenantExists(tenantName)) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java
new file mode 100644
index 00000000000..67cce912395
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java
@@ -0,0 +1,143 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.application;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import com.yahoo.slime.Cursor;
+import com.yahoo.vespa.config.server.http.JSONResponse;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * File distribution status for each host in the application
+ *
+ * @author hmusum
+ */
+public class FileDistributionStatus extends AbstractComponent {
+
+    enum Status {UNKNOWN, FINISHED, IN_PROGRESS}
+
+    private final Supervisor supervisor = new Supervisor(new Transport());
+
+    public StatusAllHosts status(Application application) {
+        List<HostStatus> hostStatuses = new ArrayList<>();
+        application.getModel().getHosts()
+                .forEach(host -> host.getServices()
+                        .stream()
+                        .filter(service -> "configproxy".equals(service.getServiceType()))
+                        .forEach(service -> hostStatuses.add(getHostStatus(service.getHostName(), getRpcPort(service)))));
+        return createStatusForAllHosts(hostStatuses);
+    }
+
+    HostStatus getHostStatus(String hostname, int port) {
+        Target target = supervisor.connect(new Spec(hostname, port));
+        Request request = new Request("filedistribution.getActiveFileReferencesStatus");
+        target.invokeSync(request, 1.0);
+        HostStatus hostStatus = createHostStatusFromResponse(hostname, request);
+        target.close();
+        return hostStatus;
+    }
+
+    private HostStatus createHostStatusFromResponse(String hostname, Request request) {
+        if (request.isError()) {
+            return new HostStatus(hostname,
+                                  Status.UNKNOWN,
+                                  Collections.emptyMap(),
+                                  "error: " + request.errorMessage() + "(" + request.errorCode() + ")");
+        } else {
+            Map<String, Double> fileReferenceStatuses = new HashMap<>();
+            String[] fileReferences = request.returnValues().get(0).asStringArray();
+            double[] downloadStatus = request.returnValues().get(1).asDoubleArray();
+
+            boolean allDownloaded = true;
+            for (int i = 0; i < fileReferences.length; i++) {
+                fileReferenceStatuses.put(fileReferences[i], downloadStatus[i]);
+                if (downloadStatus[i] < 1.0) {
+                    allDownloaded = false;
+                }
+            }
+
+            return new HostStatus(hostname, allDownloaded ? Status.FINISHED : Status.IN_PROGRESS, fileReferenceStatuses, "");
+        }
+    }
+
+    private StatusAllHosts createStatusForAllHosts(List<HostStatus> hostStatuses) {
+        boolean allFinished = true;
+        for (HostStatus hostStatus : hostStatuses) {
+            if (hostStatus.status != Status.FINISHED) {
+                allFinished = false;
+                break;
+            }
+        }
+        return new StatusAllHosts(allFinished ? Status.FINISHED : Status.IN_PROGRESS, hostStatuses);
+    }
+
+    private static Integer getRpcPort(ServiceInfo service) {
+        return service.getPorts().stream()
+                .filter(port -> port.getTags().contains("rpc"))
+                .map(PortInfo::getPort)
+                .findFirst()
+                .orElseThrow(() -> new RuntimeException("Could not find rpc port for config proxy for " + service.getHostName()));
+    }
+
+    static class StatusAllHosts extends JSONResponse {
+
+        private StatusAllHosts(Status status, List<HostStatus> hostStatuses) {
+            super(200);
+            Cursor hostsArray = object.setArray("hosts");
+            for (HostStatus hostStatus : hostStatuses) {
+                Cursor host = hostsArray.addObject();
+                host.setString("hostname", hostStatus.hostname);
+                host.setString("status", hostStatus.status.name());
+                hostStatus.errorMessage.ifPresent(message -> host.setString("message", message));
+                Cursor fileReferences = host.setArray("fileReferences");
+                hostStatus.fileReferenceStatuses.forEach((key, value) -> fileReferences.addObject().setDouble(key, value));
+            }
+
+            object.setString("status", status.name());
+        }
+    }
+
+    static class HostStatus {
+
+        private final String hostname;
+        private final Status status;
+        private final Map<String, Double> fileReferenceStatuses;
+        private final Optional<String> errorMessage;
+
+        HostStatus(String hostname, Status status, Map<String, Double> fileReferenceStatuses) {
+            this.hostname = hostname;
+            this.status = status;
+            this.fileReferenceStatuses = fileReferenceStatuses;
+            this.errorMessage = Optional.empty();
+        }
+
+        HostStatus(String hostname, Status status, Map<String, Double> fileReferenceStatuses, String errorMessage) {
+            this.hostname = hostname;
+            this.status = status;
+            this.fileReferenceStatuses = fileReferenceStatuses;
+            this.errorMessage = Optional.of(errorMessage);
+        }
+
+        public String hostname() {
+            return hostname;
+        }
+
+        @Override
+        public String toString() {
+            return hostname + ": " + status + ", " + fileReferenceStatuses + " " + errorMessage.orElse("");
+        }
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 819f1a35cf3..876966c6550 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.logging.AccessLog;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.application.BindingMatch;
 import com.yahoo.vespa.config.server.ApplicationRepository;
@@ -27,7 +26,6 @@ import com.yahoo.vespa.config.server.tenant.Tenant;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
-import java.util.concurrent.Executor;
 
 /**
  * Operations on applications (delete, wait for config convergence, restart, application content etc.)
@@ -94,6 +92,10 @@ public class ApplicationHandler extends HttpHandler {
             return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri());
         }
 
+        if (isFiledistributionStatusRequest(request)) {
+            return applicationRepository.filedistributionStatus(tenant, applicationId);
+        }
+
         return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId));
     }
 
@@ -154,6 +156,7 @@ public class ApplicationHandler extends HttpHandler {
                 // WARNING: UPDATE src/main/resources/configserver-app/services.xml IF YOU MAKE ANY CHANGES TO THESE BINDINGS!
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log",
+                "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*",
@@ -182,6 +185,11 @@ public class ApplicationHandler extends HttpHandler {
                 request.getUri().getPath().contains("/content/");
     }
 
+    private static boolean isFiledistributionStatusRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+                request.getUri().getPath().contains("/filedistributionstatus");
+    }
+
     private static String getHostNameFromRequest(HttpRequest req) {
         BindingMatch<?> bm = getBindingMatch(req);
         return bm.group(7);
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 1a0b61fbe96..e5a724a07e8 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -117,6 +117,8 @@
       <!-- WARNING: THIS LIST *MUST* MATCH THE ONE IN ApplicationHandler::getBindingMatch -->
       <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding>
       <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding>
+      <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding>
+      <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding>
       <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding>
       <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding>
       <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log</binding>
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java
new file mode 100644
index 00000000000..35a86ad4757
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java
@@ -0,0 +1,163 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.application;
+
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Version;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.vespa.config.server.ServerCache;
+import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+import static com.yahoo.vespa.config.server.application.FileDistributionStatus.HostStatus;
+import static com.yahoo.vespa.config.server.application.FileDistributionStatus.Status;
+
+/**
+ * @author hmusum
+ */
+public class FileDistributionStatusTest {
+
+    private TenantName tenant = TenantName.from("mytenant");
+    private ApplicationId appId = ApplicationId.from(tenant, ApplicationName.from("myapp"), InstanceName.from("myinstance"));
+    private Application application;
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+
+    @Test
+    public void require_status_finished() throws IOException {
+        Map<String, Double> fileReferenceStatuses = new HashMap<>();
+        fileReferenceStatuses.put("1234", 1.0);
+        FileDistributionStatus status = new MockStatus(statusFinished("localhost", Status.FINISHED, fileReferenceStatuses));
+        application = createApplication("localhost");
+
+        HttpResponse response = status.status(application);
+        assertResponse(200,
+                       "{" +
+                               "\"hosts\":[" +
+                               "{\"hostname\":\"localhost\"," +
+                               "\"status\":\"FINISHED\"," +
+                               "\"fileReferences\":[" +
+                               "{\"1234\":1.0}]}" +
+                               "]," +
+                               "\"status\":\"FINISHED\"}",
+                       response);
+    }
+
+    @Test
+    public void require_status_in_progress_one_host() throws IOException {
+        Map<String, Double> fileReferenceStatuses = new HashMap<>();
+        fileReferenceStatuses.put("1234", 0.2);
+        FileDistributionStatus status = new MockStatus(statusWithError("localhost2", Status.IN_PROGRESS, fileReferenceStatuses, ""));
+        application = createApplication("localhost2");
+
+        HttpResponse response = status.status(application);
+        assertResponse(200,
+                       "{" +
+                               "\"hosts\":[" +
+                               "{\"hostname\":\"localhost2\"," +
+                               "\"status\":\"IN_PROGRESS\"," +
+                               "\"message\":\"\"," +
+                               "\"fileReferences\":[" +
+                               "{\"1234\":0.2}]}" +
+                               "]," +
+                               "\"status\":\"IN_PROGRESS\"}",
+                       response);
+    }
+
+    @Test
+    public void require_different_statuses__many_hosts() throws IOException {
+        application = createApplication("localhost", "localhost2");
+
+        Map<String, Double> fileReferenceStatuses = new HashMap<>();
+        fileReferenceStatuses.put("1234", 0.2);
+        fileReferenceStatuses.put("abcd", 1.0);
+        HostStatus localhost = statusWithError("localhost", Status.IN_PROGRESS, fileReferenceStatuses, "connection timed out");
+
+        Map<String, Double> fileReferenceStatuses2 = new HashMap<>();
+        fileReferenceStatuses2.put("1234", 1.0);
+        HostStatus localhost2 = statusFinished("localhost2", Status.FINISHED, fileReferenceStatuses2);
+
+        FileDistributionStatus status = new MockStatus(new HashSet<>(Arrays.asList(localhost, localhost2)));
+        application = createApplication("localhost", "localhost2");
+        HttpResponse response = status.status(application);
+        assertResponse(200,
+                       "{" +
+                               "\"hosts\":[" +
+                               "{\"hostname\":\"localhost\"," +
+                               "\"status\":\"IN_PROGRESS\"," +
+                               "\"message\":\"connection timed out\"," +
+                               "\"fileReferences\":[" +
+                               "{\"1234\":0.2},{\"abcd\":1.0}]}," +
+                               "{\"hostname\":\"localhost2\"," +
+                               "\"status\":\"FINISHED\"," +
+                               "\"fileReferences\":[" +
+                               "{\"1234\":1.0}]}" +
+                               "]," +
+                               "\"status\":\"IN_PROGRESS\"}",
+                       response);
+    }
+
+    private void assertResponse(int statusCode, String expectedResponse, HttpResponse response) throws IOException {
+        assertEquals(statusCode, response.getStatus());
+        assertEquals(expectedResponse, SessionHandlerTest.getRenderedString(response));
+    }
+
+    private HostStatus statusFinished(String hostname, Status status, Map<String, Double> fileReferenceStatuses) {
+        return new HostStatus(hostname, status, fileReferenceStatuses);
+    }
+
+    private HostStatus statusWithError(String hostname,
+                                       Status status,
+                                       Map<String, Double> fileReferenceStatuses,
+                                       String errorMessage) {
+        return new HostStatus(hostname, status, fileReferenceStatuses, errorMessage);
+    }
+
+    private Application createApplication(String... hostname) {
+        return createApplication(Arrays.asList(hostname));
+    }
+
+    private Application createApplication(List<String> hostnames) {
+        Model mockModel = MockModel.createConfigProxies(hostnames, 1337);
+        return new Application(mockModel, new ServerCache(), 3, Version.fromIntValues(0, 0, 0), MetricUpdater.createTestUpdater(), appId);
+    }
+
+    private static class MockStatus extends FileDistributionStatus {
+
+        private final Map<String, HostStatus> statuses = new HashMap<>();
+
+        // host status to be returned in getHostStatus()
+        MockStatus(HostStatus status) {
+            this(Collections.singleton(status));
+        }
+
+        // host status per host to be returned in getHostStatus()
+        MockStatus(Set<HostStatus> status) {
+            status.forEach(s -> statuses.put(s.hostname(), s));
+        }
+
+        @Override
+        HostStatus getHostStatus(String hostname, int port) {
+            return statuses.get(hostname);
+        }
+    }
+
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java
index cf5463b7f4c..91b21ad83d6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java
@@ -16,9 +16,11 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 // Model with two services, one that does not have a state port
@@ -49,6 +51,20 @@ class MockModel implements Model {
         return new MockModel(Collections.singleton(hostInfo));
     }
 
+    static MockModel createConfigProxy(String hostname, int rpcPort) {
+        return createConfigProxies(Collections.singletonList(hostname), rpcPort);
+    }
+
+    static MockModel createConfigProxies(List<String> hostnames, int rpcPort) {
+        Set<HostInfo> hostInfos = new HashSet<>();
+        hostnames.forEach(hostname -> {
+            ServiceInfo configProxy = createServiceInfo(hostname, "configproxy", "configproxy",
+                                            ClusterSpec.Type.admin, rpcPort, "rpc");
+            hostInfos.add(new HostInfo(hostname, Collections.singletonList(configProxy)));
+        });
+        return new MockModel(hostInfos);
+    }
+
     static private ServiceInfo createServiceInfo(
             String hostname,
             String name,
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 8ac64e5b28a..49ef978cc5f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -13,7 +13,6 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.logging.AccessLog;
 import com.yahoo.jdisc.Response;
 import com.yahoo.path.Path;
 import com.yahoo.vespa.config.server.ApplicationRepository;
@@ -30,7 +29,6 @@ import com.yahoo.vespa.config.server.http.StaticResponse;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
-import com.yahoo.vespa.config.server.monitoring.Metrics;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.LocalSession;
 import com.yahoo.vespa.config.server.session.MockSessionZKClient;
@@ -67,7 +65,6 @@ import static org.mockito.Mockito.when;
 
 /**
  * @author hmusum
- * @since 5.4
  */
 public class ApplicationHandlerTest {
 
@@ -83,7 +80,7 @@ public class ApplicationHandlerTest {
     private final HttpProxy mockHttpProxy = mock(HttpProxy.class);
 
     @Before
-    public void setup() throws Exception {
+    public void setup() {
         TestTenantBuilder testBuilder = new TestTenantBuilder();
         testBuilder.createTenant(mytenantName).withReloadHandler(new MockReloadHandler());
         testBuilder.createTenant(foobar).withReloadHandler(new MockReloadHandler());
@@ -275,6 +272,27 @@ public class ApplicationHandlerTest {
         tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app), clock));
     }
 
+    @Test
+    public void testFileDistributionStatus() throws Exception {
+        long sessionId = 1;
+        ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
+        addMockApplication(tenants.getTenant(mytenantName), application, sessionId, Clock.systemUTC());
+        Zone zone = Zone.defaultZone();
+
+        HttpResponse response = fileDistributionStatus(application, zone);
+        assertEquals(200, response.getStatus());
+        SessionHandlerTest.getRenderedString(response);
+        assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"IN_PROGRESS\"}",
+                     SessionHandlerTest.getRenderedString(response));
+
+        // 404 for unknown application
+        ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build();
+        HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone);
+        assertEquals(404, responseForUnknown.getStatus());
+        assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: mytenant.unknown\"}",
+                     SessionHandlerTest.getRenderedString(responseForUnknown));
+    }
+
     private static Tenants addApplication(ApplicationId applicationId, long sessionId) throws Exception {
         // This method is a good illustration of the spaghetti wiring resulting from no design
         // TODO: When this setup looks sane we have refactored sufficiently that there is a design
@@ -394,6 +412,11 @@ public class ApplicationHandlerTest {
         return SessionHandlerTest.getRenderedString(response);
     }
 
+    private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) {
+        String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus";
+        return mockHandler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+    }
+
     private static class MockStateApiFactory implements ApplicationConvergenceChecker.StateApiFactory {
         public boolean createdApi = false;
         @Override