Merge pull request #8449 from vespa-engine/jvenstad/cache-orchestrator-host-statuses-for-reads

Jvenstad/cache orchestrator host statuses for reads

30 files changed, 404 insertions, 523 deletions

diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
index 73abd70a5ae..aa157366a60 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
@@ -11,7 +11,9 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * (Only the suspended applications part of this is in use)
@@ -34,6 +36,11 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        return hostName -> Optional.of(getNodeStatus(hostName));
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus state) {}
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 39a2787ca9b..42dbcdf7a86 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -34,7 +35,7 @@ import java.util.stream.Collectors;
 public class MetricsReporter extends Maintainer {
 
     private final Metric metric;
-    private final Orchestrator orchestrator;
+    private final Function<HostName, Optional<HostStatus>> orchestrator;
     private final ServiceMonitor serviceMonitor;
     private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>();
     private final Supplier<Integer> pendingRedeploymentsSupplier;
@@ -48,7 +49,7 @@ public class MetricsReporter extends Maintainer {
                            JobControl jobControl) {
         super(nodeRepository, interval, jobControl);
         this.metric = metric;
-        this.orchestrator = orchestrator;
+        this.orchestrator = orchestrator.getNodeStatuses();
         this.serviceMonitor = serviceMonitor;
         this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
     }
@@ -129,13 +130,9 @@ public class MetricsReporter extends Maintainer {
                 node.status().hardwareDivergence().isPresent() ? 1 : 0,
                 context);
 
-        try {
-            HostStatus status = orchestrator.getNodeStatus(new HostName(node.hostname()));
-            boolean allowedToBeDown = status == HostStatus.ALLOWED_TO_BE_DOWN;
-            metric.set("allowedToBeDown", allowedToBeDown ? 1 : 0, context);
-        } catch (HostNameNotFoundException e) {
-            // Ignore
-        }
+        orchestrator.apply(new HostName(node.hostname()))
+                    .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0)
+                    .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context));
 
         long numberOfServices;
         HostName hostName = new HostName(node.hostname());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 2a2ca6bfd87..5b942497be8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -22,7 +22,9 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
 * @author bratseth
@@ -40,7 +42,7 @@ class NodesResponse extends HttpResponse {
 
     private final NodeFilter filter;
     private final boolean recursive;
-    private final Orchestrator orchestrator;
+    private final Function<HostName, Optional<HostStatus>> orchestrator;
     private final NodeRepository nodeRepository;
     private final Slime slime;
     private final NodeSerializer serializer = new NodeSerializer();
@@ -52,7 +54,7 @@ class NodesResponse extends HttpResponse {
         this.nodeParentUrl = toNodeParentUrl(request);
         filter = NodesApiHandler.toNodeFilter(request);
         this.recursive = request.getBooleanProperty("recursive");
-        this.orchestrator = orchestrator;
+        this.orchestrator = orchestrator.getNodeStatuses();
         this.nodeRepository = nodeRepository;
 
         slime = new Slime();
@@ -158,11 +160,9 @@ class NodesResponse extends HttpResponse {
             object.setLong("currentRestartGeneration", node.allocation().get().restartGeneration().current());
             object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(node.allocation().get().membership().cluster().vespaVersion()).asString());
             object.setString("wantedVespaVersion", node.allocation().get().membership().cluster().vespaVersion().toFullString());
-            try {
-                object.setBool("allowedToBeDown", 
-                               orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN);
-            }
-            catch (HostNameNotFoundException e) {/* ok */ }
+            orchestrator.apply(new HostName(node.hostname()))
+                        .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN)
+                        .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown));
         }
         object.setLong("rebootGeneration", node.status().reboot().wanted());
         object.setLong("currentRebootGeneration", node.status().reboot().current());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 70750dd6672..96ec0349fb2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -11,7 +11,9 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * @author bratseth
@@ -32,6 +34,11 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        return hostName -> Optional.of(getNodeStatus(hostName));
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus state) {}
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java
index 8215dc8ecd0..b9c5df9d999 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java
@@ -62,7 +62,11 @@ public class ServiceMonitorStub implements ServiceMonitor {
     }
 
     @Override
-    public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
+    public ServiceModel getServiceModelSnapshot() {
+        return new ServiceModel(getAllApplicationInstances());
+    }
+
+    private Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
         // Convert apps information to the response payload to return
         Map<ApplicationInstanceReference, ApplicationInstance> status = new HashMap<>();
         for (Map.Entry<ApplicationId, MockDeployer.ApplicationContext> app : apps.entrySet()) {
@@ -84,8 +88,4 @@ public class ServiceMonitorStub implements ServiceMonitor {
         return status;
     }
 
-    @Override
-    public ServiceModel getServiceModelSnapshot() {
-        return new ServiceModel(getAllApplicationInstances());
-    }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
index 1e502439aeb..57b942d85e4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
@@ -38,7 +38,6 @@ import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -90,7 +89,7 @@ public class MetricsReporterTest {
 
         Orchestrator orchestrator = mock(Orchestrator.class);
         ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
         ServiceModel serviceModel = mock(ServiceModel.class);
         when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
         when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap());
@@ -137,7 +136,7 @@ public class MetricsReporterTest {
 
         Orchestrator orchestrator = mock(Orchestrator.class);
         ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
         ServiceModel serviceModel = mock(ServiceModel.class);
         when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
         when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap());
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index a639d07e504..59b320cf501 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -10,7 +10,9 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * The orchestrator is used to coordinate the need of vespa services to restart or
@@ -46,6 +48,14 @@ public interface Orchestrator {
      */
     HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
 
+    /**
+     * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this.
+     *
+     * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads.
+     * @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}.
+     */
+    Function<HostName, Optional<HostStatus>> getNodeStatuses();
+
     void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
 
     /**
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 33cfa310a68..bd8e31cd283 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -35,8 +35,10 @@ import java.time.Clock;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -104,6 +106,14 @@ public class OrchestratorImpl implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        Function<ApplicationInstanceReference, Set<HostName>> suspendedHosts = statusService.getSuspendedHostsByApplication();
+        return hostName -> instanceLookupService.findInstanceByHost(hostName)
+                                                .map(application -> suspendedHosts.apply(application.reference()).contains(hostName)
+                                                                    ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS);
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException {
         ApplicationInstanceReference reference = getApplicationInstance(hostName).reference();
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
@@ -133,13 +143,13 @@ public class OrchestratorImpl implements Orchestrator {
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
         try (MutableStatusRegistry statusRegistry = statusService
                 .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) {
-            final HostStatus currentHostState = statusRegistry.getHostStatus(hostName);
+            HostStatus currentHostState = statusRegistry.getHostStatus(hostName);
 
             if (HostStatus.NO_REMARKS == currentHostState) {
                 return;
             }
 
-            ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(appInstance.reference()).getApplicationInstanceStatus();
+            ApplicationInstanceStatus appStatus = statusRegistry.getStatus();
             if (appStatus == ApplicationInstanceStatus.NO_REMARKS) {
                 policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry);
             }
@@ -181,7 +191,7 @@ public class OrchestratorImpl implements Orchestrator {
 
         try (MutableStatusRegistry hostStatusRegistry =
                      statusService.lockApplicationInstance_forCurrentThreadOnly(context, applicationReference)) {
-            ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(applicationReference).getApplicationInstanceStatus();
+            ApplicationInstanceStatus appStatus = hostStatusRegistry.getStatus();
             if (appStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
                 return;
             }
@@ -195,8 +205,8 @@ public class OrchestratorImpl implements Orchestrator {
 
     @Override
     public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) throws ApplicationIdNotFoundException {
-        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId,instanceLookupService);
-        return statusService.forApplicationInstance(appRef).getApplicationInstanceStatus();
+        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService);
+        return statusService.getApplicationInstanceStatus(appRef);
     }
 
     @Override
@@ -305,7 +315,7 @@ public class OrchestratorImpl implements Orchestrator {
     }
 
     private HostStatus getNodeStatus(ApplicationInstanceReference applicationRef, HostName hostName) {
-        return statusService.forApplicationInstance(applicationRef).getHostStatus(hostName);
+        return statusService.getHostStatus(applicationRef, hostName);
     }
 
     private void setApplicationStatus(ApplicationId appId, ApplicationInstanceStatus status) 
@@ -316,7 +326,7 @@ public class OrchestratorImpl implements Orchestrator {
                      statusService.lockApplicationInstance_forCurrentThreadOnly(context, appRef)) {
 
             // Short-circuit if already in wanted state
-            if (status == statusRegistry.getApplicationInstanceStatus()) return;
+            if (status == statusRegistry.getStatus()) return;
 
             // Set content clusters for this application in maintenance on suspend
             if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
index 79506d042e2..91da046840d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
@@ -12,20 +12,15 @@ import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceCluster;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.TenantId;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toSet;
 
 /**
@@ -57,14 +52,6 @@ public class OrchestratorUtil {
                 .collect(toSet());
     }
 
-    public static Map<HostName, HostStatus> getHostStatusMap(Collection<HostName> hosts,
-                                                             ReadOnlyStatusRegistry hostStatusService) {
-        return hosts.stream()
-                .collect(Collectors.toMap(
-                        hostName -> hostName,
-                        hostName -> hostStatusService.getHostStatus(hostName)));
-    }
-
     private static boolean hasServiceInstanceOnHost(ServiceCluster serviceCluster, HostName hostName) {
         return serviceInstancesOnHost(serviceCluster, hostName).count() > 0;
     }
@@ -75,11 +62,6 @@ public class OrchestratorUtil {
                 .filter(instance -> instance.hostName().equals(hostName));
     }
 
-    public static <K, V1, V2> Map<K, V2> mapValues(Map<K, V1> map, Function<V1, V2> valueConverter) {
-        return map.entrySet().stream()
-                .collect(toMap(Map.Entry::getKey, entry -> valueConverter.apply(entry.getValue())));
-    }
-
     private static final Pattern APPLICATION_INSTANCE_REFERENCE_REST_FORMAT_PATTERN = Pattern.compile("^([^:]+):(.+)$");
 
     /** Returns an ApplicationInstanceReference constructed from the serialized format used in the REST API. */
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java
index d1d5f3e8c95..1a859cfacc5 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java
@@ -29,42 +29,17 @@ public class ServiceMonitorInstanceLookupService implements InstanceLookupServic
 
     @Override
     public Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference) {
-        Map<ApplicationInstanceReference, ApplicationInstance> instanceMap
-                = serviceMonitor.getAllApplicationInstances();
-        return Optional.ofNullable(instanceMap.get(applicationInstanceReference));
+        return serviceMonitor.getServiceModelSnapshot().getApplicationInstance(applicationInstanceReference);
     }
 
     @Override
     public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) {
-        Map<ApplicationInstanceReference, ApplicationInstance> instanceMap 
-                = serviceMonitor.getAllApplicationInstances();
-        List<ApplicationInstance> applicationInstancesUsingHost = instanceMap.entrySet().stream()
-                .filter(entry -> applicationInstanceUsesHost(entry.getValue(), hostName))
-                .map(Map.Entry::getValue)
-                .collect(Collectors.toList());
-        if (applicationInstancesUsingHost.isEmpty()) {
-            return Optional.empty();
-        }
-        if (applicationInstancesUsingHost.size() > 1) {
-            throw new IllegalStateException(
-                    "Major assumption broken: Multiple application instances contain host " + hostName.s()
-                            + ": " + applicationInstancesUsingHost);
-        }
-        return Optional.of(applicationInstancesUsingHost.get(0));
+        return Optional.ofNullable(serviceMonitor.getServiceModelSnapshot().getApplicationsByHostName().get(hostName));
     }
 
     @Override
     public Set<ApplicationInstanceReference> knownInstances() {
-        return serviceMonitor.getAllApplicationInstances().keySet();
-    }
-
-    private static boolean applicationInstanceUsesHost(ApplicationInstance applicationInstance,
-                                                       HostName hostName) {
-        return applicationInstance.serviceClusters().stream()
-                .anyMatch(serviceCluster ->
-                        serviceCluster.serviceInstances().stream()
-                                .anyMatch(serviceInstance ->
-                                        serviceInstance.hostName().equals(hostName)));
+        return serviceMonitor.getServiceModelSnapshot().getAllApplicationInstances().keySet();
     }
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
index 5800b48da75..db2fb9b4a18 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 import java.util.Collection;
 import java.util.Comparator;
@@ -40,7 +41,9 @@ public class ApplicationApiImpl implements ApplicationApi {
         this.nodeGroup = nodeGroup;
         this.hostStatusService = hostStatusService;
         Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance);
-        this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(), hostStatusService::getHostStatus));
+        this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(),
+                                                                     hostName -> hostStatusService.getSuspendedHosts().contains(hostName)
+                                                                                 ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS));
         this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory);
     }
 
@@ -91,7 +94,7 @@ public class ApplicationApiImpl implements ApplicationApi {
 
     @Override
     public ApplicationInstanceStatus getApplicationStatus() {
-        return hostStatusService.getApplicationInstanceStatus();
+        return hostStatusService.getStatus();
     }
 
     @Override
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
index b6e7014cac0..96930c1cda2 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.orchestrator.model.StorageNode;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 import java.util.logging.Logger;
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
index e2487301326..aa7636227c8 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 /**
  * @author oyving
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
index cd20a01f6af..65ef1c9eaf0 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
@@ -34,7 +34,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostStatusMap;
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseAppInstanceReference;
 
@@ -82,11 +81,13 @@ public class InstanceResource {
                 = instanceLookupService.findInstanceById(instanceId)
                 .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()));
 
-        Set<HostName> hostsUsedByApplicationInstance = getHostsUsedByApplicationInstance(applicationInstance);
-        Map<HostName, HostStatus> hostStatusMap = getHostStatusMap(hostsUsedByApplicationInstance,
-                                                                   statusService.forApplicationInstance(instanceId));
-        Map<HostName, String> hostStatusStringMap = OrchestratorUtil.mapValues(hostStatusMap, HostStatus::name);
-        return InstanceStatusResponse.create(applicationInstance, hostStatusStringMap);
+        Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference());
+        Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance)
+                .stream()
+                .collect(Collectors.toMap(hostName -> hostName,
+                                          hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name()
+                                                                                        : HostStatus.NO_REMARKS.name()));
+        return InstanceStatusResponse.create(applicationInstance, hostStatusMap);
     }
 
     @GET
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java
deleted file mode 100644
index 70128ae12eb..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java
+++ /dev/null
@@ -1,121 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.status;
-
-import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.orchestrator.OrchestratorContext;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Implementation of the StatusService interface for testing.
- *
- * @author oyving
- */
-public class InMemoryStatusService implements StatusService {
-
-    private final Map<HostName, HostStatus> hostServiceStatus = new HashMap<>();
-    private final Set<ApplicationInstanceReference> applicationStatus = new HashSet<>();
-    private final LockService<ApplicationInstanceReference> instanceLockService = new LockService<>();
-
-    private void setHostStatus(HostName hostName, HostStatus status) {
-        hostServiceStatus.put(hostName, status);
-    }
-
-    @Override
-    public ReadOnlyStatusRegistry forApplicationInstance(ApplicationInstanceReference applicationInstanceReference) {
-        return new ReadOnlyStatusRegistry() {
-            @Override
-            public HostStatus getHostStatus(HostName hostName) {
-                return hostServiceStatus.getOrDefault(hostName, HostStatus.NO_REMARKS);
-            }
-
-            @Override
-            public ApplicationInstanceStatus getApplicationInstanceStatus() {
-                return applicationStatus.contains(applicationInstanceReference) ?
-                        ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN : ApplicationInstanceStatus.NO_REMARKS;
-            }
-        };
-    }
-
-
-    @Override
-    public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly(
-            OrchestratorContext context,
-            ApplicationInstanceReference applicationInstanceReference) {
-        Lock lock = instanceLockService.get(applicationInstanceReference);
-        return new InMemoryMutableStatusRegistry(lock, applicationInstanceReference);
-    }
-
-    @Override
-    public Set<ApplicationInstanceReference> getAllSuspendedApplications() {
-        return applicationStatus;
-    }
-
-    private class InMemoryMutableStatusRegistry implements MutableStatusRegistry {
-
-        private final Lock lockHandle;
-        private final ApplicationInstanceReference ref;
-
-        public InMemoryMutableStatusRegistry(Lock lockHandle, ApplicationInstanceReference ref) {
-            this.lockHandle = lockHandle;
-            this.ref = ref;
-        }
-
-        @Override
-        public void setHostState(HostName hostName, HostStatus status) {
-            setHostStatus(hostName, status);
-        }
-
-        @Override
-        public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) {
-            if (applicationInstanceStatus == ApplicationInstanceStatus.NO_REMARKS) {
-                applicationStatus.remove(ref);
-            } else {
-                applicationStatus.add(ref);
-            }
-        }
-
-        @Override
-        public HostStatus getHostStatus(HostName hostName) {
-            return hostServiceStatus.getOrDefault(hostName, HostStatus.NO_REMARKS);
-        }
-
-        @Override
-        public ApplicationInstanceStatus getApplicationInstanceStatus() {
-            return applicationStatus.contains(ref) ? ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN :
-            ApplicationInstanceStatus.NO_REMARKS;
-        }
-
-        @Override
-        public void close() {
-            //lockHandle.unlock();   TODO this casues illeal state monitor exception - how to use it properly
-        }
-    }
-
-    private static class LockService<T> {
-
-        private final Map<T, Lock> locks;
-
-        public LockService() {
-            this.locks = new HashMap<>();
-        }
-
-        public Lock get(T lockId) {
-            synchronized (this) {
-                Lock lock = locks.computeIfAbsent(
-                    lockId,
-                    id -> new ReentrantLock()
-                );
-
-                return lock;
-            }
-        }
-    }
-
-}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
index b699a9f8962..e36f0f70bbd 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.orchestrator.status;
 
 import com.yahoo.vespa.applicationmodel.HostName;
 
+import java.util.Set;
+
 /**
  * Registry of the suspension and host statuses for an application instance.
  *
@@ -10,7 +12,22 @@ import com.yahoo.vespa.applicationmodel.HostName;
  * @author Tony Vaagenes
  * @author bakksjo
  */
-public interface MutableStatusRegistry extends ReadOnlyStatusRegistry, AutoCloseable {
+public interface MutableStatusRegistry extends AutoCloseable {
+
+    /**
+     * Returns the status of this application.
+     */
+    ApplicationInstanceStatus getStatus();
+
+    /**
+     * Returns the status of the given host.
+     */
+    HostStatus getHostStatus(HostName hostName);
+
+    /**
+     * Returns the set of all suspended hosts for this application.
+     */
+    Set<HostName> getSuspendedHosts();
 
     /**
      * Sets the state for the given host.
@@ -27,7 +44,6 @@ public interface MutableStatusRegistry extends ReadOnlyStatusRegistry, AutoClose
      * so we override it here to strip the exception from the signature.
      */
     @Override
-    @NoThrow
     void close();
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/NoThrow.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/NoThrow.java
deleted file mode 100644
index fd6757f83cf..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/NoThrow.java
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.status;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Used to annotate methods that do not throw Exceptions.
- * They are still allowed to throw Errors, such as AssertionError
- *
- * TODO: move to vespajlib or find a suitable replacement
- * @author Tony Vaagenes
- */
-@Target(ElementType.METHOD)
-@Retention(RetentionPolicy.SOURCE)
-@interface NoThrow {}
-
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ReadOnlyStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ReadOnlyStatusRegistry.java
deleted file mode 100644
index 09300ef18a8..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ReadOnlyStatusRegistry.java
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.status;
-
-import com.yahoo.vespa.applicationmodel.HostName;
-
-/**
- * Read-only view of statuses for the application instance and its hosts.
- *
- * @author oyving
- * @author Tony Vaagenes
- * @author bakksjo
- */
-public interface ReadOnlyStatusRegistry {
-
-    /**
-     * Gets the current state for the given host.
-     */
-    HostStatus getHostStatus(HostName hostName);
-
-    /**
-     * Gets the current status for the application instance.
-     */
-    ApplicationInstanceStatus getApplicationInstanceStatus();
-
-}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
index 99f6c113193..9f91e08d344 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
@@ -2,32 +2,22 @@
 package com.yahoo.vespa.orchestrator.status;
 
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 
 import java.util.Set;
+import java.util.function.Function;
 
 /**
- * Service that can produce registries for the suspension of an application
- * and its hosts.
+ * Service that can produce registries for the suspension of an application and its hosts.
  *
- * The registry classes are pr application instance.
- * TODO Remove readonly registry class (replace with actual methods) - only adds complexity.
+ * The registry class is per locked application instance.
  *
- * @author oyving
+ * @author Øyvind Grønnesby
  * @author Tony Vaagenes
  * @author smorgrav
  */
 public interface StatusService {
-    /**
-     * Returns a readable host status registry for the given application instance. No locking is involved,
-     * so this call will never block. However, since it is possible that mutations are going on simultaneously
-     * with accessing this registry, the view obtained through the returned registry must be considered to be
-     * possibly inconsistent snapshot values. It is not recommended that this method is used for anything other
-     * than monitoring, logging, debugging, etc. It should never be used for multi-step operations (e.g.
-     * read-then-write) where consistency is required. For those cases, use
-     * {@link #lockApplicationInstance_forCurrentThreadOnly(OrchestratorContext, ApplicationInstanceReference)}.
-     */
-    ReadOnlyStatusRegistry forApplicationInstance(ApplicationInstanceReference applicationInstanceReference);
 
     /**
      * Returns a mutable host status registry for a locked application instance. All operations performed on
@@ -64,4 +54,24 @@ public interface StatusService {
      * @return A Map between the application instance and its status.
      */
     Set<ApplicationInstanceReference> getAllSuspendedApplications();
+
+    /**
+     * Returns a fresh, but not necessarily consistent mapping from applications to their set of suspended hosts.
+     *
+     * If the lock for an application is held when this mapping is acquired, new sets returned for that application
+     * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.)
+     */
+    Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication();
+
+    /**
+     * Returns the status of the given application. This is consistent if its lock is held.
+     */
+    ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application);
+
+
+    /**
+     * Returns the status of the given host, for the given application. This is consistent if the application's lock is held.
+     */
+    HostStatus getHostStatus(ApplicationInstanceReference application, HostName host);
+
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
index c56ff661bba..f6e01a49ce8 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
@@ -7,20 +7,23 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.curator.recipes.CuratorCounter;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorUtil;
-import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.apache.zookeeper.data.Stat;
 
 import javax.inject.Inject;
 import java.time.Duration;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * Stores instance suspension status and which hosts are allowed to go down in zookeeper.
@@ -34,27 +37,22 @@ public class ZookeeperStatusService implements StatusService {
 
     final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service";
     final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service";
+    final static String HOST_STATUS_CACHE_COUNTER_PATH = "/vespa/host-status-service-cache-counter";
 
     private final Curator curator;
+    private final CuratorCounter counter;
+
+    /** A cache of hosts allowed to be down. Access only through {@link #getValidCache()}! */
+    private final Map<ApplicationInstanceReference, Set<HostName>> hostsDown;
+
+    private volatile long cacheRefreshedAt;
 
     @Inject
     public ZookeeperStatusService(@Component Curator curator) {
         this.curator = curator;
-    }
-
-    @Override
-    public ReadOnlyStatusRegistry forApplicationInstance(ApplicationInstanceReference applicationInstanceReference) {
-        return new ReadOnlyStatusRegistry() {
-            @Override
-            public HostStatus getHostStatus(HostName hostName) {
-                return getInternalHostStatus(applicationInstanceReference, hostName);
-            }
-
-            @Override
-            public ApplicationInstanceStatus getApplicationInstanceStatus() {
-                return getInternalApplicationInstanceStatus(applicationInstanceReference);
-            }
-        };
+        this.counter = new CuratorCounter(curator, HOST_STATUS_CACHE_COUNTER_PATH);
+        this.cacheRefreshedAt = counter.get();
+        this.hostsDown = new ConcurrentHashMap<>();
     }
 
     @Override
@@ -80,6 +78,18 @@ public class ZookeeperStatusService implements StatusService {
     }
 
     /**
+     * Cache is checked for freshness when this mapping is created, and may be invalidated again later
+     * by other users of the cache. Since this function is backed by the cache, any such invalidations
+     * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it.
+     */
+    @Override
+    public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() {
+        Map<ApplicationInstanceReference, Set<HostName>> suspendedHostsByApplication = getValidCache();
+        return application -> suspendedHostsByApplication.computeIfAbsent(application, this::hostsDownFor);
+    }
+
+
+    /**
      *  1) locks the status service for an application instance.
      *  2) fails all operations in this thread when the session is lost,
      *     since session loss might cause the lock to be lost.
@@ -107,72 +117,89 @@ public class ZookeeperStatusService implements StatusService {
         }
     }
 
-    private InterProcessSemaphoreMutex acquireMutexOrThrow(long timeout, TimeUnit timeoutTimeUnit, String lockPath) throws Exception {
-        InterProcessSemaphoreMutex mutex = new InterProcessSemaphoreMutex(curator.framework(), lockPath);
-
-        log.log(LogLevel.DEBUG, "Waiting for lock on " + lockPath);
-        boolean acquired = mutex.acquire(timeout, timeoutTimeUnit);
-        if (!acquired) {
-            log.log(LogLevel.DEBUG, "Timed out waiting for lock on " + lockPath);
-            throw new TimeoutException("Timed out waiting for lock on " + lockPath);
-        }
-        log.log(LogLevel.DEBUG, "Successfully acquired lock on " + lockPath);
-        return mutex;
-    }
-
     private void setHostStatus(ApplicationInstanceReference applicationInstanceReference,
                                HostName hostName,
                                HostStatus status) {
         String path = hostAllowedDownPath(applicationInstanceReference, hostName);
 
+        boolean invalidate = false;
         try {
             switch (status) {
                 case NO_REMARKS:
-                    deleteNode_ignoreNoNodeException(path,"Host already has state NO_REMARKS, path = " + path);
+                    invalidate = deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path);
                     break;
                 case ALLOWED_TO_BE_DOWN:
-                    createNode_ignoreNodeExistsException(path,
-                                                         "Host already has state ALLOWED_TO_BE_DOWN, path = " + path);
+                    invalidate = createNode_ignoreNodeExistsException(path, "Host already has state ALLOWED_TO_BE_DOWN, path = " + path);
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unexpected status '" + status + "'.");
             }
         } catch (Exception e) {
-            //TODO: IOException with explanation
+            invalidate = true;
             throw new RuntimeException(e);
         }
+        finally {
+            if (invalidate) {
+                counter.next();
+                hostsDown.remove(applicationInstanceReference);
+            }
+        }
     }
 
-    private void deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception {
+    private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception {
         try {
             curator.framework().delete().forPath(path);
+            return true;
         } catch (NoNodeException e) {
             log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e);
+            return false;
         }
     }
 
-    private void createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) throws Exception {
+    private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) throws Exception {
         try {
             curator.framework().create()
                     .creatingParentsIfNeeded()
                     .forPath(path);
+            return true;
         } catch (NodeExistsException e) {
             log.log(LogLevel.DEBUG, debugLogMessageIfExists, e);
+            return false;
+        }
+    }
+
+    @Override
+    public HostStatus getHostStatus(ApplicationInstanceReference applicationInstanceReference, HostName hostName) {
+        return getValidCache().computeIfAbsent(applicationInstanceReference, this::hostsDownFor)
+                              .contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+    }
+
+    /** Holding an application's lock ensures the cache is up to date for that application. */
+    private Map<ApplicationInstanceReference, Set<HostName>> getValidCache() {
+        long cacheGeneration = counter.get();
+        if (counter.get() != cacheRefreshedAt) {
+            cacheRefreshedAt = cacheGeneration;
+            hostsDown.clear();
         }
+        return hostsDown;
     }
 
-    //TODO: Eliminate repeated calls to getHostStatus, replace with bulk operation.
-    private HostStatus getInternalHostStatus(ApplicationInstanceReference applicationInstanceReference, HostName hostName) {
+    private Set<HostName> hostsDownFor(ApplicationInstanceReference application) {
         try {
-            Stat statOrNull = curator.framework().checkExists().forPath(
-                    hostAllowedDownPath(applicationInstanceReference, hostName));
+            if (curator.framework().checkExists().forPath(hostsAllowedDownPath(application)) == null)
+                return Collections.emptySet();
 
-            return (statOrNull == null) ? HostStatus.NO_REMARKS : HostStatus.ALLOWED_TO_BE_DOWN;
-        } catch (Exception e) {
-            //TODO: IOException with explanation - Should we only catch IOExceptions or are they a special case?
+            return curator.framework().getChildren().forPath(hostsAllowedDownPath(application))
+                          .stream().map(HostName::new)
+                          .collect(Collectors.toUnmodifiableSet());
+        }
+        catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    /** Common implementation for the two internal classes that sets ApplicationInstanceStatus. */
-    private ApplicationInstanceStatus getInternalApplicationInstanceStatus(ApplicationInstanceReference applicationInstanceReference) {
+    @Override
+    public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference applicationInstanceReference) {
         try {
             Stat statOrNull = curator.framework().checkExists().forPath(
                     applicationInstanceSuspendedPath(applicationInstanceReference));
@@ -183,12 +210,6 @@ public class ZookeeperStatusService implements StatusService {
         }
     }
 
-    private HostStatus getHostStatusWithLock(
-            final ApplicationInstanceReference applicationInstanceReference,
-            final HostName hostName) {
-        return getInternalHostStatus(applicationInstanceReference, hostName);
-    }
-
     private static String applicationInstancePath(ApplicationInstanceReference applicationInstanceReference) {
         return HOST_STATUS_BASE_PATH + '/' +
                 applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId();
@@ -198,10 +219,6 @@ public class ZookeeperStatusService implements StatusService {
         return applicationInstancePath(applicationInstanceReference) + "/hosts-allowed-down";
     }
 
-    private static String applicationInstanceLockPath(ApplicationInstanceReference applicationInstanceReference) {
-        return applicationInstancePath(applicationInstanceReference) + "/lock";
-    }
-
     private static String applicationInstanceLock2Path(ApplicationInstanceReference applicationInstanceReference) {
         return applicationInstancePath(applicationInstanceReference) + "/lock2";
     }
@@ -229,6 +246,21 @@ public class ZookeeperStatusService implements StatusService {
         }
 
         @Override
+        public ApplicationInstanceStatus getStatus() {
+            return getApplicationInstanceStatus(applicationInstanceReference);
+        }
+
+        @Override
+        public HostStatus getHostStatus(HostName hostName) {
+            return ZookeeperStatusService.this.getHostStatus(applicationInstanceReference, hostName);
+        }
+
+        @Override
+        public Set<HostName> getSuspendedHosts() {
+            return getValidCache().computeIfAbsent(applicationInstanceReference, ZookeeperStatusService.this::hostsDownFor);
+        }
+
+        @Override
         public void setHostState(final HostName hostName, final HostStatus status) {
             if (probe) return;
             log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status);
@@ -259,17 +291,6 @@ public class ZookeeperStatusService implements StatusService {
         }
 
         @Override
-        public HostStatus getHostStatus(final HostName hostName) {
-            return getHostStatusWithLock(applicationInstanceReference, hostName);
-        }
-
-        @Override
-        public ApplicationInstanceStatus getApplicationInstanceStatus() {
-            return getInternalApplicationInstanceStatus(applicationInstanceReference);
-        }
-
-        @Override
-        @NoThrow
         public void close()  {
             try {
                 lock.close();
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index cc6b9a7dbf7..89172a831f4 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
@@ -20,9 +21,9 @@ import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.InMemoryStatusService;
-import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.service.monitor.ServiceModel;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -31,9 +32,8 @@ import org.mockito.InOrder;
 
 import java.util.Arrays;
 import java.util.Iterator;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import java.util.Map;
+import java.util.Set;
 
 import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
 import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.NO_REMARKS;
@@ -49,12 +49,10 @@ import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
 
 /**
- * Test Orchestrator with a mock backend (the InMemoryStatusService)
+ * Test Orchestrator with a mock backend (the MockCurator)
  *
  * @author smorgrav
  */
@@ -79,7 +77,7 @@ public class OrchestratorImplTest {
         clustercontroller = new ClusterControllerClientFactoryMock();
         orchestrator = new OrchestratorImpl(
                 clustercontroller,
-                new InMemoryStatusService(),
+                new ZookeeperStatusService(new MockCurator()),
                 new OrchestratorConfig(new OrchestratorConfig.Builder()),
                 new DummyInstanceLookupService());
 
@@ -312,16 +310,8 @@ public class OrchestratorImplTest {
 
     @Test
     public void testGetHost() throws Exception {
-        ClusterControllerClientFactory clusterControllerClientFactory =
-                mock(ClusterControllerClientFactory.class);
-        StatusService statusService = mock(StatusService.class);
-        InstanceLookupService lookupService = mock(InstanceLookupService.class);
-
-        orchestrator = new OrchestratorImpl(
-                clusterControllerClientFactory,
-                statusService,
-                new OrchestratorConfig(new OrchestratorConfig.Builder()),
-                lookupService);
+        ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
+        StatusService statusService = new ZookeeperStatusService(new MockCurator());
 
         HostName hostName = new HostName("host.yahoo.com");
         TenantId tenantId = new TenantId("tenant");
@@ -335,32 +325,30 @@ public class OrchestratorImplTest {
                 new ApplicationInstance(
                         tenantId,
                         applicationInstanceId,
-                        Stream.of(new ServiceCluster(
+                        Set.of(new ServiceCluster(
                                 new ClusterId("clusterId"),
                                 new ServiceType("serviceType"),
-                                Stream.of(
-                                        new ServiceInstance(
-                                                new ConfigId("configId1"),
-                                                hostName,
-                                                ServiceStatus.UP),
-                                new ServiceInstance(
-                                        new ConfigId("configId2"),
-                                        hostName,
-                                        ServiceStatus.NOT_CHECKED))
-                                        .collect(Collectors.toSet())))
-                        .collect(Collectors.toSet()));
-
-        when(lookupService.findInstanceByHost(hostName))
-                .thenReturn(Optional.of(applicationInstance));
-
-        ReadOnlyStatusRegistry readOnlyStatusRegistry = mock(ReadOnlyStatusRegistry.class);
-        when(statusService.forApplicationInstance(reference))
-                .thenReturn(readOnlyStatusRegistry);
-        when(readOnlyStatusRegistry.getHostStatus(hostName))
-                .thenReturn(HostStatus.ALLOWED_TO_BE_DOWN);
+                                Set.of(new ServiceInstance(
+                                               new ConfigId("configId1"),
+                                               hostName,
+                                               ServiceStatus.UP),
+                                       new ServiceInstance(
+                                               new ConfigId("configId2"),
+                                               hostName,
+                                               ServiceStatus.NOT_CHECKED)))));
+
+        InstanceLookupService lookupService = new ServiceMonitorInstanceLookupService(
+                () -> new ServiceModel(Map.of(reference, applicationInstance)));
 
-        Host host = orchestrator.getHost(hostName);
+        orchestrator = new OrchestratorImpl(
+                clusterControllerClientFactory,
+                statusService,
+                new OrchestratorConfig(new OrchestratorConfig.Builder()),
+                lookupService);
+
+        orchestrator.setNodeStatus(hostName, HostStatus.ALLOWED_TO_BE_DOWN);
 
+        Host host = orchestrator.getHost(hostName);
         assertEquals(reference, host.getApplicationInstanceReference());
         assertEquals(hostName, host.getHostName());
         assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostStatus());
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
index a712c1db3e8..5a11d3aa640 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
@@ -167,8 +167,7 @@ public class ApplicationApiImplTest {
     }
 
     private void verifyUpConditionWith(HostStatus hostStatus, ServiceStatus serviceStatus, boolean expectUp) {
-        HostName hostName1 = modelUtils.createNode("host1", hostStatus);
-
+        HostName hostName1 = new HostName("host1");
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
                         modelUtils.createServiceCluster(
@@ -178,6 +177,8 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode("host1", hostStatus);
+
         ApplicationApiImpl applicationApi = modelUtils.createApplicationApiImpl(applicationInstance, hostName1);
         List<HostName> upStorageNodes = expectUp ? Arrays.asList(hostName1) : new ArrayList<>();
 
@@ -189,9 +190,9 @@ public class ApplicationApiImplTest {
 
     @Test
     public void testGetNodesInGroupWithStatus() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
 
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
@@ -213,6 +214,10 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+
         verifyNodesInGroupWithoutRemarks(
                 modelUtils.createApplicationApiImpl(applicationInstance, hostName1),
                 Arrays.asList(hostName1),
@@ -242,13 +247,13 @@ public class ApplicationApiImplTest {
 
     @Test
     public void testGetStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() {
-        HostName allowedToBeDownHost1 = modelUtils.createNode("host1", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName allowedToBeDownHost3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName allowedToBeDownHost4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost5 = modelUtils.createNode("host5", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost6 = modelUtils.createNode("host6", HostStatus.NO_REMARKS);
-        HostName allowedToBeDownHost7 = modelUtils.createNode("host7", HostStatus.ALLOWED_TO_BE_DOWN);
+        HostName allowedToBeDownHost1 = new HostName("host1");
+        HostName noRemarksHost2 = new HostName("host2");
+        HostName allowedToBeDownHost3 = new HostName("host3");
+        HostName allowedToBeDownHost4 = new HostName("host4");
+        HostName noRemarksHost5 = new HostName("host5");
+        HostName noRemarksHost6 = new HostName("host6");
+        HostName allowedToBeDownHost7 = new HostName("host7");
 
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
@@ -286,6 +291,14 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode(allowedToBeDownHost1, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(noRemarksHost2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(allowedToBeDownHost3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(allowedToBeDownHost4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(noRemarksHost5, HostStatus.ALLOWED_TO_BE_DOWN); // Really?
+        modelUtils.createNode(noRemarksHost6, HostStatus.NO_REMARKS);
+        modelUtils.createNode(allowedToBeDownHost7, HostStatus.ALLOWED_TO_BE_DOWN);
+
         verifyStorageNodesAllowedToBeDown(
                 modelUtils.createApplicationApiImpl(applicationInstance, allowedToBeDownHost1), allowedToBeDownHost1);
         verifyStorageNodesAllowedToBeDown(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 8e11b85241f..0a53972b30c 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -10,6 +10,7 @@ import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Optional;
 
@@ -24,12 +25,11 @@ public class ClusterApiImplTest {
 
     @Test
     public void testServicesDownAndNotInGroup() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName5 = modelUtils.createNode("host5", HostStatus.NO_REMARKS);
-
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
+        HostName hostName4 = new HostName("host4");
+        HostName hostName5 = new HostName("host5");
 
         ServiceCluster serviceCluster = modelUtils.createServiceCluster(
                 "cluster",
@@ -42,6 +42,13 @@ public class ClusterApiImplTest {
                         modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP)
                 )
         );
+        modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster));
+
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName5, HostStatus.NO_REMARKS);
 
         ClusterApiImpl clusterApi = new ClusterApiImpl(
                 applicationApi,
@@ -67,12 +74,11 @@ public class ClusterApiImplTest {
 
     @Test
     public void testNoServices() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName5 = modelUtils.createNode("host5", HostStatus.NO_REMARKS);
-
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
+        HostName hostName4 = new HostName("host4");
+        HostName hostName5 = new HostName("host5");
 
         ServiceCluster serviceCluster = modelUtils.createServiceCluster(
                 "cluster",
@@ -85,6 +91,13 @@ public class ClusterApiImplTest {
                         modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP)
                 )
         );
+        modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster));
+
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName5, HostStatus.NO_REMARKS);
 
         verifyNoServices(serviceCluster, false, false, hostName1);
         verifyNoServices(serviceCluster, true, false, hostName2);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 9586f92af30..78f50dbfc3f 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.model;
 
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -11,37 +12,58 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.orchestrator.OrchestratorImpl;
+import com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService;
+import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
+import com.yahoo.vespa.orchestrator.config.OrchestratorConfig.Builder;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.yolean.Exceptions;
 
+import java.time.Clock;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+class ModelTestUtils {
 
-public class ModelTestUtils {
-    private final MutableStatusRegistry statusRegistry = mock(MutableStatusRegistry.class);
-    private final ClusterControllerClientFactory clusterControllerClientFactory = mock(ClusterControllerClientFactory.class);
+    private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>();
+    private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
     private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>();
-
-    ModelTestUtils() {
-        when(statusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS);
-    }
+    private final StatusService statusService = new ZookeeperStatusService(new MockCurator());
+    private final Orchestrator orchestrator = new OrchestratorImpl(clusterControllerClientFactory,
+                                                                   statusService,
+                                                                   new OrchestratorConfig(new Builder()),
+                                                                   new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)));
 
     Map<HostName, HostStatus> getHostStatusMap() {
         return hostStatusMap;
     }
 
     HostName createNode(String name, HostStatus hostStatus) {
-        HostName hostName = new HostName(name);
+        return createNode(new HostName(name), hostStatus);
+    }
+
+    HostName createNode(HostName hostName, HostStatus hostStatus) {
         hostStatusMap.put(hostName, hostStatus);
-        when(statusRegistry.getHostStatus(hostName)).thenReturn(hostStatus);
+        try {
+            orchestrator.setNodeStatus(hostName, hostStatus);
+        }
+        catch (OrchestrationException e) {
+            throw new AssertionError("Host '" + hostName + "' not owned by any application — please assign it first: " +
+                                             Exceptions.toMessageString(e));
+        }
         return hostName;
     }
 
@@ -49,26 +71,29 @@ public class ModelTestUtils {
             ApplicationInstance applicationInstance,
             HostName... hostnames) {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostnames);
-        return new ApplicationApiImpl(nodeGroup, statusRegistry, clusterControllerClientFactory);
+        MutableStatusRegistry registry = statusService.lockApplicationInstance_forCurrentThreadOnly(
+                OrchestratorContext.createContextForSingleAppOp(Clock.systemUTC()),
+                applicationInstance.reference());
+        return new ApplicationApiImpl(nodeGroup, registry, clusterControllerClientFactory);
     }
 
     ApplicationInstance createApplicationInstance(
             List<ServiceCluster> serviceClusters) {
-        Set<ServiceCluster> serviceClusterSet = serviceClusters.stream()
-                .collect(Collectors.toSet());
+        Set<ServiceCluster> serviceClusterSet = new HashSet<>(serviceClusters);
 
-        return new ApplicationInstance(
+        ApplicationInstance application = new ApplicationInstance(
                 new TenantId("tenant"),
                 new ApplicationInstanceId("application-name:foo:bar:default"),
                 serviceClusterSet);
+        applications.put(application.reference(), application);
+        return application;
     }
 
     ServiceCluster createServiceCluster(
             String clusterId,
             ServiceType serviceType,
             List<ServiceInstance> serviceInstances) {
-        Set<ServiceInstance> serviceInstanceSet = serviceInstances.stream()
-                .collect(Collectors.toSet());
+        Set<ServiceInstance> serviceInstanceSet = new HashSet<>(serviceInstances);
 
         return new ServiceCluster(
                 new ClusterId(clusterId),
@@ -86,7 +111,8 @@ public class ModelTestUtils {
                 status);
     }
 
-    public ClusterControllerClientFactory getClusterControllerClientFactory() {
+    ClusterControllerClientFactory getClusterControllerClientFactory() {
         return clusterControllerClientFactory;
     }
+
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
index c575811fd6d..94df28a6921 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
@@ -158,7 +158,8 @@ public class ApplicationSuspensionResourceTest {
                 "        <config name=\"container.handler.threadpool\">\n" +
                 "            <maxthreads>10</maxthreads>\n" +
                 "        </config>\n" +
-                "        <component id=\"com.yahoo.vespa.orchestrator.status.InMemoryStatusService\" bundle=\"orchestrator\" />\n" +
+                "        <component id=\"com.yahoo.vespa.curator.mock.MockCurator\" bundle=\"zkfacade\" />\n" +
+                "        <component id=\"com.yahoo.vespa.orchestrator.status.ZookeeperStatusService\" bundle=\"orchestrator\" />\n" +
                 "        <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" +
                 "        <component id=\"com.yahoo.vespa.orchestrator.OrchestratorImpl\" bundle=\"orchestrator\" />\n" +
                 "        <component id=\"com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock\" bundle=\"orchestrator\" />\n" +
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index e943a4f105b..55ac65c036c 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException;
 import com.yahoo.vespa.orchestrator.BatchInternalErrorException;
 import com.yahoo.vespa.orchestrator.Host;
@@ -29,14 +30,13 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.Policy;
 import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult;
 import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
-import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
-import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -58,7 +58,6 @@ import static org.fest.assertions.Assertions.assertThat;
 import static org.fest.assertions.Fail.fail;
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -73,21 +72,7 @@ public class HostResourceTest {
     private static final int SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS = 0;
     private static final TenantId TENANT_ID = new TenantId("tenantId");
     private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId");
-    private static final ApplicationInstanceReference APPLICATION_INSTANCE_REFERENCE =
-            new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID);
-
-    private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = mock(StatusService.class);
-    private static final MutableStatusRegistry EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY = mock(MutableStatusRegistry.class);
-    static {
-        when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.forApplicationInstance(eq(APPLICATION_INSTANCE_REFERENCE)))
-                .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
-        when(EVERY_HOST_IS_UP_HOST_STATUS_SERVICE.lockApplicationInstance_forCurrentThreadOnly(any(), eq(APPLICATION_INSTANCE_REFERENCE)))
-                .thenReturn(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY);
-        when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getHostStatus(any()))
-                .thenReturn(HostStatus.NO_REMARKS);
-        when(EVERY_HOST_IS_UP_MUTABLE_HOST_STATUS_REGISTRY.getApplicationInstanceStatus())
-                .thenReturn(ApplicationInstanceStatus.NO_REMARKS);
-    }
+    private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZookeeperStatusService(new MockCurator());
 
     private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class);
     static {
@@ -99,7 +84,6 @@ public class HostResourceTest {
                                 makeServiceClusterSet())));
     }
 
-
     private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() {
         @Override
         public Optional<ApplicationInstance> findInstanceById(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
index d57b0106b5b..9b1be12121d 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.TestIds;
 import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.curator.SessionFailRetryLoop.SessionFailedException;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.test.KillSession;
 import org.apache.curator.test.TestingServer;
@@ -56,10 +55,6 @@ public class ZookeeperStatusServiceTest {
         when(context.isProbe()).thenReturn(false);
     }
 
-    private static Curator createConnectedCuratorFramework(TestingServer server) throws InterruptedException {
-        return createConnectedCurator(server);
-    }
-
     private static Curator createConnectedCurator(TestingServer server) throws InterruptedException {
         Curator curator = Curator.create(server.getConnectString());
         curator.framework().blockUntilConnected(1, TimeUnit.MINUTES);
@@ -80,8 +75,7 @@ public class ZookeeperStatusServiceTest {
     @Test
     public void host_state_for_unknown_hosts_is_no_remarks() {
         assertThat(
-                zookeeperStatusService.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
-                        .getHostStatus(TestIds.HOST_NAME1),
+                zookeeperStatusService.getHostStatus(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1),
                 is(HostStatus.NO_REMARKS));
     }
 
@@ -92,71 +86,65 @@ public class ZookeeperStatusServiceTest {
 
             //shuffling to catch "clean database" failures for all cases.
             for (HostStatus hostStatus: shuffledList(HostStatus.values())) {
-                doTimes(2, () -> {
+                for (int i = 0; i < 2; i++) {
                     statusRegistry.setHostState(
                             TestIds.HOST_NAME1,
                             hostStatus);
 
-                    assertThat(statusRegistry.getHostStatus(
-                                    TestIds.HOST_NAME1),
-                            is(hostStatus));
-                });
+                    assertThat(statusRegistry.getHostStatus(TestIds.HOST_NAME1),
+                               is(hostStatus));
+                }
             }
         }
     }
 
     @Test
     public void locks_are_exclusive() throws Exception {
-        try (Curator curator = createConnectedCuratorFramework(testingServer)) {
-            ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator);
-
-            final CompletableFuture<Void> lockedSuccessfullyFuture;
-            try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                    .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator);
 
-                lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> {
-                    try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
-                            .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE))
-                    {
-                    }
-                });
+        final CompletableFuture<Void> lockedSuccessfullyFuture;
+        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
+                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
 
-                try {
-                    lockedSuccessfullyFuture.get(3, TimeUnit.SECONDS);
-                    fail("Both zookeeper host status services locked simultaneously for the same application instance");
-                } catch (TimeoutException ignored) {
+            lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> {
+                try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
+                        .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE))
+                {
                 }
-            }
+            });
 
-            lockedSuccessfullyFuture.get(1, TimeUnit.MINUTES);
+            try {
+                lockedSuccessfullyFuture.get(3, TimeUnit.SECONDS);
+                fail("Both zookeeper host status services locked simultaneously for the same application instance");
+            } catch (TimeoutException ignored) {
+            }
         }
+
+        lockedSuccessfullyFuture.get(1, TimeUnit.MINUTES);
     }
 
     @Test
     public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception {
-        try (Curator curator = createConnectedCuratorFramework(testingServer)) {
-            ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator);
+        ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator);
 
-            try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                    .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
+                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
 
-                //must run in separate thread, since having 2 locks in the same thread fails
-                CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> {
-                    try {
-                        zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE);
-                        fail("Both zookeeper host status services locked simultaneously for the same application instance");
-                    } catch (RuntimeException e) {
-                    }
+            //must run in separate thread, since having 2 locks in the same thread fails
+            CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> {
+                try {
+                    zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE);
+                    fail("Both zookeeper host status services locked simultaneously for the same application instance");
+                } catch (RuntimeException e) {
+                }
 
-                    killSession(curator.framework(), testingServer);
+                killSession(curator.framework(), testingServer);
 
-                    //Throws SessionFailedException if the SessionFailRetryLoop has not been closed.
-                    zookeeperStatusService2.forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
-                            .getHostStatus(TestIds.HOST_NAME1);
-                });
+                //Throws SessionFailedException if the SessionFailRetryLoop has not been closed.
+                statusRegistry.getHostStatus(TestIds.HOST_NAME1);
+            });
 
-                assertThat(resultOfZkOperationAfterLockFailure, notHoldsException());
-            }
+            assertThat(resultOfZkOperationAfterLockFailure, notHoldsException());
         }
     }
 
@@ -211,8 +199,7 @@ public class ZookeeperStatusServiceTest {
         // Initial state is NO_REMARK
         assertThat(
                 zookeeperStatusService
-                        .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
-                        .getApplicationInstanceStatus(),
+                        .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.NO_REMARKS));
 
         // Suspend
@@ -223,8 +210,7 @@ public class ZookeeperStatusServiceTest {
 
         assertThat(
                 zookeeperStatusService
-                        .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
-                        .getApplicationInstanceStatus(),
+                        .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN));
 
         // Resume
@@ -235,8 +221,7 @@ public class ZookeeperStatusServiceTest {
 
         assertThat(
                 zookeeperStatusService
-                        .forApplicationInstance(TestIds.APPLICATION_INSTANCE_REFERENCE)
-                        .getApplicationInstanceStatus(),
+                        .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.NO_REMARKS));
     }
 
@@ -262,17 +247,6 @@ public class ZookeeperStatusServiceTest {
         assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE2));
     }
 
-    private static void assertSessionFailed(Runnable statusServiceOperations) {
-        try {
-            statusServiceOperations.run();
-            fail("Expected session expired exception");
-        } catch (RuntimeException e) {
-            if (!(e.getCause() instanceof SessionFailedException)) {
-                throw e;
-            }
-        }
-    }
-
     //TODO: move to vespajlib
     private static <T> List<T> shuffledList(T[] values) {
         //new ArrayList necessary to avoid "write through" behaviour
@@ -281,10 +255,4 @@ public class ZookeeperStatusServiceTest {
         return list;
     }
 
-    //TODO: move to vespajlib
-    private static void doTimes(int numberOfIterations, Runnable runnable) {
-        for (int i = 0; i < numberOfIterations; i++) {
-            runnable.run();
-        }
-    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
index 50ea31eb9c4..0a40555036c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl;
 import java.util.Map;
 
 public class ServiceMonitorImpl implements ServiceMonitor {
+
     private final ServiceModelCache serviceModelProvider;
 
     @Inject
@@ -37,12 +38,8 @@ public class ServiceMonitorImpl implements ServiceMonitor {
     }
 
     @Override
-    public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
-        return serviceModelProvider.get().getAllApplicationInstances();
-    }
-
-    @Override
     public ServiceModel getServiceModelSnapshot() {
         return serviceModelProvider.get();
     }
+
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
index b62552188e1..ed72893400a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
@@ -4,9 +4,13 @@ package com.yahoo.vespa.service.monitor;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
 
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -20,9 +24,11 @@ import java.util.stream.Collectors;
 public class ServiceModel {
 
     private final Map<ApplicationInstanceReference, ApplicationInstance> applications;
+    private final Map<HostName, ApplicationInstance> applicationsByHostName;
 
     public ServiceModel(Map<ApplicationInstanceReference, ApplicationInstance> applications) {
         this.applications = Collections.unmodifiableMap(applications);
+        this.applicationsByHostName = Collections.unmodifiableMap(applicationsByHostNames(applications.values()));
     }
 
     public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
@@ -33,6 +39,10 @@ public class ServiceModel {
         return Optional.ofNullable(applications.get(reference));
     }
 
+    public Map<HostName, ApplicationInstance> getApplicationsByHostName() {
+        return applicationsByHostName;
+    }
+
     public Map<HostName, List<ServiceInstance>> getServiceInstancesByHostName() {
         return applications.values().stream()
                 .flatMap(application -> application.serviceClusters().stream())
@@ -40,4 +50,17 @@ public class ServiceModel {
                 .collect(Collectors.groupingBy(ServiceInstance::hostName, Collectors.toList()));
     }
 
+    private static Map<HostName, ApplicationInstance> applicationsByHostNames(Collection<ApplicationInstance> applications) {
+        Map<HostName, ApplicationInstance> hosts = new HashMap<>();
+        for (ApplicationInstance application : applications)
+            for (ServiceCluster cluster : application.serviceClusters())
+                for (ServiceInstance instance : cluster.serviceInstances()) {
+                    ApplicationInstance previous = hosts.put(instance.hostName(), application);
+                    if (previous != null && ! previous.equals(application))
+                        throw new IllegalStateException("Major assumption broken: Multiple application instances contain host " +
+                                                        instance.hostName().s() + ": " + Arrays.asList(previous, application));
+                }
+        return hosts;
+    }
+
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
index 5ed34673da5..49539c61e5d 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
@@ -1,11 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.monitor;
 
-import com.yahoo.vespa.applicationmodel.ApplicationInstance;
-import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
-
-import java.util.Map;
-
 /**
  * The service monitor interface. A service monitor provides up to date information about the liveness status
  * (up, down or not known) of each service instance in a Vespa zone
@@ -15,11 +10,9 @@ import java.util.Map;
 public interface ServiceMonitor {
 
     /**
-     * Returns the current liveness status (up, down or unknown) of all instances
+     * Returns a ServiceModel which contains the current liveness status (up, down or unknown) of all instances
      * of all services of all clusters of all applications in a zone.
      */
-    Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances();
-
     ServiceModel getServiceModelSnapshot();
 
 }