Expose host status cache, and use it for all bulk operations

18 files changed, 182 insertions, 91 deletions

diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
index 73abd70a5ae..aa157366a60 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
@@ -11,7 +11,9 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * (Only the suspended applications part of this is in use)
@@ -34,6 +36,11 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        return hostName -> Optional.of(getNodeStatus(hostName));
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus state) {}
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 39a2787ca9b..42dbcdf7a86 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -34,7 +35,7 @@ import java.util.stream.Collectors;
 public class MetricsReporter extends Maintainer {
 
     private final Metric metric;
-    private final Orchestrator orchestrator;
+    private final Function<HostName, Optional<HostStatus>> orchestrator;
     private final ServiceMonitor serviceMonitor;
     private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>();
     private final Supplier<Integer> pendingRedeploymentsSupplier;
@@ -48,7 +49,7 @@ public class MetricsReporter extends Maintainer {
                            JobControl jobControl) {
         super(nodeRepository, interval, jobControl);
         this.metric = metric;
-        this.orchestrator = orchestrator;
+        this.orchestrator = orchestrator.getNodeStatuses();
         this.serviceMonitor = serviceMonitor;
         this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
     }
@@ -129,13 +130,9 @@ public class MetricsReporter extends Maintainer {
                 node.status().hardwareDivergence().isPresent() ? 1 : 0,
                 context);
 
-        try {
-            HostStatus status = orchestrator.getNodeStatus(new HostName(node.hostname()));
-            boolean allowedToBeDown = status == HostStatus.ALLOWED_TO_BE_DOWN;
-            metric.set("allowedToBeDown", allowedToBeDown ? 1 : 0, context);
-        } catch (HostNameNotFoundException e) {
-            // Ignore
-        }
+        orchestrator.apply(new HostName(node.hostname()))
+                    .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0)
+                    .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context));
 
         long numberOfServices;
         HostName hostName = new HostName(node.hostname());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 2a2ca6bfd87..5b942497be8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -22,7 +22,9 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
 * @author bratseth
@@ -40,7 +42,7 @@ class NodesResponse extends HttpResponse {
 
     private final NodeFilter filter;
     private final boolean recursive;
-    private final Orchestrator orchestrator;
+    private final Function<HostName, Optional<HostStatus>> orchestrator;
     private final NodeRepository nodeRepository;
     private final Slime slime;
     private final NodeSerializer serializer = new NodeSerializer();
@@ -52,7 +54,7 @@ class NodesResponse extends HttpResponse {
         this.nodeParentUrl = toNodeParentUrl(request);
         filter = NodesApiHandler.toNodeFilter(request);
         this.recursive = request.getBooleanProperty("recursive");
-        this.orchestrator = orchestrator;
+        this.orchestrator = orchestrator.getNodeStatuses();
         this.nodeRepository = nodeRepository;
 
         slime = new Slime();
@@ -158,11 +160,9 @@ class NodesResponse extends HttpResponse {
             object.setLong("currentRestartGeneration", node.allocation().get().restartGeneration().current());
             object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(node.allocation().get().membership().cluster().vespaVersion()).asString());
             object.setString("wantedVespaVersion", node.allocation().get().membership().cluster().vespaVersion().toFullString());
-            try {
-                object.setBool("allowedToBeDown", 
-                               orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN);
-            }
-            catch (HostNameNotFoundException e) {/* ok */ }
+            orchestrator.apply(new HostName(node.hostname()))
+                        .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN)
+                        .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown));
         }
         object.setLong("rebootGeneration", node.status().reboot().wanted());
         object.setLong("currentRebootGeneration", node.status().reboot().current());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 70750dd6672..96ec0349fb2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -11,7 +11,9 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * @author bratseth
@@ -32,6 +34,11 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        return hostName -> Optional.of(getNodeStatus(hostName));
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus state) {}
 
     @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
index 1e502439aeb..57b942d85e4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java
@@ -38,7 +38,6 @@ import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -90,7 +89,7 @@ public class MetricsReporterTest {
 
         Orchestrator orchestrator = mock(Orchestrator.class);
         ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
         ServiceModel serviceModel = mock(ServiceModel.class);
         when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
         when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap());
@@ -137,7 +136,7 @@ public class MetricsReporterTest {
 
         Orchestrator orchestrator = mock(Orchestrator.class);
         ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatus(any())).thenReturn(HostStatus.NO_REMARKS);
+        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
         ServiceModel serviceModel = mock(ServiceModel.class);
         when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
         when(serviceModel.getServiceInstancesByHostName()).thenReturn(Collections.emptyMap());
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index a639d07e504..59b320cf501 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -10,7 +10,9 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * The orchestrator is used to coordinate the need of vespa services to restart or
@@ -46,6 +48,14 @@ public interface Orchestrator {
      */
     HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
 
+    /**
+     * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this.
+     *
+     * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads.
+     * @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}.
+     */
+    Function<HostName, Optional<HostStatus>> getNodeStatuses();
+
     void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
 
     /**
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index 33cfa310a68..5d5d6df0fb3 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -35,8 +35,10 @@ import java.time.Clock;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -104,6 +106,14 @@ public class OrchestratorImpl implements Orchestrator {
     }
 
     @Override
+    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
+        Function<ApplicationInstanceReference, Set<HostName>> suspendedHosts = statusService.getSuspendedHostsByApplication();
+        return hostName -> instanceLookupService.findInstanceByHost(hostName)
+                                                .map(application -> suspendedHosts.apply(application.reference()).contains(hostName)
+                                                                    ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS);
+    }
+
+    @Override
     public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException {
         ApplicationInstanceReference reference = getApplicationInstance(hostName).reference();
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
@@ -141,7 +151,7 @@ public class OrchestratorImpl implements Orchestrator {
 
             ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(appInstance.reference()).getApplicationInstanceStatus();
             if (appStatus == ApplicationInstanceStatus.NO_REMARKS) {
-                policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry);
+                policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry, statusService);
             }
         }
     }
@@ -164,6 +174,7 @@ public class OrchestratorImpl implements Orchestrator {
             ApplicationApi applicationApi = new ApplicationApiImpl(
                     nodeGroup,
                     statusRegistry,
+                    statusService,
                     clusterControllerClientFactory);
 
             policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi);
@@ -188,6 +199,7 @@ public class OrchestratorImpl implements Orchestrator {
 
             ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup,
                                                                    hostStatusRegistry,
+                                                                   statusService,
                                                                    clusterControllerClientFactory);
             policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi);
         }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
index 79506d042e2..de8f7e9f2bc 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
@@ -57,14 +57,6 @@ public class OrchestratorUtil {
                 .collect(toSet());
     }
 
-    public static Map<HostName, HostStatus> getHostStatusMap(Collection<HostName> hosts,
-                                                             ReadOnlyStatusRegistry hostStatusService) {
-        return hosts.stream()
-                .collect(Collectors.toMap(
-                        hostName -> hostName,
-                        hostName -> hostStatusService.getHostStatus(hostName)));
-    }
-
     private static boolean hasServiceInstanceOnHost(ServiceCluster serviceCluster, HostName hostName) {
         return serviceInstancesOnHost(serviceCluster, hostName).count() > 0;
     }
@@ -75,11 +67,6 @@ public class OrchestratorUtil {
                 .filter(instance -> instance.hostName().equals(hostName));
     }
 
-    public static <K, V1, V2> Map<K, V2> mapValues(Map<K, V1> map, Function<V1, V2> valueConverter) {
-        return map.entrySet().stream()
-                .collect(toMap(Map.Entry::getKey, entry -> valueConverter.apply(entry.getValue())));
-    }
-
     private static final Pattern APPLICATION_INSTANCE_REFERENCE_REST_FORMAT_PATTERN = Pattern.compile("^([^:]+):(.+)$");
 
     /** Returns an ApplicationInstanceReference constructed from the serialized format used in the REST API. */
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
index 5800b48da75..1d74fb337d9 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 import java.util.Collection;
 import java.util.Comparator;
@@ -35,12 +36,16 @@ public class ApplicationApiImpl implements ApplicationApi {
 
     public ApplicationApiImpl(NodeGroup nodeGroup,
                               MutableStatusRegistry hostStatusService,
+                              StatusService statusService,
                               ClusterControllerClientFactory clusterControllerClientFactory) {
         this.applicationInstance = nodeGroup.getApplication();
         this.nodeGroup = nodeGroup;
         this.hostStatusService = hostStatusService;
         Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance);
-        this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(), hostStatusService::getHostStatus));
+        Collection<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(nodeGroup.getApplicationReference());
+        this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(),
+                                                                     hostName -> suspendedHosts.contains(hostName)
+                                                                                 ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS));
         this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory);
     }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
index b6e7014cac0..e761991ac1a 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.orchestrator.model.StorageNode;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 import java.util.logging.Logger;
 
@@ -104,9 +105,10 @@ public class HostedVespaPolicy implements Policy {
             OrchestratorContext context,
             ApplicationInstance applicationInstance,
             HostName hostName,
-            MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException {
+            MutableStatusRegistry hostStatusService,
+            StatusService statusService) throws HostStateChangeDeniedException {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostName);
-        ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory);
+        ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup, hostStatusService, statusService, clusterControllerClientFactory);
         releaseSuspensionGrant(context, applicationApi);
     }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
index e2487301326..7744d75681b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
 
 /**
  * @author oyving
@@ -32,6 +33,7 @@ public interface Policy {
     void releaseSuspensionGrant(
             OrchestratorContext context, ApplicationInstance applicationInstance,
             HostName hostName,
-            MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException;
+            MutableStatusRegistry hostStatusService,
+            StatusService statusService) throws HostStateChangeDeniedException;
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
index cd20a01f6af..65ef1c9eaf0 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
@@ -34,7 +34,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostStatusMap;
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseAppInstanceReference;
 
@@ -82,11 +81,13 @@ public class InstanceResource {
                 = instanceLookupService.findInstanceById(instanceId)
                 .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()));
 
-        Set<HostName> hostsUsedByApplicationInstance = getHostsUsedByApplicationInstance(applicationInstance);
-        Map<HostName, HostStatus> hostStatusMap = getHostStatusMap(hostsUsedByApplicationInstance,
-                                                                   statusService.forApplicationInstance(instanceId));
-        Map<HostName, String> hostStatusStringMap = OrchestratorUtil.mapValues(hostStatusMap, HostStatus::name);
-        return InstanceStatusResponse.create(applicationInstance, hostStatusStringMap);
+        Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference());
+        Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance)
+                .stream()
+                .collect(Collectors.toMap(hostName -> hostName,
+                                          hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name()
+                                                                                        : HostStatus.NO_REMARKS.name()));
+        return InstanceStatusResponse.create(applicationInstance, hostStatusMap);
     }
 
     @GET
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
index c9e846906e3..37f814ef03a 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
@@ -68,8 +68,8 @@ public interface StatusService {
     /**
      * Returns a not necessarily consistent mapping from applications to their set of suspended hosts.
      *
-     * If the lock for an application is held when this is acquired, the view of that application's hosts
-     * is consistent and up to date for as long as the lock is held.
+     * If the lock for an application is held when this mapping is acquired, new sets returned for that application
+     * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.)
      */
     Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication();
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
index 67b14f29764..0ed0d6ad31a 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
@@ -21,6 +21,7 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -91,6 +92,13 @@ public class ZookeeperStatusService implements StatusService {
         }
     }
 
+    @Override
+    public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() {
+        Map<ApplicationInstanceReference, Set<HostName>> suspendedHostsByApplication = getValidCache();
+        return application -> suspendedHostsByApplication.computeIfAbsent(application, this::hostsDownFor);
+    }
+
+
     /**
      *  1) locks the status service for an application instance.
      *  2) fails all operations in this thread when the session is lost,
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
index a712c1db3e8..5a11d3aa640 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
@@ -167,8 +167,7 @@ public class ApplicationApiImplTest {
     }
 
     private void verifyUpConditionWith(HostStatus hostStatus, ServiceStatus serviceStatus, boolean expectUp) {
-        HostName hostName1 = modelUtils.createNode("host1", hostStatus);
-
+        HostName hostName1 = new HostName("host1");
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
                         modelUtils.createServiceCluster(
@@ -178,6 +177,8 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode("host1", hostStatus);
+
         ApplicationApiImpl applicationApi = modelUtils.createApplicationApiImpl(applicationInstance, hostName1);
         List<HostName> upStorageNodes = expectUp ? Arrays.asList(hostName1) : new ArrayList<>();
 
@@ -189,9 +190,9 @@ public class ApplicationApiImplTest {
 
     @Test
     public void testGetNodesInGroupWithStatus() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
 
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
@@ -213,6 +214,10 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+
         verifyNodesInGroupWithoutRemarks(
                 modelUtils.createApplicationApiImpl(applicationInstance, hostName1),
                 Arrays.asList(hostName1),
@@ -242,13 +247,13 @@ public class ApplicationApiImplTest {
 
     @Test
     public void testGetStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() {
-        HostName allowedToBeDownHost1 = modelUtils.createNode("host1", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName allowedToBeDownHost3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName allowedToBeDownHost4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost5 = modelUtils.createNode("host5", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName noRemarksHost6 = modelUtils.createNode("host6", HostStatus.NO_REMARKS);
-        HostName allowedToBeDownHost7 = modelUtils.createNode("host7", HostStatus.ALLOWED_TO_BE_DOWN);
+        HostName allowedToBeDownHost1 = new HostName("host1");
+        HostName noRemarksHost2 = new HostName("host2");
+        HostName allowedToBeDownHost3 = new HostName("host3");
+        HostName allowedToBeDownHost4 = new HostName("host4");
+        HostName noRemarksHost5 = new HostName("host5");
+        HostName noRemarksHost6 = new HostName("host6");
+        HostName allowedToBeDownHost7 = new HostName("host7");
 
         ApplicationInstance applicationInstance =
                 modelUtils.createApplicationInstance(Arrays.asList(
@@ -286,6 +291,14 @@ public class ApplicationApiImplTest {
                         )
                 ));
 
+        modelUtils.createNode(allowedToBeDownHost1, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(noRemarksHost2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(allowedToBeDownHost3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(allowedToBeDownHost4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(noRemarksHost5, HostStatus.ALLOWED_TO_BE_DOWN); // Really?
+        modelUtils.createNode(noRemarksHost6, HostStatus.NO_REMARKS);
+        modelUtils.createNode(allowedToBeDownHost7, HostStatus.ALLOWED_TO_BE_DOWN);
+
         verifyStorageNodesAllowedToBeDown(
                 modelUtils.createApplicationApiImpl(applicationInstance, allowedToBeDownHost1), allowedToBeDownHost1);
         verifyStorageNodesAllowedToBeDown(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 8e11b85241f..0a53972b30c 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -10,6 +10,7 @@ import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Optional;
 
@@ -24,12 +25,11 @@ public class ClusterApiImplTest {
 
     @Test
     public void testServicesDownAndNotInGroup() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName5 = modelUtils.createNode("host5", HostStatus.NO_REMARKS);
-
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
+        HostName hostName4 = new HostName("host4");
+        HostName hostName5 = new HostName("host5");
 
         ServiceCluster serviceCluster = modelUtils.createServiceCluster(
                 "cluster",
@@ -42,6 +42,13 @@ public class ClusterApiImplTest {
                         modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP)
                 )
         );
+        modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster));
+
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName5, HostStatus.NO_REMARKS);
 
         ClusterApiImpl clusterApi = new ClusterApiImpl(
                 applicationApi,
@@ -67,12 +74,11 @@ public class ClusterApiImplTest {
 
     @Test
     public void testNoServices() {
-        HostName hostName1 = modelUtils.createNode("host1", HostStatus.NO_REMARKS);
-        HostName hostName2 = modelUtils.createNode("host2", HostStatus.NO_REMARKS);
-        HostName hostName3 = modelUtils.createNode("host3", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName4 = modelUtils.createNode("host4", HostStatus.ALLOWED_TO_BE_DOWN);
-        HostName hostName5 = modelUtils.createNode("host5", HostStatus.NO_REMARKS);
-
+        HostName hostName1 = new HostName("host1");
+        HostName hostName2 = new HostName("host2");
+        HostName hostName3 = new HostName("host3");
+        HostName hostName4 = new HostName("host4");
+        HostName hostName5 = new HostName("host5");
 
         ServiceCluster serviceCluster = modelUtils.createServiceCluster(
                 "cluster",
@@ -85,6 +91,13 @@ public class ClusterApiImplTest {
                         modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP)
                 )
         );
+        modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster));
+
+        modelUtils.createNode(hostName1, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName2, HostStatus.NO_REMARKS);
+        modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName4, HostStatus.ALLOWED_TO_BE_DOWN);
+        modelUtils.createNode(hostName5, HostStatus.NO_REMARKS);
 
         verifyNoServices(serviceCluster, false, false, hostName1);
         verifyNoServices(serviceCluster, true, false, hostName2);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 9586f92af30..f9a6dc5b237 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.model;
 
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -11,37 +12,58 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.orchestrator.OrchestrationException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.orchestrator.OrchestratorImpl;
+import com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService;
+import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
+import com.yahoo.vespa.orchestrator.config.OrchestratorConfig.Builder;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.yolean.Exceptions;
 
+import java.time.Clock;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+class ModelTestUtils {
 
-public class ModelTestUtils {
-    private final MutableStatusRegistry statusRegistry = mock(MutableStatusRegistry.class);
-    private final ClusterControllerClientFactory clusterControllerClientFactory = mock(ClusterControllerClientFactory.class);
+    private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>();
+    private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
     private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>();
-
-    ModelTestUtils() {
-        when(statusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS);
-    }
+    private final StatusService statusService = new ZookeeperStatusService(new MockCurator());
+    private final Orchestrator orchestrator = new OrchestratorImpl(clusterControllerClientFactory,
+                                                                   statusService,
+                                                                   new OrchestratorConfig(new Builder()),
+                                                                   new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)));
 
     Map<HostName, HostStatus> getHostStatusMap() {
         return hostStatusMap;
     }
 
     HostName createNode(String name, HostStatus hostStatus) {
-        HostName hostName = new HostName(name);
+        return createNode(new HostName(name), hostStatus);
+    }
+
+    HostName createNode(HostName hostName, HostStatus hostStatus) {
         hostStatusMap.put(hostName, hostStatus);
-        when(statusRegistry.getHostStatus(hostName)).thenReturn(hostStatus);
+        try {
+            orchestrator.setNodeStatus(hostName, hostStatus);
+        }
+        catch (OrchestrationException e) {
+            throw new AssertionError("Host '" + hostName + "' not owned by any application — please assign it first: " +
+                                             Exceptions.toMessageString(e));
+        }
         return hostName;
     }
 
@@ -49,26 +71,29 @@ public class ModelTestUtils {
             ApplicationInstance applicationInstance,
             HostName... hostnames) {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostnames);
-        return new ApplicationApiImpl(nodeGroup, statusRegistry, clusterControllerClientFactory);
+        MutableStatusRegistry registry = statusService.lockApplicationInstance_forCurrentThreadOnly(
+                OrchestratorContext.createContextForSingleAppOp(Clock.systemUTC()),
+                applicationInstance.reference());
+        return new ApplicationApiImpl(nodeGroup, registry, statusService, clusterControllerClientFactory);
     }
 
     ApplicationInstance createApplicationInstance(
             List<ServiceCluster> serviceClusters) {
-        Set<ServiceCluster> serviceClusterSet = serviceClusters.stream()
-                .collect(Collectors.toSet());
+        Set<ServiceCluster> serviceClusterSet = new HashSet<>(serviceClusters);
 
-        return new ApplicationInstance(
+        ApplicationInstance application = new ApplicationInstance(
                 new TenantId("tenant"),
                 new ApplicationInstanceId("application-name:foo:bar:default"),
                 serviceClusterSet);
+        applications.put(application.reference(), application);
+        return application;
     }
 
     ServiceCluster createServiceCluster(
             String clusterId,
             ServiceType serviceType,
             List<ServiceInstance> serviceInstances) {
-        Set<ServiceInstance> serviceInstanceSet = serviceInstances.stream()
-                .collect(Collectors.toSet());
+        Set<ServiceInstance> serviceInstanceSet = new HashSet<>(serviceInstances);
 
         return new ServiceCluster(
                 new ClusterId(clusterId),
@@ -86,7 +111,8 @@ public class ModelTestUtils {
                 status);
     }
 
-    public ClusterControllerClientFactory getClusterControllerClientFactory() {
+    ClusterControllerClientFactory getClusterControllerClientFactory() {
         return clusterControllerClientFactory;
     }
+
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index 55ac65c036c..2a8fdc8290c 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -119,7 +119,8 @@ public class HostResourceTest {
         public void releaseSuspensionGrant(
                 OrchestratorContext context, ApplicationInstance applicationInstance,
                 HostName hostName,
-                MutableStatusRegistry hostStatusRegistry) {
+                MutableStatusRegistry hostStatusRegistry,
+                StatusService statusService) {
         }
     }
 
@@ -220,7 +221,8 @@ public class HostResourceTest {
         public void releaseSuspensionGrant(
                 OrchestratorContext context, ApplicationInstance applicationInstance,
                 HostName hostName,
-                MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException {
+                MutableStatusRegistry hostStatusRegistry,
+                StatusService statusService) throws HostStateChangeDeniedException {
             doThrow();
         }