Support monitoring health of tenant hosts

16 files changed, 359 insertions, 58 deletions

diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java
index 7648432dc11..28ee5076191 100644
--- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java
+++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java
@@ -9,11 +9,6 @@ import java.util.Objects;
  * @author bjorncs
  */
 public class ClusterId {
-    // Common cluster IDs
-    public static final ClusterId ADMIN = new ClusterId("admin");
-    public static final ClusterId NODE_ADMIN = new ClusterId("node-admin");
-    public static final ClusterId ROUTING = new ClusterId("routing");
-
     private final String id;
 
     public ClusterId(String id) {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index d564b21d12a..61164017303 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -30,6 +30,12 @@ public class Flags {
             "Takes effect only at bootstrap of config server/controller",
             HOSTNAME);
 
+    public static final UnboundBooleanFlag MONITOR_TENANT_HOST_HEALTH = defineFeatureFlag(
+            "monitor-tenant-hosts-health", false,
+            "Whether service monitor will monitor /state/v1/health of the host admins on the tenant hosts.",
+            "Flag is read when config server starts",
+            HOSTNAME);
+
     public static final UnboundIntFlag DROP_CACHES = defineIntFlag("drop-caches", 3,
             "The int value to write into /proc/sys/vm/drop_caches for each tick. " +
             "1 is page cache, 2 is dentries inodes, 3 is both page cache and dentries inodes, etc.",
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java
index 91759b32086..25479b7b03a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java
@@ -9,9 +9,6 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
  * A service/application model of the config server with health status.
  */
 public class ConfigServerApplication extends ConfigServerLikeApplication {
-
-    public static final ConfigServerApplication CONFIG_SERVER_APPLICATION = new ConfigServerApplication();
-
     public ConfigServerApplication() {
         super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ServiceType.CONFIG_SERVER);
     }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java
index 5e6cb23e9c1..7772f989746 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java
@@ -9,8 +9,10 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
  * @author hakonhall
  */
 public abstract class HostAdminApplication extends InfraApplication {
+    public static final int HOST_ADMIN_HEALT_PORT = 8080;
+
     protected HostAdminApplication(String applicationName, NodeType nodeType) {
         super(applicationName, nodeType, ClusterSpec.Type.container, ClusterSpec.Id.from(applicationName),
-                ServiceType.HOST_ADMIN, 8080);
+                ServiceType.HOST_ADMIN, HOST_ADMIN_HEALT_PORT);
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java
index 65198e72c89..70354c0f16d 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java
@@ -1,9 +1,15 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.duper;
 
+import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
 
 import java.util.Objects;
 
@@ -17,14 +23,84 @@ import java.util.Objects;
 public class ZoneApplication {
     private ZoneApplication() {}
 
-    public static final ApplicationId ZONE_APPLICATION_ID = InfraApplication
+    private static final ApplicationId ZONE_APPLICATION_ID = InfraApplication
             .createHostedVespaApplicationId("routing");
+    private static final ClusterId NODE_ADMIN_CLUSTER_ID = new ClusterId("node-admin");
+    private static final ClusterId ROUTING_CLUSTER_ID = new ClusterId("routing");
+
+    public static ApplicationId getApplicationId() {
+        return ZONE_APPLICATION_ID;
+    }
+
+    public static TenantName getTenantName() {
+        return ZONE_APPLICATION_ID.tenant();
+    }
+
+    public static ApplicationName getApplicationName() {
+        return ZONE_APPLICATION_ID.application();
+    }
+
+    public static NodeType getNodeAdminNodeType() {
+        return NodeType.host;
+    }
+
+    public static ClusterId getNodeAdminClusterId() {
+        return NODE_ADMIN_CLUSTER_ID;
+    }
+
+    public static ClusterSpec.Type getNodeAdminClusterSpecType() {
+        return ClusterSpec.Type.container;
+    }
+
+    public static ClusterSpec.Id getNodeAdminClusterSpecId() {
+        return new ClusterSpec.Id(getNodeAdminClusterId().s());
+    }
+
+    public static ServiceType getNodeAdminServiceType() {
+        return ServiceType.CONTAINER;
+    }
+
+    public static int getNodeAdminHealthPort() {
+        return HostAdminApplication.HOST_ADMIN_HEALT_PORT;
+    }
+
+    public static NodeType getRoutingNodeType() {
+        return NodeType.proxy;
+    }
+
+    public static ClusterId getRoutingClusterId() {
+        return ROUTING_CLUSTER_ID;
+    }
+
+    public static ClusterSpec.Type getRoutingClusterSpecType() {
+        return ClusterSpec.Type.container;
+    }
+
+    public static ClusterSpec.Id getRoutingClusterSpecId() {
+        return new ClusterSpec.Id(getRoutingClusterId().s());
+    }
+
+    public static ServiceType getRoutingServiceType() {
+        return ServiceType.CONTAINER;
+    }
+
+    public static int getRoutingHealthPort() {
+        return 4088;
+    }
 
     public static boolean isNodeAdminService(ApplicationId applicationId,
                                              ClusterId clusterId,
                                              ServiceType serviceType) {
-        return Objects.equals(applicationId, ZONE_APPLICATION_ID) &&
-                Objects.equals(serviceType, ServiceType.CONTAINER) &&
-                Objects.equals(clusterId, ClusterId.NODE_ADMIN);
+        return Objects.equals(applicationId, getApplicationId()) &&
+                Objects.equals(serviceType, getNodeAdminServiceType()) &&
+                Objects.equals(clusterId, getNodeAdminClusterId());
+    }
+
+    /** Whether a {@link ServiceInfo} belongs to the zone application's node-admin cluster. */
+    public static boolean isNodeAdminServiceInfo(ApplicationId applicationId, ServiceInfo serviceInfo) {
+        return isNodeAdminService(
+                applicationId,
+                ApplicationInstanceGenerator.getClusterId(serviceInfo),
+                ApplicationInstanceGenerator.toServiceType(serviceInfo));
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
index 5fab8ac8591..5eac6fbb000 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
@@ -47,11 +47,6 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
     }
 
     @Override
-    public boolean wouldMonitor(ApplicationId applicationId) {
-        return true;
-    }
-
-    @Override
     public ServiceStatus getStatus(ApplicationId applicationId,
                                    ClusterId clusterId,
                                    ServiceType serviceType,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 71938a9f1dc..ddfbdf59b3c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.RunletExecutorImpl;
@@ -47,20 +48,34 @@ public class HealthMonitorManager implements MonitorManager {
 
     private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>();
     private final DuperModelManager duperModel;
+    private final boolean monitorTenantHostHealth;
     private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory;
 
     @Inject
     public HealthMonitorManager(DuperModelManager duperModel, FlagSource flagSource) {
-        this(duperModel, new StateV1HealthModel(
-                TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE)));
+        this(duperModel, Flags.MONITOR_TENANT_HOST_HEALTH.bindTo(flagSource).value());
     }
 
-    private HealthMonitorManager(DuperModelManager duperModel, StateV1HealthModel healthModel) {
-        this(duperModel, id -> new ApplicationHealthMonitor(id, healthModel));
+    private HealthMonitorManager(DuperModelManager duperModel, boolean monitorTenantHostHealth) {
+        this(duperModel, monitorTenantHostHealth,
+                new StateV1HealthModel(
+                        TARGET_HEALTH_STALENESS,
+                        HEALTH_REQUEST_TIMEOUT,
+                        KEEP_ALIVE,
+                        new RunletExecutorImpl(THREAD_POOL_SIZE),
+                        monitorTenantHostHealth));
     }
 
-    HealthMonitorManager(DuperModelManager duperModel, ApplicationHealthMonitorFactory applicationHealthMonitorFactory) {
+    private HealthMonitorManager(DuperModelManager duperModel, boolean monitorTenantHostHealth, StateV1HealthModel healthModel) {
+        this(duperModel, monitorTenantHostHealth, id -> new ApplicationHealthMonitor(id, healthModel));
+    }
+
+    /** Default access due to testing. */
+    HealthMonitorManager(DuperModelManager duperModel,
+                         boolean monitorTenantHostHealth,
+                         ApplicationHealthMonitorFactory applicationHealthMonitorFactory) {
         this.duperModel = duperModel;
+        this.monitorTenantHostHealth = monitorTenantHostHealth;
         this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
     }
 
@@ -86,22 +101,41 @@ public class HealthMonitorManager implements MonitorManager {
                                    ClusterId clusterId,
                                    ServiceType serviceType,
                                    ConfigId configId) {
-        if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) {
-            // If node admin doesn't run in a JDisc container, it must be monitored with health.
-            // TODO: Do proper health check
+        ApplicationHealthMonitor monitor = healthMonitors.get(applicationId);
+
+        if (!monitorTenantHostHealth && ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) {
+            // Legacy: The zone app is not health monitored (monitor == null), but the node-admin cluster's services
+            // are hard-coded to be UP
             return ServiceStatus.UP;
         }
 
-        ApplicationHealthMonitor monitor = healthMonitors.get(applicationId);
         if (monitor == null) {
             return ServiceStatus.NOT_CHECKED;
         }
 
+        if (monitorTenantHostHealth && applicationId.equals(ZoneApplication.getApplicationId())) {
+            // New: The zone app is health monitored (monitor != null), possibly even the routing cluster
+            // which is a normal jdisc container (unnecessary but harmless), but the node-admin cluster
+            // are tenant Docker hosts running host admin that are monitored via /state/v1/health.
+            if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) {
+                return monitor.getStatus(applicationId, clusterId, serviceType, configId);
+            } else {
+                return ServiceStatus.NOT_CHECKED;
+            }
+        }
+
         return monitor.getStatus(applicationId, clusterId, serviceType, configId);
     }
 
-    @Override
-    public boolean wouldMonitor(ApplicationId id) {
-        return duperModel.isSupportedInfraApplication(id);
+    private boolean wouldMonitor(ApplicationId id) {
+        if (duperModel.isSupportedInfraApplication(id)) {
+            return true;
+        }
+
+        if (monitorTenantHostHealth && id.equals(ZoneApplication.getApplicationId())) {
+            return true;
+        }
+
+        return false;
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
index 5e8979deb9f..04943f81478 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
@@ -6,6 +6,7 @@ import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.RunletExecutor;
 import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
 import com.yahoo.vespa.service.model.ServiceId;
@@ -29,23 +30,36 @@ public class StateV1HealthModel implements AutoCloseable {
     private final Duration requestTimeout;
     private final Duration connectionKeepAlive;
     private final RunletExecutor executor;
+    private final boolean monitorTenantHostHealth;
 
     StateV1HealthModel(Duration targetHealthStaleness,
                        Duration requestTimeout,
                        Duration connectionKeepAlive,
-                       RunletExecutor executor) {
+                       RunletExecutor executor,
+                       boolean monitorTenantHostHealth) {
         this.targetHealthStaleness = targetHealthStaleness;
         this.requestTimeout = requestTimeout;
         this.connectionKeepAlive = connectionKeepAlive;
         this.executor = executor;
+        this.monitorTenantHostHealth = monitorTenantHostHealth;
     }
 
     Map<ServiceId, HealthEndpoint> extractHealthEndpoints(ApplicationInfo application) {
         Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
 
+        boolean isZoneApplication = application.getApplicationId().equals(ZoneApplication.getApplicationId());
+
         for (HostInfo hostInfo : application.getModel().getHosts()) {
             HostName hostname = HostName.from(hostInfo.getHostname());
             for (ServiceInfo serviceInfo : hostInfo.getServices()) {
+
+                if (monitorTenantHostHealth && isZoneApplication &&
+                        !ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) {
+                    // Only the node admin/host admin cluster of the zone application should be monitored
+                    // TODO: Move the node admin cluster out to a separate infrastructure application
+                    continue;
+                }
+
                 ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo);
                 for (PortInfo portInfo : serviceInfo.getPorts()) {
                     if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
index eacaf820f3d..cfd9269d9c4 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
@@ -38,11 +38,6 @@ public class UnionMonitorManager implements MonitorManager {
     }
 
     @Override
-    public boolean wouldMonitor(ApplicationId id) {
-        return healthMonitorManager.wouldMonitor(id) || slobrokMonitorManager.wouldMonitor(id);
-    }
-
-    @Override
     public void applicationActivated(ApplicationInfo application) {
         slobrokMonitorManager.applicationActivated(application);
         healthMonitorManager.applicationActivated(application);
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
index f15101d4439..3ca9446df26 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
@@ -131,7 +131,7 @@ public class ApplicationInstanceGenerator {
                 toConfigId(serviceInfo));
     }
 
-    private static ClusterId getClusterId(ServiceInfo serviceInfo) {
+    public static ClusterId getClusterId(ServiceInfo serviceInfo) {
         return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse(""));
     }
 
@@ -141,7 +141,7 @@ public class ApplicationInstanceGenerator {
         return new ServiceClusterKey(clusterId, serviceType);
     }
 
-    private static ServiceType toServiceType(ServiceInfo serviceInfo) {
+    public static ServiceType toServiceType(ServiceInfo serviceInfo) {
         return new ServiceType(serviceInfo.getServiceType());
     }
 
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java
index 848cf68c48b..88c72a7d47a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java
@@ -22,7 +22,4 @@ public interface ServiceStatusProvider {
                             ClusterId clusterId,
                             ServiceType serviceType,
                             ConfigId configId);
-
-    /** Returns true if the status provider would start monitoring the application. */
-    boolean wouldMonitor(ApplicationId applicationId);
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index 13f24bc3694..0f3a3cfbe68 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -108,8 +108,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
         }
     }
 
-    @Override
-    public boolean wouldMonitor(ApplicationId applicationId) {
+    private boolean wouldMonitor(ApplicationId applicationId) {
         if (duperModel.isSupportedInfraApplication(applicationId)) {
             return false;
         }
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java
new file mode 100644
index 00000000000..773643c1d09
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java
@@ -0,0 +1,90 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.duper;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.HostName;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author hakonhall
+ */
+public class TestZoneApplication {
+
+    private final List<HostName> nodeAdminHostnames;
+    private final List<HostName> routingHostnames;
+
+    private TestZoneApplication(List<HostName> nodeAdminHostnames, List<HostName> routingHostnames) {
+        this.nodeAdminHostnames = nodeAdminHostnames;
+        this.routingHostnames = routingHostnames;
+    }
+
+    public ApplicationInfo makeApplicationInfo() {
+        // Make a test ApplicationInfo by:
+        //  1. Make an ApplicationInfo as-if the node-admin cluster of the zone application were the only cluster.
+        //     Make sure to get the correct tenant name, application name, cluster id, service type, hostnames,
+        //     services, and ports. This should be easy with the help of InfraApplication.
+        ApplicationInfo nodeAdminPart = new NodeAdminPartOfZoneApplication().makeApplicationInfo(nodeAdminHostnames);
+
+        //  2. Make an ApplicationInfo as-if the routing cluster of the zone application were the only cluster.
+        //     Don't care if the application is not perfect.
+        ApplicationInfo routingPart = new RoutingPartOfZoneApplication().makeApplicationInfo(routingHostnames);
+
+        //  3. Take HostInfo from (1) and (2) to make a single ApplicationInfo.
+        List<HostInfo> allHostInfos = new ArrayList<>();
+        allHostInfos.addAll(nodeAdminPart.getModel().getHosts());
+        allHostInfos.addAll(routingPart.getModel().getHosts());
+
+        return new ApplicationInfo(nodeAdminPart.getApplicationId(), 0, new HostsModel(allHostInfos));
+    }
+
+    public static class Builder {
+        private List<HostName> nodeAdminHostnames = null;
+        private List<HostName> routingHostnames = null;
+
+        public Builder addNodeAdminCluster(String... hostnames) {
+            this.nodeAdminHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList());
+            return this;
+        }
+
+        public Builder addRoutingCluster(String... hostnames) {
+            this.routingHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList());
+            return this;
+        }
+
+        public TestZoneApplication build() {
+            return new TestZoneApplication(Objects.requireNonNull(nodeAdminHostnames), Objects.requireNonNull(routingHostnames));
+        }
+    }
+
+    private static class NodeAdminPartOfZoneApplication extends InfraApplication {
+        public NodeAdminPartOfZoneApplication() {
+            super(ZoneApplication.getApplicationName().value(),
+                    ZoneApplication.getNodeAdminNodeType(),
+                    ZoneApplication.getNodeAdminClusterSpecType(),
+                    ZoneApplication.getNodeAdminClusterSpecId(),
+                    ZoneApplication.getNodeAdminServiceType(),
+                    ZoneApplication.getNodeAdminHealthPort());
+        }
+    }
+
+    /**
+     * This InfraApplication is bogus (containing host admin instead of jdisc container), but the tests are
+     * not supposed to explore this cluster.
+     */
+    private static class RoutingPartOfZoneApplication extends InfraApplication {
+        public RoutingPartOfZoneApplication() {
+            super(ZoneApplication.getApplicationName().value(),
+                    ZoneApplication.getRoutingNodeType(),
+                    ZoneApplication.getRoutingClusterSpecType(),
+                    ZoneApplication.getRoutingClusterSpecId(),
+                    ZoneApplication.getRoutingServiceType(),
+                    ZoneApplication.getRoutingHealthPort());
+        }
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
index 061807ba76f..6e18bebf791 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
@@ -3,18 +3,16 @@ package com.yahoo.vespa.service.health;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
-import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
 import com.yahoo.vespa.service.duper.ControllerHostApplication;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.InfraApplication;
 import com.yahoo.vespa.service.duper.ProxyHostApplication;
+import com.yahoo.vespa.service.duper.TestZoneApplication;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.monitor.ConfigserverUtil;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.util.List;
@@ -23,6 +21,7 @@ import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -33,16 +32,17 @@ public class HealthMonitorManagerTest {
     private final DuperModelManager duperModel = mock(DuperModelManager.class);
     private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class);
     private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class);
-    private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorFactory);
+    private HealthMonitorManager manager;
 
-    @Before
-    public void setUp() {
+    public void setUp(boolean monitorTenantHostHealth) {
+        manager = new HealthMonitorManager(duperModel, monitorTenantHostHealth, monitorFactory);
         when(duperModel.getConfigServerApplication()).thenReturn(configServerApplication);
         when(monitorFactory.create(any())).thenReturn(monitor);
     }
 
     @Test
     public void addAndRemove() {
+        setUp(false);
         ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer();
         when(duperModel.isSupportedInfraApplication(applicationInfo.getApplicationId())).thenReturn(true);
 
@@ -57,16 +57,90 @@ public class HealthMonitorManagerTest {
 
     @Test
     public void withHostAdmin() {
+        setUp(false);
         ServiceStatus status = manager.getStatus(
-                ZoneApplication.ZONE_APPLICATION_ID,
-                ClusterId.NODE_ADMIN,
-                ServiceType.CONTAINER,
+                ZoneApplication.getApplicationId(),
+                ZoneApplication.getNodeAdminClusterId(),
+                ZoneApplication.getNodeAdminServiceType(),
                 new ConfigId("config-id-1"));
         assertEquals(ServiceStatus.UP, status);
     }
 
     @Test
+    public void verifyZoneApplicationIsNotMonitoredByDefault() {
+        verifyZoneApplicationIsMonitored(false, false);
+    }
+
+    @Test
+    public void verifyZoneApplicationIsMonitored() {
+        verifyZoneApplicationIsMonitored(true, true);
+    }
+
+    private void verifyZoneApplicationIsMonitored(boolean monitorTenantHostHealth, boolean isMonitored) {
+        setUp(monitorTenantHostHealth);
+
+        ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder()
+                .addNodeAdminCluster("h1", "h2")
+                .addRoutingCluster("r1")
+                .build()
+                .makeApplicationInfo();
+
+        verify(monitorFactory, times(0)).create(zoneApplicationInfo.getApplicationId());
+        verify(monitor, times(0)).monitor(any());
+        manager.applicationActivated(zoneApplicationInfo);
+        verify(monitorFactory, times(isMonitored ? 1 : 0)).create(zoneApplicationInfo.getApplicationId());
+        verify(monitor, times(isMonitored ? 1 : 0)).monitor(any());
+
+        when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.DOWN);
+        verifyNodeAdminGetStatus(0);
+        if (isMonitored) {
+            assertEquals(ServiceStatus.DOWN, getNodeAdminStatus());
+            verifyNodeAdminGetStatus(1);
+        } else {
+            assertEquals(ServiceStatus.UP, getNodeAdminStatus());
+            verifyNodeAdminGetStatus(0);
+        }
+
+        verifyRoutingGetStatus(0);
+        assertEquals(ServiceStatus.NOT_CHECKED, getRoutingStatus());
+        verifyRoutingGetStatus(0);
+    }
+
+    private void verifyNodeAdminGetStatus(int invocations) {
+        verify(monitor, times(invocations)).getStatus(
+                eq(ZoneApplication.getApplicationId()),
+                eq(ZoneApplication.getNodeAdminClusterId()),
+                any(),
+                any());
+    }
+
+    private void verifyRoutingGetStatus(int invocations) {
+        verify(monitor, times(invocations)).getStatus(
+                eq(ZoneApplication.getApplicationId()),
+                eq(ZoneApplication.getRoutingClusterId()),
+                any(),
+                any());
+    }
+
+    private ServiceStatus getNodeAdminStatus() {
+        return manager.getStatus(
+                ZoneApplication.getApplicationId(),
+                ZoneApplication.getNodeAdminClusterId(),
+                ZoneApplication.getNodeAdminServiceType(),
+                new ConfigId("foo"));
+    }
+
+    private ServiceStatus getRoutingStatus() {
+        return manager.getStatus(
+                ZoneApplication.getApplicationId(),
+                ZoneApplication.getRoutingClusterId(),
+                ZoneApplication.getRoutingServiceType(),
+                new ConfigId("bar"));
+    }
+
+    @Test
     public void infrastructureApplication() {
+        setUp(false);
         ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
         when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true);
         List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList());
@@ -85,6 +159,7 @@ public class HealthMonitorManagerTest {
 
     @Test
     public void threadPoolSize() {
+        setUp(false);
         assertEquals(9, HealthMonitorManager.THREAD_POOL_SIZE);
     }
 
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
index 480691772bb..e38589b6f81 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
@@ -9,6 +9,8 @@ import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.service.duper.ProxyHostApplication;
+import com.yahoo.vespa.service.duper.TestZoneApplication;
+import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.Cancellable;
 import com.yahoo.vespa.service.executor.RunletExecutor;
 import com.yahoo.vespa.service.model.ServiceId;
@@ -34,14 +36,18 @@ public class StateV1HealthModelTest {
     private Duration healthStaleness = Duration.ofSeconds(1);
     private Duration requestTimeout = Duration.ofSeconds(2);
     private Duration keepAlive = Duration.ofSeconds(3);
-    private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor);
     private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
     private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::from).collect(Collectors.toList());
     private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
-    private final Map<ServiceId, HealthEndpoint> endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo);
+
+    private StateV1HealthModel model;
+    private Map<ServiceId, HealthEndpoint> endpoints;
 
     @Test
     public void test() {
+        model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor, false);
+        endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo);
+
         assertEquals(2, endpoints.size());
 
         ApplicationId applicationId = ApplicationId.from("hosted-vespa", "proxy-host", "default");
@@ -63,4 +69,23 @@ public class StateV1HealthModelTest {
             assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus());
         }
     }
+
+    @Test
+    public void testMonitoringTenantHostHealth() {
+        model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor, true);
+        ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder()
+                .addNodeAdminCluster("h1")
+                .addRoutingCluster("r1")
+                .build()
+                .makeApplicationInfo();
+
+        endpoints = model.extractHealthEndpoints(zoneApplicationInfo);
+        assertEquals(1, endpoints.size());
+        HealthEndpoint endpoint = endpoints.values().iterator().next();
+        assertEquals("http://h1:8080/state/v1/health", endpoint.description());
+        ServiceId serviceId = endpoint.getServiceId();
+        assertEquals(ZoneApplication.getApplicationId(), serviceId.getApplicationId());
+        assertEquals(ZoneApplication.getNodeAdminClusterId(), serviceId.getClusterId());
+        assertEquals(ZoneApplication.getNodeAdminServiceType(), serviceId.getServiceType());
+    }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java
index a0c7aca5869..b97fd6c64a5 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java
@@ -3,16 +3,14 @@ package com.yahoo.vespa.service.manager;
 
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.health.HealthMonitorManager;
 import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl;
 import org.junit.Test;
 
-import static com.yahoo.vespa.applicationmodel.ClusterId.NODE_ADMIN;
 import static com.yahoo.vespa.applicationmodel.ServiceStatus.DOWN;
 import static com.yahoo.vespa.applicationmodel.ServiceStatus.NOT_CHECKED;
 import static com.yahoo.vespa.applicationmodel.ServiceStatus.UP;
-import static com.yahoo.vespa.applicationmodel.ServiceType.CONTAINER;
-import static com.yahoo.vespa.service.duper.ZoneApplication.ZONE_APPLICATION_ID;
 import static org.junit.Assert.assertSame;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -38,7 +36,10 @@ public class UnionMonitorManagerTest {
                           ServiceStatus expectedStatus) {
         when(healthMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(healthStatus);
         when(slobrokMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(slobrokStatus);
-        ServiceStatus status = manager.getStatus(ZONE_APPLICATION_ID, NODE_ADMIN, CONTAINER, new ConfigId("config-id"));
+        ServiceStatus status = manager.getStatus(
+                ZoneApplication.getApplicationId(),
+                ZoneApplication.getNodeAdminClusterId(),
+                ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id"));
         assertSame(expectedStatus, status);
     }
 }
\ No newline at end of file