From c8aa5a5ba53c24206f390a1eee39ec3e704a5fc4 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Wed, 16 Jan 2019 08:14:14 +0100 Subject: Support monitoring health of tenant hosts --- .../yahoo/vespa/applicationmodel/ClusterId.java | 5 -- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++ .../service/duper/ConfigServerApplication.java | 3 - .../vespa/service/duper/HostAdminApplication.java | 4 +- .../yahoo/vespa/service/duper/ZoneApplication.java | 84 ++++++++++++++++++- .../service/health/ApplicationHealthMonitor.java | 5 -- .../vespa/service/health/HealthMonitorManager.java | 58 +++++++++++--- .../vespa/service/health/StateV1HealthModel.java | 16 +++- .../vespa/service/manager/UnionMonitorManager.java | 5 -- .../model/ApplicationInstanceGenerator.java | 4 +- .../service/monitor/ServiceStatusProvider.java | 3 - .../service/slobrok/SlobrokMonitorManagerImpl.java | 3 +- .../vespa/service/duper/TestZoneApplication.java | 90 +++++++++++++++++++++ .../service/health/HealthMonitorManagerTest.java | 93 +++++++++++++++++++--- .../service/health/StateV1HealthModelTest.java | 29 ++++++- .../service/manager/UnionMonitorManagerTest.java | 9 ++- 16 files changed, 359 insertions(+), 58 deletions(-) create mode 100644 service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java index 7648432dc11..28ee5076191 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java @@ -9,11 +9,6 @@ import java.util.Objects; * @author bjorncs */ public class ClusterId { - // Common cluster IDs - public static final ClusterId ADMIN = new ClusterId("admin"); - public static final ClusterId NODE_ADMIN = new ClusterId("node-admin"); - public static final ClusterId ROUTING = new ClusterId("routing"); - private final String id; public ClusterId(String id) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d564b21d12a..61164017303 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -30,6 +30,12 @@ public class Flags { "Takes effect only at bootstrap of config server/controller", HOSTNAME); + public static final UnboundBooleanFlag MONITOR_TENANT_HOST_HEALTH = defineFeatureFlag( + "monitor-tenant-hosts-health", false, + "Whether service monitor will monitor /state/v1/health of the host admins on the tenant hosts.", + "Flag is read when config server starts", + HOSTNAME); + public static final UnboundIntFlag DROP_CACHES = defineIntFlag("drop-caches", 3, "The int value to write into /proc/sys/vm/drop_caches for each tick. " + "1 is page cache, 2 is dentries inodes, 3 is both page cache and dentries inodes, etc.", diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java index 91759b32086..25479b7b03a 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java @@ -9,9 +9,6 @@ import com.yahoo.vespa.applicationmodel.ServiceType; * A service/application model of the config server with health status. */ public class ConfigServerApplication extends ConfigServerLikeApplication { - - public static final ConfigServerApplication CONFIG_SERVER_APPLICATION = new ConfigServerApplication(); - public ConfigServerApplication() { super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ServiceType.CONFIG_SERVER); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java index 5e6cb23e9c1..7772f989746 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java @@ -9,8 +9,10 @@ import com.yahoo.vespa.applicationmodel.ServiceType; * @author hakonhall */ public abstract class HostAdminApplication extends InfraApplication { + public static final int HOST_ADMIN_HEALT_PORT = 8080; + protected HostAdminApplication(String applicationName, NodeType nodeType) { super(applicationName, nodeType, ClusterSpec.Type.container, ClusterSpec.Id.from(applicationName), - ServiceType.HOST_ADMIN, 8080); + ServiceType.HOST_ADMIN, HOST_ADMIN_HEALT_PORT); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java index 65198e72c89..70354c0f16d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java @@ -1,9 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import java.util.Objects; @@ -17,14 +23,84 @@ import java.util.Objects; public class ZoneApplication { private ZoneApplication() {} - public static final ApplicationId ZONE_APPLICATION_ID = InfraApplication + private static final ApplicationId ZONE_APPLICATION_ID = InfraApplication .createHostedVespaApplicationId("routing"); + private static final ClusterId NODE_ADMIN_CLUSTER_ID = new ClusterId("node-admin"); + private static final ClusterId ROUTING_CLUSTER_ID = new ClusterId("routing"); + + public static ApplicationId getApplicationId() { + return ZONE_APPLICATION_ID; + } + + public static TenantName getTenantName() { + return ZONE_APPLICATION_ID.tenant(); + } + + public static ApplicationName getApplicationName() { + return ZONE_APPLICATION_ID.application(); + } + + public static NodeType getNodeAdminNodeType() { + return NodeType.host; + } + + public static ClusterId getNodeAdminClusterId() { + return NODE_ADMIN_CLUSTER_ID; + } + + public static ClusterSpec.Type getNodeAdminClusterSpecType() { + return ClusterSpec.Type.container; + } + + public static ClusterSpec.Id getNodeAdminClusterSpecId() { + return new ClusterSpec.Id(getNodeAdminClusterId().s()); + } + + public static ServiceType getNodeAdminServiceType() { + return ServiceType.CONTAINER; + } + + public static int getNodeAdminHealthPort() { + return HostAdminApplication.HOST_ADMIN_HEALT_PORT; + } + + public static NodeType getRoutingNodeType() { + return NodeType.proxy; + } + + public static ClusterId getRoutingClusterId() { + return ROUTING_CLUSTER_ID; + } + + public static ClusterSpec.Type getRoutingClusterSpecType() { + return ClusterSpec.Type.container; + } + + public static ClusterSpec.Id getRoutingClusterSpecId() { + return new ClusterSpec.Id(getRoutingClusterId().s()); + } + + public static ServiceType getRoutingServiceType() { + return ServiceType.CONTAINER; + } + + public static int getRoutingHealthPort() { + return 4088; + } public static boolean isNodeAdminService(ApplicationId applicationId, ClusterId clusterId, ServiceType serviceType) { - return Objects.equals(applicationId, ZONE_APPLICATION_ID) && - Objects.equals(serviceType, ServiceType.CONTAINER) && - Objects.equals(clusterId, ClusterId.NODE_ADMIN); + return Objects.equals(applicationId, getApplicationId()) && + Objects.equals(serviceType, getNodeAdminServiceType()) && + Objects.equals(clusterId, getNodeAdminClusterId()); + } + + /** Whether a {@link ServiceInfo} belongs to the zone application's node-admin cluster. */ + public static boolean isNodeAdminServiceInfo(ApplicationId applicationId, ServiceInfo serviceInfo) { + return isNodeAdminService( + applicationId, + ApplicationInstanceGenerator.getClusterId(serviceInfo), + ApplicationInstanceGenerator.toServiceType(serviceInfo)); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index 5fab8ac8591..5eac6fbb000 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -46,11 +46,6 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring())); } - @Override - public boolean wouldMonitor(ApplicationId applicationId) { - return true; - } - @Override public ServiceStatus getStatus(ApplicationId applicationId, ClusterId clusterId, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 71938a9f1dc..ddfbdf59b3c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutorImpl; @@ -47,20 +48,34 @@ public class HealthMonitorManager implements MonitorManager { private final ConcurrentHashMap healthMonitors = new ConcurrentHashMap<>(); private final DuperModelManager duperModel; + private final boolean monitorTenantHostHealth; private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory; @Inject public HealthMonitorManager(DuperModelManager duperModel, FlagSource flagSource) { - this(duperModel, new StateV1HealthModel( - TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE))); + this(duperModel, Flags.MONITOR_TENANT_HOST_HEALTH.bindTo(flagSource).value()); } - private HealthMonitorManager(DuperModelManager duperModel, StateV1HealthModel healthModel) { - this(duperModel, id -> new ApplicationHealthMonitor(id, healthModel)); + private HealthMonitorManager(DuperModelManager duperModel, boolean monitorTenantHostHealth) { + this(duperModel, monitorTenantHostHealth, + new StateV1HealthModel( + TARGET_HEALTH_STALENESS, + HEALTH_REQUEST_TIMEOUT, + KEEP_ALIVE, + new RunletExecutorImpl(THREAD_POOL_SIZE), + monitorTenantHostHealth)); } - HealthMonitorManager(DuperModelManager duperModel, ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { + private HealthMonitorManager(DuperModelManager duperModel, boolean monitorTenantHostHealth, StateV1HealthModel healthModel) { + this(duperModel, monitorTenantHostHealth, id -> new ApplicationHealthMonitor(id, healthModel)); + } + + /** Default access due to testing. */ + HealthMonitorManager(DuperModelManager duperModel, + boolean monitorTenantHostHealth, + ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { this.duperModel = duperModel; + this.monitorTenantHostHealth = monitorTenantHostHealth; this.applicationHealthMonitorFactory = applicationHealthMonitorFactory; } @@ -86,22 +101,41 @@ public class HealthMonitorManager implements MonitorManager { ClusterId clusterId, ServiceType serviceType, ConfigId configId) { - if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { - // If node admin doesn't run in a JDisc container, it must be monitored with health. - // TODO: Do proper health check + ApplicationHealthMonitor monitor = healthMonitors.get(applicationId); + + if (!monitorTenantHostHealth && ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { + // Legacy: The zone app is not health monitored (monitor == null), but the node-admin cluster's services + // are hard-coded to be UP return ServiceStatus.UP; } - ApplicationHealthMonitor monitor = healthMonitors.get(applicationId); if (monitor == null) { return ServiceStatus.NOT_CHECKED; } + if (monitorTenantHostHealth && applicationId.equals(ZoneApplication.getApplicationId())) { + // New: The zone app is health monitored (monitor != null), possibly even the routing cluster + // which is a normal jdisc container (unnecessary but harmless), but the node-admin cluster + // are tenant Docker hosts running host admin that are monitored via /state/v1/health. + if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { + return monitor.getStatus(applicationId, clusterId, serviceType, configId); + } else { + return ServiceStatus.NOT_CHECKED; + } + } + return monitor.getStatus(applicationId, clusterId, serviceType, configId); } - @Override - public boolean wouldMonitor(ApplicationId id) { - return duperModel.isSupportedInfraApplication(id); + private boolean wouldMonitor(ApplicationId id) { + if (duperModel.isSupportedInfraApplication(id)) { + return true; + } + + if (monitorTenantHostHealth && id.equals(ZoneApplication.getApplicationId())) { + return true; + } + + return false; } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index 5e8979deb9f..04943f81478 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.model.ServiceId; @@ -29,23 +30,36 @@ public class StateV1HealthModel implements AutoCloseable { private final Duration requestTimeout; private final Duration connectionKeepAlive; private final RunletExecutor executor; + private final boolean monitorTenantHostHealth; StateV1HealthModel(Duration targetHealthStaleness, Duration requestTimeout, Duration connectionKeepAlive, - RunletExecutor executor) { + RunletExecutor executor, + boolean monitorTenantHostHealth) { this.targetHealthStaleness = targetHealthStaleness; this.requestTimeout = requestTimeout; this.connectionKeepAlive = connectionKeepAlive; this.executor = executor; + this.monitorTenantHostHealth = monitorTenantHostHealth; } Map extractHealthEndpoints(ApplicationInfo application) { Map endpoints = new HashMap<>(); + boolean isZoneApplication = application.getApplicationId().equals(ZoneApplication.getApplicationId()); + for (HostInfo hostInfo : application.getModel().getHosts()) { HostName hostname = HostName.from(hostInfo.getHostname()); for (ServiceInfo serviceInfo : hostInfo.getServices()) { + + if (monitorTenantHostHealth && isZoneApplication && + !ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) { + // Only the node admin/host admin cluster of the zone application should be monitored + // TODO: Move the node admin cluster out to a separate infrastructure application + continue; + } + ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java index eacaf820f3d..cfd9269d9c4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java @@ -37,11 +37,6 @@ public class UnionMonitorManager implements MonitorManager { return slobrokMonitorManager.getStatus(applicationId, clusterId, serviceType, configId); } - @Override - public boolean wouldMonitor(ApplicationId id) { - return healthMonitorManager.wouldMonitor(id) || slobrokMonitorManager.wouldMonitor(id); - } - @Override public void applicationActivated(ApplicationInfo application) { slobrokMonitorManager.applicationActivated(application); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index f15101d4439..3ca9446df26 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -131,7 +131,7 @@ public class ApplicationInstanceGenerator { toConfigId(serviceInfo)); } - private static ClusterId getClusterId(ServiceInfo serviceInfo) { + public static ClusterId getClusterId(ServiceInfo serviceInfo) { return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); } @@ -141,7 +141,7 @@ public class ApplicationInstanceGenerator { return new ServiceClusterKey(clusterId, serviceType); } - private static ServiceType toServiceType(ServiceInfo serviceInfo) { + public static ServiceType toServiceType(ServiceInfo serviceInfo) { return new ServiceType(serviceInfo.getServiceType()); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java index 848cf68c48b..88c72a7d47a 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java @@ -22,7 +22,4 @@ public interface ServiceStatusProvider { ClusterId clusterId, ServiceType serviceType, ConfigId configId); - - /** Returns true if the status provider would start monitoring the application. */ - boolean wouldMonitor(ApplicationId applicationId); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 13f24bc3694..0f3a3cfbe68 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -108,8 +108,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { } } - @Override - public boolean wouldMonitor(ApplicationId applicationId) { + private boolean wouldMonitor(ApplicationId applicationId) { if (duperModel.isSupportedInfraApplication(applicationId)) { return false; } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java new file mode 100644 index 00000000000..773643c1d09 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java @@ -0,0 +1,90 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.HostName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author hakonhall + */ +public class TestZoneApplication { + + private final List nodeAdminHostnames; + private final List routingHostnames; + + private TestZoneApplication(List nodeAdminHostnames, List routingHostnames) { + this.nodeAdminHostnames = nodeAdminHostnames; + this.routingHostnames = routingHostnames; + } + + public ApplicationInfo makeApplicationInfo() { + // Make a test ApplicationInfo by: + // 1. Make an ApplicationInfo as-if the node-admin cluster of the zone application were the only cluster. + // Make sure to get the correct tenant name, application name, cluster id, service type, hostnames, + // services, and ports. This should be easy with the help of InfraApplication. + ApplicationInfo nodeAdminPart = new NodeAdminPartOfZoneApplication().makeApplicationInfo(nodeAdminHostnames); + + // 2. Make an ApplicationInfo as-if the routing cluster of the zone application were the only cluster. + // Don't care if the application is not perfect. + ApplicationInfo routingPart = new RoutingPartOfZoneApplication().makeApplicationInfo(routingHostnames); + + // 3. Take HostInfo from (1) and (2) to make a single ApplicationInfo. + List allHostInfos = new ArrayList<>(); + allHostInfos.addAll(nodeAdminPart.getModel().getHosts()); + allHostInfos.addAll(routingPart.getModel().getHosts()); + + return new ApplicationInfo(nodeAdminPart.getApplicationId(), 0, new HostsModel(allHostInfos)); + } + + public static class Builder { + private List nodeAdminHostnames = null; + private List routingHostnames = null; + + public Builder addNodeAdminCluster(String... hostnames) { + this.nodeAdminHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()); + return this; + } + + public Builder addRoutingCluster(String... hostnames) { + this.routingHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()); + return this; + } + + public TestZoneApplication build() { + return new TestZoneApplication(Objects.requireNonNull(nodeAdminHostnames), Objects.requireNonNull(routingHostnames)); + } + } + + private static class NodeAdminPartOfZoneApplication extends InfraApplication { + public NodeAdminPartOfZoneApplication() { + super(ZoneApplication.getApplicationName().value(), + ZoneApplication.getNodeAdminNodeType(), + ZoneApplication.getNodeAdminClusterSpecType(), + ZoneApplication.getNodeAdminClusterSpecId(), + ZoneApplication.getNodeAdminServiceType(), + ZoneApplication.getNodeAdminHealthPort()); + } + } + + /** + * This InfraApplication is bogus (containing host admin instead of jdisc container), but the tests are + * not supposed to explore this cluster. + */ + private static class RoutingPartOfZoneApplication extends InfraApplication { + public RoutingPartOfZoneApplication() { + super(ZoneApplication.getApplicationName().value(), + ZoneApplication.getRoutingNodeType(), + ZoneApplication.getRoutingClusterSpecType(), + ZoneApplication.getRoutingClusterSpecId(), + ZoneApplication.getRoutingServiceType(), + ZoneApplication.getRoutingHealthPort()); + } + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 061807ba76f..6e18bebf791 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -3,18 +3,16 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.ControllerHostApplication; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.InfraApplication; import com.yahoo.vespa.service.duper.ProxyHostApplication; +import com.yahoo.vespa.service.duper.TestZoneApplication; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; -import org.junit.Before; import org.junit.Test; import java.util.List; @@ -23,6 +21,7 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -33,16 +32,17 @@ public class HealthMonitorManagerTest { private final DuperModelManager duperModel = mock(DuperModelManager.class); private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class); private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class); - private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorFactory); + private HealthMonitorManager manager; - @Before - public void setUp() { + public void setUp(boolean monitorTenantHostHealth) { + manager = new HealthMonitorManager(duperModel, monitorTenantHostHealth, monitorFactory); when(duperModel.getConfigServerApplication()).thenReturn(configServerApplication); when(monitorFactory.create(any())).thenReturn(monitor); } @Test public void addAndRemove() { + setUp(false); ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer(); when(duperModel.isSupportedInfraApplication(applicationInfo.getApplicationId())).thenReturn(true); @@ -57,16 +57,90 @@ public class HealthMonitorManagerTest { @Test public void withHostAdmin() { + setUp(false); ServiceStatus status = manager.getStatus( - ZoneApplication.ZONE_APPLICATION_ID, - ClusterId.NODE_ADMIN, - ServiceType.CONTAINER, + ZoneApplication.getApplicationId(), + ZoneApplication.getNodeAdminClusterId(), + ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id-1")); assertEquals(ServiceStatus.UP, status); } + @Test + public void verifyZoneApplicationIsNotMonitoredByDefault() { + verifyZoneApplicationIsMonitored(false, false); + } + + @Test + public void verifyZoneApplicationIsMonitored() { + verifyZoneApplicationIsMonitored(true, true); + } + + private void verifyZoneApplicationIsMonitored(boolean monitorTenantHostHealth, boolean isMonitored) { + setUp(monitorTenantHostHealth); + + ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder() + .addNodeAdminCluster("h1", "h2") + .addRoutingCluster("r1") + .build() + .makeApplicationInfo(); + + verify(monitorFactory, times(0)).create(zoneApplicationInfo.getApplicationId()); + verify(monitor, times(0)).monitor(any()); + manager.applicationActivated(zoneApplicationInfo); + verify(monitorFactory, times(isMonitored ? 1 : 0)).create(zoneApplicationInfo.getApplicationId()); + verify(monitor, times(isMonitored ? 1 : 0)).monitor(any()); + + when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.DOWN); + verifyNodeAdminGetStatus(0); + if (isMonitored) { + assertEquals(ServiceStatus.DOWN, getNodeAdminStatus()); + verifyNodeAdminGetStatus(1); + } else { + assertEquals(ServiceStatus.UP, getNodeAdminStatus()); + verifyNodeAdminGetStatus(0); + } + + verifyRoutingGetStatus(0); + assertEquals(ServiceStatus.NOT_CHECKED, getRoutingStatus()); + verifyRoutingGetStatus(0); + } + + private void verifyNodeAdminGetStatus(int invocations) { + verify(monitor, times(invocations)).getStatus( + eq(ZoneApplication.getApplicationId()), + eq(ZoneApplication.getNodeAdminClusterId()), + any(), + any()); + } + + private void verifyRoutingGetStatus(int invocations) { + verify(monitor, times(invocations)).getStatus( + eq(ZoneApplication.getApplicationId()), + eq(ZoneApplication.getRoutingClusterId()), + any(), + any()); + } + + private ServiceStatus getNodeAdminStatus() { + return manager.getStatus( + ZoneApplication.getApplicationId(), + ZoneApplication.getNodeAdminClusterId(), + ZoneApplication.getNodeAdminServiceType(), + new ConfigId("foo")); + } + + private ServiceStatus getRoutingStatus() { + return manager.getStatus( + ZoneApplication.getApplicationId(), + ZoneApplication.getRoutingClusterId(), + ZoneApplication.getRoutingServiceType(), + new ConfigId("bar")); + } + @Test public void infrastructureApplication() { + setUp(false); ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); List hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList()); @@ -85,6 +159,7 @@ public class HealthMonitorManagerTest { @Test public void threadPoolSize() { + setUp(false); assertEquals(9, HealthMonitorManager.THREAD_POOL_SIZE); } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index 480691772bb..e38589b6f81 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.duper.ProxyHostApplication; +import com.yahoo.vespa.service.duper.TestZoneApplication; +import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.Cancellable; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ServiceId; @@ -34,14 +36,18 @@ public class StateV1HealthModelTest { private Duration healthStaleness = Duration.ofSeconds(1); private Duration requestTimeout = Duration.ofSeconds(2); private Duration keepAlive = Duration.ofSeconds(3); - private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor); private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); private final List hostnames = Stream.of("host1", "host2").map(HostName::from).collect(Collectors.toList()); private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); - private final Map endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo); + + private StateV1HealthModel model; + private Map endpoints; @Test public void test() { + model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor, false); + endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo); + assertEquals(2, endpoints.size()); ApplicationId applicationId = ApplicationId.from("hosted-vespa", "proxy-host", "default"); @@ -63,4 +69,23 @@ public class StateV1HealthModelTest { assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus()); } } + + @Test + public void testMonitoringTenantHostHealth() { + model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor, true); + ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder() + .addNodeAdminCluster("h1") + .addRoutingCluster("r1") + .build() + .makeApplicationInfo(); + + endpoints = model.extractHealthEndpoints(zoneApplicationInfo); + assertEquals(1, endpoints.size()); + HealthEndpoint endpoint = endpoints.values().iterator().next(); + assertEquals("http://h1:8080/state/v1/health", endpoint.description()); + ServiceId serviceId = endpoint.getServiceId(); + assertEquals(ZoneApplication.getApplicationId(), serviceId.getApplicationId()); + assertEquals(ZoneApplication.getNodeAdminClusterId(), serviceId.getClusterId()); + assertEquals(ZoneApplication.getNodeAdminServiceType(), serviceId.getServiceType()); + } } \ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java index a0c7aca5869..b97fd6c64a5 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java @@ -3,16 +3,14 @@ package com.yahoo.vespa.service.manager; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; -import static com.yahoo.vespa.applicationmodel.ClusterId.NODE_ADMIN; import static com.yahoo.vespa.applicationmodel.ServiceStatus.DOWN; import static com.yahoo.vespa.applicationmodel.ServiceStatus.NOT_CHECKED; import static com.yahoo.vespa.applicationmodel.ServiceStatus.UP; -import static com.yahoo.vespa.applicationmodel.ServiceType.CONTAINER; -import static com.yahoo.vespa.service.duper.ZoneApplication.ZONE_APPLICATION_ID; import static org.junit.Assert.assertSame; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; @@ -38,7 +36,10 @@ public class UnionMonitorManagerTest { ServiceStatus expectedStatus) { when(healthMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(healthStatus); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(slobrokStatus); - ServiceStatus status = manager.getStatus(ZONE_APPLICATION_ID, NODE_ADMIN, CONTAINER, new ConfigId("config-id")); + ServiceStatus status = manager.getStatus( + ZoneApplication.getApplicationId(), + ZoneApplication.getNodeAdminClusterId(), + ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id")); assertSame(expectedStatus, status); } } \ No newline at end of file -- cgit v1.2.3