diff options
Diffstat (limited to 'service-monitor')
13 files changed, 99 insertions, 29 deletions
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index e728d1ea914..873baeae710 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -8,13 +8,14 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Responsible for monitoring a whole application using /state/v1/health. @@ -24,6 +25,10 @@ import java.util.Set; class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { private final ApplicationId applicationId; private final StateV1HealthModel healthModel; + + // Guards against concurrent access to monitors field w/objects in 1. monitor() (called from e.g. DuperModel), + // 2. getStatus() called from caching layer and Orchestrator above, and 3. REST calls to getAllServiceStatuses(). + private final Object guard = new Object(); private final Map<ServiceId, HealthMonitor> monitors = new HashMap<>(); ApplicationHealthMonitor(ApplicationId applicationId, StateV1HealthModel healthModel) { @@ -38,13 +43,15 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { Map<ServiceId, HealthEndpoint> endpoints = healthModel.extractHealthEndpoints(applicationInfo); - // Remove obsolete monitors - Set<ServiceId> removed = new HashSet<>(monitors.keySet()); - removed.removeAll(endpoints.keySet()); - removed.stream().map(monitors::remove).forEach(HealthMonitor::close); + synchronized (guard) { + // Remove obsolete monitors + Set<ServiceId> removed = new HashSet<>(monitors.keySet()); + removed.removeAll(endpoints.keySet()); + removed.stream().map(monitors::remove).forEach(HealthMonitor::close); - // Add new monitors. - endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring())); + // Add new monitors. + endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring())); + } } @Override @@ -53,12 +60,23 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { ServiceType serviceType, ConfigId configId) { ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId); - HealthMonitor monitor = monitors.get(serviceId); - if (monitor == null) { - return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); + + synchronized (guard) { + HealthMonitor monitor = monitors.get(serviceId); + if (monitor == null) { + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); + } + + return monitor.getStatus(); } + } - return monitor.getStatus(); + public Map<ServiceId, ServiceStatusInfo> getAllServiceStatuses() { + synchronized (guard) { + return monitors.entrySet().stream().collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> entry.getValue().getStatus())); + } } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java index 8c4997634a0..eae5ef3f284 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; /** * An endpoint 1-1 with a service and that can be health monitored. diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index b802c6c5413..ad9a50f8127 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -14,9 +14,14 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutorImpl; +import com.yahoo.vespa.service.manager.HealthMonitorApi; import com.yahoo.vespa.service.manager.MonitorManager; +import com.yahoo.vespa.service.monitor.ServiceId; import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** @@ -24,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; * * @author hakon */ -public class HealthMonitorManager implements MonitorManager { +public class HealthMonitorManager implements MonitorManager, HealthMonitorApi { // Weight the following against each other: // - The number of threads N working on health checking // - The health request timeout T @@ -139,4 +144,19 @@ public class HealthMonitorManager implements MonitorManager { return false; } + + @Override + public List<ApplicationId> getMonitoredApplicationIds() { + return Collections.list(healthMonitors.keys()); + } + + @Override + public Map<ServiceId, ServiceStatusInfo> getServices(ApplicationId applicationId) { + ApplicationHealthMonitor applicationHealthMonitor = healthMonitors.get(applicationId); + if (applicationHealthMonitor == null) { + return Collections.emptyMap(); + } + + return applicationHealthMonitor.getAllServiceStatuses(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java index 8eca03c616f..73f0480bf96 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.executor.RunletExecutor; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; import java.net.URL; import java.time.Duration; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index 04943f81478..f015ecadbfc 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -6,10 +6,11 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.service.duper.HostAdminApplication; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; import java.time.Duration; import java.util.Arrays; @@ -53,20 +54,26 @@ public class StateV1HealthModel implements AutoCloseable { HostName hostname = HostName.from(hostInfo.getHostname()); for (ServiceInfo serviceInfo : hostInfo.getServices()) { - if (monitorTenantHostHealth && isZoneApplication && - !ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) { - // Only the node admin/host admin cluster of the zone application should be monitored - // TODO: Move the node admin cluster out to a separate infrastructure application - continue; + boolean isNodeAdmin = false; + if (monitorTenantHostHealth && isZoneApplication) { + if (ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) { + isNodeAdmin = true; + } else { + // Only the node admin/host admin cluster of the zone application should be monitored + // TODO: Move the node admin cluster out to a separate infrastructure application + continue; + } } ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) { + // The host-admin-in-zone-application is one big hack. + int port = isNodeAdmin ? HostAdminApplication.HOST_ADMIN_HEALT_PORT : portInfo.getPort(); StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint( serviceId, hostname, - portInfo.getPort(), + port, targetHealthStaleness, requestTimeout, connectionKeepAlive, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java index 972e81ce822..6a6768aa78b 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -13,15 +13,17 @@ import java.util.Optional; * @author hakonhall */ class StateV1HealthUpdater implements HealthUpdater { + private final String endpoint; private final StateV1HealthClient healthClient; private volatile ServiceStatusInfo serviceStatusInfo = new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) { - this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive)); + this(url.toString(), new StateV1HealthClient(url, requestTimeout, connectionKeepAlive)); } - StateV1HealthUpdater(StateV1HealthClient healthClient) { + StateV1HealthUpdater(String endpoint, StateV1HealthClient healthClient) { + this.endpoint = endpoint; this.healthClient = healthClient; } @@ -46,7 +48,8 @@ class StateV1HealthUpdater implements HealthUpdater { Optional<Instant> newSince = newServiceStatus == serviceStatusInfo.serviceStatus() ? serviceStatusInfo.since() : Optional.of(now); - serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now), healthInfo.getErrorDescription()); + serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now), + healthInfo.getErrorDescription(), Optional.of(endpoint)); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java new file mode 100644 index 00000000000..7bc22db93cd --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java @@ -0,0 +1,21 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.manager; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; +import com.yahoo.vespa.service.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +import java.util.List; +import java.util.Map; + +/** + * The API of {@link HealthMonitorManager} which is exported to other bundles (typically for REST). + * + * @author hakonhall + */ +public interface HealthMonitorApi extends ServiceStatusProvider { + List<ApplicationId> getMonitoredApplicationIds(); + Map<ServiceId, ServiceStatusInfo> getServices(ApplicationId applicationId); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 9e6b64f0c91..0a4e8e75adf 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java index c0bcff7284c..a97cb1812c1 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.model; +package com.yahoo.vespa.service.monitor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index 8861fa7db9f..2ad0eb8f67d 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ConfigserverUtil; import org.junit.Test; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index b6b88b0c0d3..fcbc3cd9b9f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.service.duper.TestZoneApplication; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.Cancellable; import com.yahoo.vespa.service.executor.RunletExecutor; -import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceId; import org.junit.Test; import java.time.Duration; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java index 533e5173ccc..e3fb7d08d93 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java @@ -17,7 +17,7 @@ public class StateV1HealthMonitorTest { public void downThenUpThenDown() throws Exception { StateV1HealthClient client = mock(StateV1HealthClient.class); - StateV1HealthUpdater updater = new StateV1HealthUpdater(client); + StateV1HealthUpdater updater = new StateV1HealthUpdater("https://foo/state/v1/health", client); RunletExecutor executor = new RunletExecutorImpl(2); try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) { assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus().serviceStatus()); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java index b8f108a480e..a491307d6dc 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java @@ -165,6 +165,6 @@ public class StateV1HealthUpdaterTest { private StateV1HealthUpdater makeUpdater(CloseableHttpClient client, Function<HttpEntity, String> getContentFunction) { ApacheHttpClient apacheHttpClient = new ApacheHttpClient(url, client); StateV1HealthClient healthClient = new StateV1HealthClient(apacheHttpClient, getContentFunction); - return new StateV1HealthUpdater(healthClient); + return new StateV1HealthUpdater(url.toString(), healthClient); } }
\ No newline at end of file |