From cd7f0447817eaf21898f172bdc2a4fc8bb721d1a Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 25 Jan 2019 09:27:12 +0100 Subject: Metadata about /state/v1/health status The service monitor uses /state/v1/health to monitor config servers and the host admins (but not yet tenant host admins). This commit adds some metadata about the status of a service: - The time the status was last checked - The time the status changed to the current This can be used to e.g. make more intelligent decisions in the Orchestrator, e.g. only allowing a service to suspend if it has been DOWN longer than X seconds (to avoid spurious DOWN to break redundancy and uptime guarantees). --- .../service/health/ApplicationHealthMonitor.java | 11 +++-- .../com/yahoo/vespa/service/health/HealthInfo.java | 30 ++---------- .../yahoo/vespa/service/health/HealthMonitor.java | 4 +- .../vespa/service/health/HealthMonitorManager.java | 15 +++--- .../yahoo/vespa/service/health/HealthUpdater.java | 5 +- .../vespa/service/health/StateV1HealthMonitor.java | 6 +-- .../vespa/service/health/StateV1HealthUpdater.java | 25 ++++++++-- .../vespa/service/manager/UnionMonitorManager.java | 15 +++--- .../model/ApplicationInstanceGenerator.java | 9 ++-- .../vespa/service/model/ServiceMonitorImpl.java | 4 +- .../service/monitor/ServiceStatusProvider.java | 9 ++-- .../service/slobrok/SlobrokMonitorManagerImpl.java | 20 ++++---- .../health/ApplicationHealthMonitorTest.java | 10 ++-- .../service/health/HealthMonitorManagerTest.java | 13 ++--- .../service/health/StateV1HealthModelTest.java | 2 +- .../service/health/StateV1HealthMonitorTest.java | 7 ++- .../service/health/StateV1HealthUpdaterTest.java | 55 ++++++++++------------ .../service/manager/UnionMonitorManagerTest.java | 7 +-- .../model/ApplicationInstanceGeneratorTest.java | 3 +- .../yahoo/vespa/service/model/ExampleModel.java | 3 +- .../vespa/service/model/ModelGeneratorTest.java | 3 +- .../slobrok/SlobrokMonitorManagerImplTest.java | 2 +- .../vespa/service/slobrok/SlobrokMonitorTest.java | 7 +-- 23 files changed, 130 insertions(+), 135 deletions(-) (limited to 'service-monitor/src') diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index 5eac6fbb000..e728d1ea914 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.model.ServiceId; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; @@ -47,14 +48,14 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { } @Override - public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + public ServiceStatusInfo getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId); HealthMonitor monitor = monitors.get(serviceId); if (monitor == null) { - return ServiceStatus.NOT_CHECKED; + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } return monitor.getStatus(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthInfo.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthInfo.java index 17d9e6b7b49..17670c705d8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthInfo.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthInfo.java @@ -1,10 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.yolean.Exceptions; -import java.time.Instant; import java.util.Optional; import java.util.OptionalInt; @@ -19,7 +17,6 @@ public class HealthInfo { private final Optional exception; private final OptionalInt httpStatusCode; private final Optional healthStatusCode; - private final Instant time; static HealthInfo fromException(Exception exception) { return new HealthInfo(Optional.of(exception), OptionalInt.empty(), Optional.empty()); @@ -33,39 +30,18 @@ public class HealthInfo { return new HealthInfo(Optional.empty(), OptionalInt.empty(), Optional.of(healthStatusCode)); } - static HealthInfo empty() { - return new HealthInfo(Optional.empty(), OptionalInt.empty(), Optional.empty()); - } - - private HealthInfo(Optional exception, - OptionalInt httpStatusCode, - Optional healthStatusCode) { + private HealthInfo(Optional exception, OptionalInt httpStatusCode, Optional healthStatusCode) { this.exception = exception; this.httpStatusCode = httpStatusCode; this.healthStatusCode = healthStatusCode; - this.time = Instant.now(); } public boolean isHealthy() { return healthStatusCode.map(UP_STATUS_CODE::equals).orElse(false); } - public ServiceStatus toServiceStatus() { - // Bootstrapping ServiceStatus: To avoid thundering herd problem at startup, - // the clients will not fetch the health immediately. What should the ServiceStatus - // be before the first health has been fetched? - // - // NOT_CHECKED: Logically the right thing, but if an Orchestrator gets a suspend request - // in this window, and another service within the cluster is down, it ends up allowing - // suspension when it shouldn't have done so. - // - // DOWN: Only safe initial value, possibly except if the first initial delay is long, - // as that could indicate it has been down for too long. - return isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN; - } - - public Instant time() { - return time; + public Optional getErrorDescription() { + return isHealthy() ? Optional.empty() : Optional.of(toString()); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java index f0e13548f58..14ec977c98c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; /** * @author hakonhall */ interface HealthMonitor extends AutoCloseable { - ServiceStatus getStatus(); + ServiceStatusInfo getStatus(); @Override void close(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index ddfbdf59b3c..b802c6c5413 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -97,20 +98,20 @@ public class HealthMonitorManager implements MonitorManager { } @Override - public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + public ServiceStatusInfo getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { ApplicationHealthMonitor monitor = healthMonitors.get(applicationId); if (!monitorTenantHostHealth && ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { // Legacy: The zone app is not health monitored (monitor == null), but the node-admin cluster's services // are hard-coded to be UP - return ServiceStatus.UP; + return new ServiceStatusInfo(ServiceStatus.UP); } if (monitor == null) { - return ServiceStatus.NOT_CHECKED; + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } if (monitorTenantHostHealth && applicationId.equals(ZoneApplication.getApplicationId())) { @@ -120,7 +121,7 @@ public class HealthMonitorManager implements MonitorManager { if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { return monitor.getStatus(applicationId, clusterId, serviceType, configId); } else { - return ServiceStatus.NOT_CHECKED; + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java index 4ed49e17e9f..8101336c638 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java @@ -1,14 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.executor.Runlet; /** * A {@link HealthUpdater} will probe the health with {@link #run()}, whose result can be fetched with the - * thread-safe method {@link #getLatestHealthInfo()}. + * thread-safe method {@link #getServiceStatusInfo()}. * * @author hakonhall */ interface HealthUpdater extends Runlet { - HealthInfo getLatestHealthInfo(); + ServiceStatusInfo getServiceStatusInfo(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java index d37797c7be9..7a6494e0122 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.executor.Cancellable; import com.yahoo.vespa.service.executor.RunletExecutor; @@ -22,8 +22,8 @@ class StateV1HealthMonitor implements HealthMonitor { } @Override - public ServiceStatus getStatus() { - return updater.getLatestHealthInfo().toServiceStatus(); + public ServiceStatusInfo getStatus() { + return updater.getServiceStatusInfo(); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java index 011ec3b3212..972e81ce822 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -1,8 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; + import java.net.URL; import java.time.Duration; +import java.time.Instant; +import java.util.Optional; /** * @author hakonhall @@ -10,7 +15,7 @@ import java.time.Duration; class StateV1HealthUpdater implements HealthUpdater { private final StateV1HealthClient healthClient; - private volatile HealthInfo lastHealthInfo = HealthInfo.empty(); + private volatile ServiceStatusInfo serviceStatusInfo = new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) { this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive)); @@ -21,17 +26,27 @@ class StateV1HealthUpdater implements HealthUpdater { } @Override - public HealthInfo getLatestHealthInfo() { - return lastHealthInfo; + public ServiceStatusInfo getServiceStatusInfo() { + return serviceStatusInfo; } @Override public void run() { + // Get time before fetching rather than after, to make the resulting age be an upper limit. + Instant now = Instant.now(); + + HealthInfo healthInfo; try { - lastHealthInfo = healthClient.get(); + healthInfo = healthClient.get(); } catch (Exception e) { - lastHealthInfo = HealthInfo.fromException(e); + healthInfo = HealthInfo.fromException(e); } + + ServiceStatus newServiceStatus = healthInfo.isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN; + Optional newSince = newServiceStatus == serviceStatusInfo.serviceStatus() ? + serviceStatusInfo.since() : Optional.of(now); + + serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now), healthInfo.getErrorDescription()); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java index cfd9269d9c4..3490ad4a5d2 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java @@ -1,11 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.manager; +import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; @@ -17,19 +19,20 @@ public class UnionMonitorManager implements MonitorManager { private final SlobrokMonitorManagerImpl slobrokMonitorManager; private final HealthMonitorManager healthMonitorManager; + @Inject public UnionMonitorManager(SlobrokMonitorManagerImpl slobrokMonitorManager, HealthMonitorManager healthMonitorManager) { this.slobrokMonitorManager = slobrokMonitorManager; this.healthMonitorManager = healthMonitorManager; } @Override - public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + public ServiceStatusInfo getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { // Trust the new health monitoring status if it actually monitors the particular service. - ServiceStatus status = healthMonitorManager.getStatus(applicationId, clusterId, serviceType, configId); - if (status != ServiceStatus.NOT_CHECKED) { + ServiceStatusInfo status = healthMonitorManager.getStatus(applicationId, clusterId, serviceType, configId); + if (status.serviceStatus() != ServiceStatus.NOT_CHECKED) { return status; } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 3ca9446df26..9e6b64f0c91 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceClusterKey; import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.duper.ConfigServerApplication; @@ -99,11 +99,8 @@ public class ApplicationInstanceGenerator { HostName hostName, ServiceStatusProvider serviceStatusProvider) { ConfigId configId = toConfigId(serviceInfo); - - ServiceStatus status = serviceStatusProvider.getStatus( - applicationId, - clusterId, - toServiceType(serviceInfo), configId); + ServiceType serviceType = toServiceType(serviceInfo); + ServiceStatusInfo status = serviceStatusProvider.getStatus(applicationId, clusterId, serviceType, configId); return new ServiceInstance(configId, hostName, status); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index cff98957590..50ea31eb9c4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -21,12 +21,10 @@ public class ServiceMonitorImpl implements ServiceMonitor { @Inject public ServiceMonitorImpl(DuperModelManager duperModelManager, - SlobrokMonitorManagerImpl slobrokMonitorManager, - HealthMonitorManager healthMonitorManager, + UnionMonitorManager monitorManager, Metric metric, Timer timer, Zone zone) { - UnionMonitorManager monitorManager = new UnionMonitorManager(slobrokMonitorManager, healthMonitorManager); duperModelManager.registerListener(monitorManager); ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java index 30a20cf9980..9486bf505ab 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; /** @@ -19,9 +20,9 @@ public interface ServiceStatusProvider { * service status provider does does not monitor the service status for * the particular application, cluster, service type, and config id. */ - ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId); + ServiceStatusInfo getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 4e47bf010d7..203ad7266db 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -9,6 +9,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.manager.MonitorManager; @@ -84,27 +85,26 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { } @Override - public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + public ServiceStatusInfo getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { if (!wouldMonitor(applicationId)) { - return ServiceStatus.NOT_CHECKED; + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } Optional slobrokServiceName = findSlobrokServiceName(serviceType, configId); if (slobrokServiceName.isPresent()) { synchronized (monitor) { SlobrokMonitor slobrokMonitor = slobrokMonitors.get(applicationId); - if (slobrokMonitor != null && - slobrokMonitor.registeredInSlobrok(slobrokServiceName.get())) { - return ServiceStatus.UP; + if (slobrokMonitor != null && slobrokMonitor.registeredInSlobrok(slobrokServiceName.get())) { + return new ServiceStatusInfo(ServiceStatus.UP); } else { - return ServiceStatus.DOWN; + return new ServiceStatusInfo(ServiceStatus.DOWN); } } } else { - return ServiceStatus.NOT_CHECKED; + return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index 821f5282998..8861fa7db9f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.model.ServiceId; import com.yahoo.vespa.service.monitor.ConfigserverUtil; @@ -47,9 +48,9 @@ public class ApplicationHealthMonitorTest { verify(endpoint1, times(1)).startMonitoring(); verify(endpoint2, times(1)).startMonitoring(); - when(monitor1.getStatus()).thenReturn(ServiceStatus.UP); - when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN); - when(monitor3.getStatus()).thenReturn(ServiceStatus.UP); + when(monitor1.getStatus()).thenReturn(new ServiceStatusInfo(ServiceStatus.UP)); + when(monitor2.getStatus()).thenReturn(new ServiceStatusInfo(ServiceStatus.DOWN)); + when(monitor3.getStatus()).thenReturn(new ServiceStatusInfo(ServiceStatus.UP)); assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1")); assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2")); @@ -94,6 +95,7 @@ public class ApplicationHealthMonitorTest { configServerApplication.getApplicationId(), configServerApplication.getClusterId(), configServerApplication.getServiceType(), - configServerApplication.configIdFor(HostName.from(hostname))); + configServerApplication.configIdFor(HostName.from(hostname))) + .serviceStatus(); } } \ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 6e18bebf791..0c0a644165d 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.ControllerHostApplication; import com.yahoo.vespa.service.duper.DuperModelManager; @@ -62,7 +63,7 @@ public class HealthMonitorManagerTest { ZoneApplication.getApplicationId(), ZoneApplication.getNodeAdminClusterId(), ZoneApplication.getNodeAdminServiceType(), - new ConfigId("config-id-1")); + new ConfigId("config-id-1")).serviceStatus(); assertEquals(ServiceStatus.UP, status); } @@ -91,7 +92,7 @@ public class HealthMonitorManagerTest { verify(monitorFactory, times(isMonitored ? 1 : 0)).create(zoneApplicationInfo.getApplicationId()); verify(monitor, times(isMonitored ? 1 : 0)).monitor(any()); - when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.DOWN); + when(monitor.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.DOWN)); verifyNodeAdminGetStatus(0); if (isMonitored) { assertEquals(ServiceStatus.DOWN, getNodeAdminStatus()); @@ -127,7 +128,7 @@ public class HealthMonitorManagerTest { ZoneApplication.getApplicationId(), ZoneApplication.getNodeAdminClusterId(), ZoneApplication.getNodeAdminServiceType(), - new ConfigId("foo")); + new ConfigId("foo")).serviceStatus(); } private ServiceStatus getRoutingStatus() { @@ -135,7 +136,7 @@ public class HealthMonitorManagerTest { ZoneApplication.getApplicationId(), ZoneApplication.getRoutingClusterId(), ZoneApplication.getRoutingServiceType(), - new ConfigId("bar")); + new ConfigId("bar")).serviceStatus(); } @Test @@ -149,7 +150,7 @@ public class HealthMonitorManagerTest { manager.applicationActivated(proxyHostApplicationInfo); verify(monitorFactory, times(1)).create(proxyHostApplicationInfo.getApplicationId()); - when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP); + when(monitor.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.UP)); assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1"); ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); @@ -168,7 +169,7 @@ public class HealthMonitorManagerTest { infraApplication.getApplicationId(), infraApplication.getClusterId(), infraApplication.getServiceType(), - infraApplication.configIdFor(HostName.from(hostname))); + infraApplication.configIdFor(HostName.from(hostname))).serviceStatus(); assertEquals(expected, actual); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index e38589b6f81..b6b88b0c0d3 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -66,7 +66,7 @@ public class StateV1HealthModelTest { Cancellable cancellable = mock(Cancellable.class); when(executor.scheduleWithFixedDelay(any(), any())).thenReturn(cancellable); try (HealthMonitor healthMonitor = endpoint1.startMonitoring()) { - assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus()); + assertEquals(ServiceStatus.NOT_CHECKED, healthMonitor.getStatus().serviceStatus()); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java index c892118990f..533e5173ccc 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java @@ -16,20 +16,19 @@ public class StateV1HealthMonitorTest { @Test public void downThenUpThenDown() throws Exception { StateV1HealthClient client = mock(StateV1HealthClient.class); - when(client.get()).thenReturn(HealthInfo.empty()); StateV1HealthUpdater updater = new StateV1HealthUpdater(client); RunletExecutor executor = new RunletExecutorImpl(2); try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) { - assertEquals(ServiceStatus.DOWN, monitor.getStatus()); + assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus().serviceStatus()); when(client.get()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE)); - while (monitor.getStatus() != ServiceStatus.UP) { + while (monitor.getStatus().serviceStatus() != ServiceStatus.UP) { try { Thread.sleep(2); } catch (InterruptedException ignored) { } } when(client.get()).thenReturn(HealthInfo.fromException(new IllegalStateException("foo"))); - while (monitor.getStatus() != ServiceStatus.DOWN) { + while (monitor.getStatus().serviceStatus() != ServiceStatus.DOWN) { try { Thread.sleep(2); } catch (InterruptedException ignored) { } } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java index e7b7a829dac..b8f108a480e 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.service.health; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; @@ -15,7 +16,7 @@ import java.net.URL; import java.util.function.Function; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; @@ -31,7 +32,7 @@ public class StateV1HealthUpdaterTest { @Test public void successfulRequestResponse() throws IOException { - HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + ServiceStatusInfo info = getServiceStatusInfoFromJsonResponse("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + " \"from\": 1.528789829249E9,\n" + @@ -41,13 +42,13 @@ public class StateV1HealthUpdaterTest { " \"status\": {\"code\": \"up\"},\n" + " \"time\": 1528789889364\n" + "}"); - assertTrue(info.isHealthy()); - assertEquals(ServiceStatus.UP, info.toServiceStatus()); + assertEquals(ServiceStatus.UP, info.serviceStatus()); + assertNull(info.errorOrNull()); } @Test public void notUpResponse() throws IOException { - HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + ServiceStatusInfo info = getServiceStatusInfoFromJsonResponse("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + " \"from\": 1.528789829249E9,\n" + @@ -57,14 +58,13 @@ public class StateV1HealthUpdaterTest { " \"status\": {\"code\": \"initializing\"},\n" + " \"time\": 1528789889364\n" + "}"); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertEquals("Bad health status code 'initializing'", info.toString()); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertEquals("Bad health status code 'initializing'", info.errorOrNull()); } @Test public void noCodeInResponse() throws IOException { - HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + ServiceStatusInfo info = getServiceStatusInfoFromJsonResponse("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + " \"from\": 1.528789829249E9,\n" + @@ -74,14 +74,13 @@ public class StateV1HealthUpdaterTest { " \"status\": {\"foo\": \"bar\"},\n" + " \"time\": 1528789889364\n" + "}"); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertEquals("Bad health status code 'down'", info.toString()); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertEquals("Bad health status code 'down'", info.errorOrNull()); } @Test public void noStatusInResponse() throws IOException { - HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + ServiceStatusInfo info = getServiceStatusInfoFromJsonResponse("{\n" + " \"metrics\": {\n" + " \"snapshot\": {\n" + " \"from\": 1.528789829249E9,\n" + @@ -90,20 +89,18 @@ public class StateV1HealthUpdaterTest { " },\n" + " \"time\": 1528789889364\n" + "}"); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertEquals("Bad health status code 'down'", info.toString()); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertEquals("Bad health status code 'down'", info.errorOrNull()); } @Test public void badJson() throws IOException { - HealthInfo info = getHealthInfoFromJsonResponse("} foo bar"); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertTrue(info.toString().startsWith("Exception: Unexpected close marker '}': ")); + ServiceStatusInfo info = getServiceStatusInfoFromJsonResponse("} foo bar"); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertTrue(info.errorOrNull().startsWith("Exception: Unexpected close marker '}': ")); } - private HealthInfo getHealthInfoFromJsonResponse(String content) + private ServiceStatusInfo getServiceStatusInfoFromJsonResponse(String content) throws IOException { CloseableHttpClient client = mock(CloseableHttpClient.class); @@ -121,7 +118,7 @@ public class StateV1HealthUpdaterTest { try (StateV1HealthUpdater updater = makeUpdater(client, entry -> content)) { when(httpEntity.getContentLength()).thenReturn((long) content.length()); updater.run(); - return updater.getLatestHealthInfo(); + return updater.getServiceStatusInfo(); } } @@ -133,10 +130,9 @@ public class StateV1HealthUpdaterTest { try (StateV1HealthUpdater updater = makeUpdater(client, entry -> "")) { updater.run(); - HealthInfo info = updater.getLatestHealthInfo(); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertEquals("Exception: exception string", info.toString()); + ServiceStatusInfo info = updater.getServiceStatusInfo(); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertEquals("Exception: exception string", info.errorOrNull()); } } @@ -160,10 +156,9 @@ public class StateV1HealthUpdaterTest { try (HealthUpdater updater = makeUpdater(client, entry -> content)) { when(httpEntity.getContentLength()).thenReturn((long) content.length()); updater.run(); - HealthInfo info = updater.getLatestHealthInfo(); - assertFalse(info.isHealthy()); - assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); - assertEquals("Bad HTTP response status code 500", info.toString()); + ServiceStatusInfo info = updater.getServiceStatusInfo(); + assertEquals(ServiceStatus.DOWN, info.serviceStatus()); + assertEquals("Bad HTTP response status code 500", info.errorOrNull()); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java index b97fd6c64a5..5cfe70fae5f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.service.manager; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; @@ -34,12 +35,12 @@ public class UnionMonitorManagerTest { private void testWith(ServiceStatus healthStatus, ServiceStatus slobrokStatus, ServiceStatus expectedStatus) { - when(healthMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(healthStatus); - when(slobrokMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(slobrokStatus); + when(healthMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(healthStatus)); + when(slobrokMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(slobrokStatus)); ServiceStatus status = manager.getStatus( ZoneApplication.getApplicationId(), ZoneApplication.getNodeAdminClusterId(), - ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id")); + ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id")).serviceStatus(); assertSame(expectedStatus, status); } } \ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index 68f5aa8c451..7eba54977cc 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; @@ -34,7 +35,7 @@ public class ApplicationInstanceGeneratorTest { @Test public void toApplicationInstance() { - when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.NOT_CHECKED); + when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.NOT_CHECKED)); Zone zone = mock(Zone.class); ApplicationInfo configServer = configServerApplication.makeApplicationInfo( configServerList.stream().map(HostName::from).collect(Collectors.toList())); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java index 96e69f3cb69..0f7c0dde357 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java @@ -32,8 +32,9 @@ public class ExampleModel { static final String TENANT = "tenant"; static final String APPLICATION_NAME = "application"; public static final String INSTANCE_NAME = "default"; + public static final ApplicationId APPLICATION_ID = ApplicationId.from(TENANT, APPLICATION_NAME, INSTANCE_NAME); - static SuperModel createExampleSuperModelWithOneRpcPort(String hostname, int rpcPort) { + public static SuperModel createExampleSuperModelWithOneRpcPort(String hostname, int rpcPort) { List hosts = Stream.of(hostname).collect(Collectors.toList()); ApplicationInfo applicationInfo = ExampleModel diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index b77bdb4e54e..37da0611d53 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ConfigserverUtil; import com.yahoo.vespa.service.duper.ConfigServerApplication; @@ -43,7 +44,7 @@ public class ModelGeneratorTest { SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) - .thenReturn(ServiceStatus.UP); + .thenReturn(new ServiceStatusInfo(ServiceStatus.UP)); ServiceModel serviceModel = modelGenerator.toServiceModel( diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java index bc6bff6fba8..0b8a13a8869 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java @@ -79,7 +79,7 @@ public class SlobrokMonitorManagerImplTest { return slobrokMonitorManager.getStatus( application.getApplicationId(), clusterId, - new ServiceType(serviceType), new ConfigId("config.id")); + new ServiceType(serviceType), new ConfigId("config.id")).serviceStatus(); } @Test diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorTest.java index 907e52298bf..8bec3bf6cd8 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorTest.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.service.slobrok; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.SuperModel; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.vespa.service.model.ExampleModel; import org.junit.Test; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class SlobrokMonitorTest { private final SlobrokList slobrokList = mock(SlobrokList.class); @@ -24,14 +26,13 @@ public class SlobrokMonitorTest { @Test public void testUpdateSlobrokList2() { - /* final String hostname = "hostname"; final int port = 1; SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(hostname, port); - slobrokMonitor.updateSlobrokList(superModel.getApplicationInfo()); + slobrokMonitor.updateSlobrokList(superModel.getApplicationInfo(ExampleModel.APPLICATION_ID).get()); String[] expectedSpecs = new String[] {"tcp/" + hostname + ":" + port}; - verify(slobrokList).setup(expectedSpecs); */ + verify(slobrokList).setup(expectedSpecs); } } -- cgit v1.2.3