From 8bb08512378671a93a0c2fbee74e4ee8aeb48549 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Wed, 13 Jun 2018 13:04:15 +0200 Subject: Monitor cfg app health if activated --- .../internal/health/ApplicationHealthMonitor.java | 25 +++- .../monitor/internal/health/HealthClient.java | 139 ++++++++++------- .../monitor/internal/health/HealthEndpoint.java | 35 ++--- .../monitor/internal/health/HealthInfo.java | 14 +- .../monitor/internal/health/HealthMonitor.java | 34 +++-- .../internal/health/HealthMonitorManager.java | 31 ++-- .../internal/health/HttpHealthEndpoint.java | 44 ++++++ .../internal/health/HttpsHealthEndpoint.java | 53 +++++++ .../health/ApplicationHealthMonitorTest.java | 112 +++++++++++++- .../monitor/internal/health/HealthClientTest.java | 165 +++++++++++++++++++++ .../monitor/internal/health/HealthMonitorTest.java | 26 +++- 11 files changed, 560 insertions(+), 118 deletions(-) create mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java create mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java create mode 100644 service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java (limited to 'service-monitor/src') diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java index bd2658db8aa..7c78d61da30 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * Responsible for monitoring a whole application using /state/v1/health. @@ -37,7 +38,13 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos private final Map healthMonitors; public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) { - return new ApplicationHealthMonitor(makeHealthMonitors(application)); + return startMonitoring(application, HealthMonitor::new); + } + + /** For testing. */ + static ApplicationHealthMonitor startMonitoring(ApplicationInfo application, + Function mapper) { + return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper)); } private ApplicationHealthMonitor(Map healthMonitors) { @@ -64,7 +71,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos healthMonitors.clear(); } - private static Map makeHealthMonitors(ApplicationInfo application) { + private static Map makeHealthMonitors( + ApplicationInfo application, Function monitorFactory) { Map healthMonitors = new HashMap<>(); for (HostInfo hostInfo : application.getModel().getHosts()) { for (ServiceInfo serviceInfo : hostInfo.getServices()) { @@ -73,7 +81,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos application, hostInfo, serviceInfo, - portInfo) + portInfo, + monitorFactory) .ifPresent(healthMonitor -> healthMonitors.put( ApplicationInstanceGenerator.getServiceId(application, serviceInfo), healthMonitor)); @@ -87,14 +96,14 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos ApplicationInfo applicationInfo, HostInfo hostInfo, ServiceInfo serviceInfo, - PortInfo portInfo) { + PortInfo portInfo, + Function monitorFactory) { if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) { HostName hostname = HostName.from(hostInfo.getHostname()); HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort()); - // todo: make HealthMonitor - // HealthMonitor healthMonitor = new HealthMonitor(endpoint); - // healthMonitor.startMonitoring(); - return Optional.empty(); + HealthMonitor healthMonitor = monitorFactory.apply(endpoint); + healthMonitor.startMonitoring(); + return Optional.of(healthMonitor); } return Optional.empty(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java index 43a02a385be..128ca1b5d18 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java @@ -14,7 +14,6 @@ import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; import org.apache.http.impl.client.HttpClients; @@ -23,8 +22,16 @@ import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLContext; +import java.util.function.Function; +import java.util.function.Supplier; + +import static com.yahoo.yolean.Exceptions.uncheck; /** + * Health client + * + * NOT thread-safe. + * * @author hakon */ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.Listener { @@ -47,27 +54,76 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List }; private final HealthEndpoint endpoint; + private final Supplier clientSupplier; + private final Function getContentFunction; - private volatile CloseableHttpClient httpClient; + private CloseableHttpClient httpClient = null; public HealthClient(HealthEndpoint endpoint) { + this(endpoint, + () -> makeCloseableHttpClient(endpoint), + entity -> uncheck(() -> EntityUtils.toString(entity))); + } + + /** For testing. */ + HealthClient(HealthEndpoint endpoint, + Supplier clientSupplier, + Function getContentFunction) { this.endpoint = endpoint; + this.clientSupplier = clientSupplier; + this.getContentFunction = getContentFunction; } public void start() { - endpoint.getServiceIdentityProvider().ifPresent(provider -> { - onCredentialsUpdate(provider.getIdentitySslContext(), null); - provider.addIdentityListener(this); - }); + updateHttpClient(); + endpoint.registerListener(this); } @Override public void onCredentialsUpdate(SSLContext sslContext, AthenzService ignored) { - SSLConnectionSocketFactory socketFactory = - new SSLConnectionSocketFactory(sslContext, endpoint.getHostnameVerifier().orElse(null)); + updateHttpClient(); + } + + public HealthEndpoint getEndpoint() { + return endpoint; + } + public HealthInfo getHealthInfo() { + try { + return probeHealth(); + } catch (Exception e) { + return HealthInfo.fromException(e); + } + } + + @Override + public void close() { + endpoint.removeListener(this); + + if (httpClient != null) { + try { + httpClient.close(); + } catch (Exception e) { + // ignore + } + httpClient = null; + } + } + + private void updateHttpClient() { + CloseableHttpClient httpClient = clientSupplier.get(); + + if (this.httpClient != null) { + // Note: close() can be called any number of times. + uncheck(() -> this.httpClient.close()); + } + + this.httpClient = httpClient; + } + + private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) { Registry registry = RegistryBuilder.create() - .register("https", socketFactory) + .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory()) .build(); HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry); @@ -78,7 +134,7 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data .build(); - this.httpClient = HttpClients.custom() + return HttpClients.custom() .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY) .setConnectionManager(connectionManager) .disableAutomaticRetries() @@ -86,54 +142,37 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List .build(); } - public HealthInfo getHealthInfo() { - try { - return probeHealth(); - } catch (Exception e) { - return HealthInfo.fromException(e); - } - } - - @Override - public void close() { - endpoint.getServiceIdentityProvider().ifPresent(provider -> provider.removeIdentityListener(this)); - - try { - httpClient.close(); - } catch (Exception e) { - // ignore - } - httpClient = null; - } - private HealthInfo probeHealth() throws Exception { HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString()); - CloseableHttpResponse httpResponse; CloseableHttpClient httpClient = this.httpClient; if (httpClient == null) { - throw new IllegalStateException("HTTP client has closed"); + throw new IllegalStateException("HTTP client never started or has closed"); } - httpResponse = httpClient.execute(httpget); + CloseableHttpResponse httpResponse = httpClient.execute(httpget); - int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode < 200 || httpStatusCode >= 300) { - return HealthInfo.fromBadHttpStatusCode(httpStatusCode); - } - - HttpEntity bodyEntity = httpResponse.getEntity(); - long contentLength = bodyEntity.getContentLength(); - if (contentLength > MAX_CONTENT_LENGTH) { - throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); - } - String body = EntityUtils.toString(bodyEntity); - HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class); - - if (healthResponse.status == null || healthResponse.status.code == null) { - return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); - } else { - return HealthInfo.fromHealthStatusCode(healthResponse.status.code); + try { + int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); + if (httpStatusCode < 200 || httpStatusCode >= 300) { + return HealthInfo.fromBadHttpStatusCode(httpStatusCode); + } + + HttpEntity bodyEntity = httpResponse.getEntity(); + long contentLength = bodyEntity.getContentLength(); + if (contentLength > MAX_CONTENT_LENGTH) { + throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); + } + String body = getContentFunction.apply(bodyEntity); + HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class); + + if (healthResponse.status == null || healthResponse.status.code == null) { + return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); + } else { + return HealthInfo.fromHealthStatusCode(healthResponse.status.code); + } + } finally { + httpResponse.close(); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java index e9d17a9ab70..bc667a60bc7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java @@ -5,25 +5,22 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import org.apache.http.conn.socket.ConnectionSocketFactory; import javax.net.ssl.HostnameVerifier; import java.net.URL; import java.util.Collections; -import java.util.Optional; import static com.yahoo.yolean.Exceptions.uncheck; /** * @author hakon */ -class HealthEndpoint { - private final URL url; - private final Optional hostnameVerifier; - private final Optional serviceIdentityProvider; +public interface HealthEndpoint { static HealthEndpoint forHttp(HostName hostname, int port) { URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health")); - return new HealthEndpoint(url, Optional.empty(), Optional.empty()); + return new HttpHealthEndpoint(url); } static HealthEndpoint forHttps(HostName hostname, @@ -32,26 +29,12 @@ class HealthEndpoint { AthenzIdentity remoteIdentity) { URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health")); HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity)); - return new HealthEndpoint(url, Optional.of(serviceIdentityProvider), Optional.of(peerVerifier)); + return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier); } - private HealthEndpoint(URL url, - Optional serviceIdentityProvider, - Optional hostnameVerifier) { - this.url = url; - this.serviceIdentityProvider = serviceIdentityProvider; - this.hostnameVerifier = hostnameVerifier; - } - - public URL getStateV1HealthUrl() { - return url; - } - - public Optional getServiceIdentityProvider() { - return serviceIdentityProvider; - } - - public Optional getHostnameVerifier() { - return hostnameVerifier; - } + URL getStateV1HealthUrl(); + ConnectionSocketFactory getConnectionSocketFactory(); + void registerListener(ServiceIdentityProvider.Listener listener); + void removeListener(ServiceIdentityProvider.Listener listener); + String toString(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java index a3fe3cb3106..8b724afba5f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java @@ -50,7 +50,17 @@ public class HealthInfo { return healthStatusCode.map(UP_STATUS_CODE::equals).orElse(false); } - public ServiceStatus toSerivceStatus() { + public ServiceStatus toServiceStatus() { + // Bootstrapping ServiceStatus: To avoid thundering herd problem at startup, + // the clients will not fetch the health immediately. What should the ServiceStatus + // be before the first health has been fetched? + // + // NOT_CHECKED: Logically the right thing, but if an Orchestrator gets a suspend request + // in this window, and another service within the cluster is down, it ends up allowing + // suspension when it shouldn't have done so. + // + // DOWN: Only safe initial value, possibly except if the first initial delay is long, + // as that could indicate it has been down for too long. return isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN; } @@ -65,7 +75,7 @@ public class HealthInfo { } else if (healthStatusCode.isPresent()) { return "Bad health status code '" + healthStatusCode.get() + "'"; } else if (exception.isPresent()) { - return Exceptions.toMessageString(exception.get()); + return "Exception: " + Exceptions.toMessageString(exception.get()); } else if (httpStatusCode.isPresent()) { return "Bad HTTP response status code " + httpStatusCode.getAsInt(); } else { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java index fd809b32918..5518f9ddeba 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java @@ -13,40 +13,48 @@ import java.util.logging.Logger; /** * Used to monitor the health of a single URL endpoint. * + *

Must be closed on successful start of monitoring ({} + * + *

Thread-safe + * * @author hakon */ public class HealthMonitor implements AutoCloseable { private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName()); - private static final Duration DELAY = Duration.ofSeconds(20); + + /** The duration between each health request. */ + private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10); + // About 'static': Javadoc says "Instances of java.util.Random are threadsafe." private static final Random random = new Random(); private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private final HealthClient healthClient; + private final Duration delay; private volatile HealthInfo lastHealthInfo = HealthInfo.empty(); public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) { - this.healthClient = new HealthClient(stateV1HealthEndpoint); + this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY); } /** For testing. */ - HealthMonitor(HealthClient healthClient) { + HealthMonitor(HealthClient healthClient, Duration delay) { this.healthClient = healthClient; + this.delay = delay; } public void startMonitoring() { healthClient.start(); executor.scheduleWithFixedDelay( this::updateSynchronously, - initialDelayInSeconds(DELAY.getSeconds()), - DELAY.getSeconds(), - TimeUnit.SECONDS); + initialDelayInMillis(delay.toMillis()), + delay.toMillis(), + TimeUnit.MILLISECONDS); } public ServiceStatus getStatus() { - // todo: return lastHealthInfo.toServiceStatus(); - return ServiceStatus.NOT_CHECKED; + return lastHealthInfo.toServiceStatus(); } @Override @@ -63,11 +71,17 @@ public class HealthMonitor implements AutoCloseable { healthClient.close(); } - private long initialDelayInSeconds(long maxInitialDelayInSeconds) { + private long initialDelayInMillis(long maxInitialDelayInSeconds) { return random.nextLong() % maxInitialDelayInSeconds; } private void updateSynchronously() { - lastHealthInfo = healthClient.getHealthInfo(); + try { + lastHealthInfo = healthClient.getHealthInfo(); + } catch (Throwable t) { + // An uncaught exception will kill the executor.scheduleWithFixedDelay thread! + logger.log(LogLevel.WARNING, "Failed to get health info for " + + healthClient.getEndpoint(), t); + } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java index 473ef5e3a94..383cb6961a7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java @@ -9,17 +9,20 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.application.ZoneApplication; import com.yahoo.vespa.service.monitor.internal.MonitorManager; -import java.util.HashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** + * Manages all /state/v1/health related monitoring. + * * @author hakon */ public class HealthMonitorManager implements MonitorManager { - private final Map healthMonitors = new HashMap<>(); + private final ConcurrentHashMap healthMonitors = + new ConcurrentHashMap<>(); private final ConfigserverConfig configserverConfig; @Inject @@ -29,7 +32,7 @@ public class HealthMonitorManager implements MonitorManager { @Override public void applicationActivated(ApplicationInfo application) { - if (applicationMonitored(application.getApplicationId())) { + if (applicationMonitoredForHealth(application.getApplicationId())) { ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(application); healthMonitors.put(application.getApplicationId(), monitor); @@ -38,11 +41,9 @@ public class HealthMonitorManager implements MonitorManager { @Override public void applicationRemoved(ApplicationId id) { - if (applicationMonitored(id)) { - ApplicationHealthMonitor monitor = healthMonitors.remove(id); - if (monitor != null) { - monitor.close(); - } + ApplicationHealthMonitor monitor = healthMonitors.remove(id); + if (monitor != null) { + monitor.close(); } } @@ -58,11 +59,15 @@ public class HealthMonitorManager implements MonitorManager { return ServiceStatus.UP; } - return ServiceStatus.NOT_CHECKED; + ApplicationHealthMonitor monitor = healthMonitors.get(applicationId); + if (monitor == null) { + return ServiceStatus.NOT_CHECKED; + } + + return monitor.getStatus(applicationId, clusterId, serviceType, configId); } - private boolean applicationMonitored(ApplicationId id) { - // todo: health-check config server - return false; + private boolean applicationMonitoredForHealth(ApplicationId id) { + return id.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java new file mode 100644 index 00000000000..5f4a1d1024b --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; + +import java.net.URL; + +/** + * @author hakon + */ +class HttpHealthEndpoint implements HealthEndpoint { + private final URL url; + private final ConnectionSocketFactory socketFactory; + + HttpHealthEndpoint(URL url) { + this.url = url; + this.socketFactory = PlainConnectionSocketFactory.getSocketFactory(); + } + + @Override + public URL getStateV1HealthUrl() { + return url; + } + + @Override + public ConnectionSocketFactory getConnectionSocketFactory() { + return socketFactory; + } + + @Override + public void registerListener(ServiceIdentityProvider.Listener listener) { + } + + @Override + public void removeListener(ServiceIdentityProvider.Listener listener) { + } + + @Override + public String toString() { + return url.toString(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java new file mode 100644 index 00000000000..d1198cda78d --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java @@ -0,0 +1,53 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.net.URL; + +/** + * @author hakon + */ +public class HttpsHealthEndpoint implements HealthEndpoint { + private final URL url; + private final HostnameVerifier hostnameVerifier; + private final ServiceIdentityProvider serviceIdentityProvider; + + HttpsHealthEndpoint(URL url, + ServiceIdentityProvider serviceIdentityProvider, + HostnameVerifier hostnameVerifier) { + this.url = url; + this.serviceIdentityProvider = serviceIdentityProvider; + this.hostnameVerifier = hostnameVerifier; + } + + @Override + public URL getStateV1HealthUrl() { + return url; + } + + @Override + public ConnectionSocketFactory getConnectionSocketFactory() { + SSLContext sslContext = serviceIdentityProvider.getIdentitySslContext(); + return new SSLConnectionSocketFactory(sslContext, hostnameVerifier); + } + + @Override + public void registerListener(ServiceIdentityProvider.Listener listener) { + serviceIdentityProvider.addIdentityListener(listener); + } + + @Override + public void removeListener(ServiceIdentityProvider.Listener listener) { + serviceIdentityProvider.removeIdentityListener(listener); + } + + @Override + public String toString() { + return url.toString(); + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java index 51b0503565f..b0fdb14726f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java @@ -6,19 +6,121 @@ import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.internal.ConfigserverUtil; import org.junit.Test; -import static com.yahoo.vespa.applicationmodel.ServiceStatus.NOT_CHECKED; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ApplicationHealthMonitorTest { @Test public void sanityCheck() { - ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring( - ConfigserverUtil.makeExampleConfigServer()); - ServiceStatus status = monitor.getStatus( + MonitorFactory monitorFactory = new MonitorFactory(); + + HealthMonitor monitor1 = mock(HealthMonitor.class); + HealthMonitor monitor2 = mock(HealthMonitor.class); + HealthMonitor monitor3 = mock(HealthMonitor.class); + + monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1); + monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2); + monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3); + + when(monitor1.getStatus()).thenReturn(ServiceStatus.UP); + when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN); + when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED); + + ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring( + ConfigserverUtil.makeExampleConfigServer(), + monitorFactory); + + ServiceStatus status1 = applicationMonitor.getStatus( ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), ConfigServerApplication.CLUSTER_ID, ConfigServerApplication.SERVICE_TYPE, ConfigServerApplication.configIdFrom(0)); - assertEquals(NOT_CHECKED, status); + assertEquals(ServiceStatus.UP, status1); + + ServiceStatus status2 = applicationMonitor.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + ConfigServerApplication.configIdFrom(1)); + assertEquals(ServiceStatus.DOWN, status2); + + ServiceStatus status3 = applicationMonitor.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + ConfigServerApplication.configIdFrom(2)); + assertEquals(ServiceStatus.NOT_CHECKED, status3); + } + + private static class MonitorFactory implements Function { + private Map endpointMonitors = new HashMap<>(); + + public void expectEndpoint(String url, HealthMonitor monitorToReturn) { + endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn)); + } + + @Override + public HealthMonitor apply(HealthEndpoint endpoint) { + String url = endpoint.getStateV1HealthUrl().toString(); + EndpointInfo info = endpointMonitors.get(url); + if (info == null) { + throw new IllegalArgumentException("Endpoint not expected: " + url); + } + + if (info.isEndpointDiscovered()) { + throw new IllegalArgumentException("A HealthMonitor has already been created to " + url); + } + + info.setEndpointDiscovered(true); + + return info.getMonitorToReturn(); + } + } + + private static class EndpointInfo { + private final String url; + private final HealthMonitor monitorToReturn; + + private boolean endpointDiscovered = false; + + private EndpointInfo(String url, HealthMonitor monitorToReturn) { + this.url = url; + this.monitorToReturn = monitorToReturn; + } + + public String getUrl() { + return url; + } + + public boolean isEndpointDiscovered() { + return endpointDiscovered; + } + + public void setEndpointDiscovered(boolean endpointDiscovered) { + this.endpointDiscovered = endpointDiscovered; + } + + public HealthMonitor getMonitorToReturn() { + return monitorToReturn; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointInfo that = (EndpointInfo) o; + return Objects.equals(url, that.url); + } + + @Override + public int hashCode() { + return Objects.hash(url); + } } } \ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java new file mode 100644 index 00000000000..c3e06faaf92 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java @@ -0,0 +1,165 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.health; + +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class HealthClientTest { + @Test + public void successfulRequestResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"code\": \"up\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertTrue(info.isHealthy()); + assertEquals(ServiceStatus.UP, info.toServiceStatus()); + } + + @Test + public void notUpResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"code\": \"initializing\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'initializing'", info.toString()); + } + + @Test + public void noCodeInResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"status\": {\"foo\": \"bar\"},\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'down'", info.toString()); + } + + @Test + public void noStatusInResponse() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.528789829249E9,\n" + + " \"to\": 1.528789889249E9\n" + + " }\n" + + " },\n" + + " \"time\": 1528789889364\n" + + "}"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad health status code 'down'", info.toString()); + } + + @Test + public void badJson() throws IOException { + HealthInfo info = getHealthInfoFromJsonResponse("} foo bar"); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertTrue(info.toString().startsWith("Exception: Unexpected close marker '}': ")); + } + + private HealthInfo getHealthInfoFromJsonResponse(String content) + throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(client.execute(any())).thenReturn(response); + + StatusLine statusLine = mock(StatusLine.class); + when(response.getStatusLine()).thenReturn(statusLine); + + when(statusLine.getStatusCode()).thenReturn(200); + + HttpEntity httpEntity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(httpEntity); + + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) { + healthClient.start(); + + when(httpEntity.getContentLength()).thenReturn((long) content.length()); + return healthClient.getHealthInfo(); + } + } + + @Test + public void testRequestException() throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string")); + + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> "")) { + healthClient.start(); + HealthInfo info = healthClient.getHealthInfo(); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Exception: exception string", info.toString()); + } + } + + @Test + public void testBadHttpResponseCode() + throws IOException { + HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); + CloseableHttpClient client = mock(CloseableHttpClient.class); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(client.execute(any())).thenReturn(response); + + StatusLine statusLine = mock(StatusLine.class); + when(response.getStatusLine()).thenReturn(statusLine); + + when(statusLine.getStatusCode()).thenReturn(500); + + HttpEntity httpEntity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(httpEntity); + + String content = "{}"; + try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) { + healthClient.start(); + + when(httpEntity.getContentLength()).thenReturn((long) content.length()); + HealthInfo info = healthClient.getHealthInfo(); + assertFalse(info.isHealthy()); + assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); + assertEquals("Bad HTTP response status code 500", info.toString()); + } + } +} \ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java index cca1530ad97..2a203027353 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java @@ -4,18 +4,36 @@ package com.yahoo.vespa.service.monitor.internal.health; import com.yahoo.vespa.applicationmodel.ServiceStatus; import org.junit.Test; -import java.net.MalformedURLException; +import java.time.Duration; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class HealthMonitorTest { @Test - public void basicTests() throws MalformedURLException { + public void initiallyDown() { HealthClient healthClient = mock(HealthClient.class); - try (HealthMonitor monitor = new HealthMonitor(healthClient)) { + try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) { monitor.startMonitoring(); - assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus()); + assertEquals(ServiceStatus.DOWN, monitor.getStatus()); + } + } + + @Test + public void eventuallyUp() { + HealthClient healthClient = mock(HealthClient.class); + when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE)); + try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) { + monitor.startMonitoring(); + + while (monitor.getStatus() != ServiceStatus.UP) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // ignore + } + } } } } \ No newline at end of file -- cgit v1.2.3