diff options
author | Håkon Hallingstad <hakon@oath.com> | 2018-12-17 10:34:40 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2018-12-17 10:34:40 +0100 |
commit | 3e01a6396bd6150ec69d272ff8a39a78e2a7e10d (patch) | |
tree | ae9a56e0dda80de12d0b4a40163aa718774b4b63 /service-monitor | |
parent | ffffe2f773f3c6ab82823f41e349033356e45bc7 (diff) |
Use thread pool for health monitoring in service-monitor
This is necessary to avoid using too many threads when monitoring the
host-admin on the tenant Docker hosts.
Diffstat (limited to 'service-monitor')
32 files changed, 1231 insertions, 539 deletions
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 024282d3d21..f559e9336c8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -13,7 +13,7 @@ import java.util.TreeMap; import java.util.logging.Logger; /** - * A non-thread-safe mutable container of ApplicationInfo in the DuperModel, also taking care of listeners on changes. + * A non-thread-safe mutable container of ApplicationInfo, also taking care of listeners on changes. * * @author hakonhall */ diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 8c74fe0396e..09140423010 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; -import com.yahoo.vespa.service.health.ApplicationHealthMonitor; +import com.yahoo.vespa.service.health.StateV1HealthModel; import com.yahoo.vespa.service.model.ModelGenerator; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -107,7 +107,7 @@ public abstract class InfraApplication implements InfraApplicationApi { } private HostInfo makeHostInfo(HostName hostname) { - PortInfo portInfo = new PortInfo(healthPort, ApplicationHealthMonitor.PORT_TAGS_HEALTH); + PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS); Map<String, String> properties = new HashMap<>(); properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, getClusterId().s()); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java new file mode 100644 index 00000000000..80c35851fa5 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +/** + * @author hakonhall + */ +@FunctionalInterface +public interface Cancellable { + void cancel(); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java new file mode 100644 index 00000000000..316b810c682 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java @@ -0,0 +1,103 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import com.yahoo.log.LogLevel; + +import java.time.Duration; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Provides the {@link Cancellable} returned by {@link RunletExecutorImpl#scheduleWithFixedDelay(Runlet, Duration)}, + * and ensuring the correct semantic execution of the {@link Runlet}. + * + * @author hakonhall + */ +class CancellableImpl implements Cancellable, Runnable { + private static final Logger logger = Logger.getLogger(CancellableImpl.class.getName()); + + private final Object monitor = new Object(); + private Runlet runlet; + private Optional<Runnable> periodicExecutionCancellation = Optional.empty(); + private boolean running = false; + private boolean cancelled = false; + + public CancellableImpl(Runlet runlet) { + this.runlet = runlet; + } + + /** + * Provide a way for {@code this} to cancel the periodic execution of {@link #run()}. + * + * <p>Must be called happens-before {@link #cancel()}. + */ + void setPeriodicExecutionCancellationCallback(Runnable periodicExecutionCancellation) { + synchronized (monitor) { + if (cancelled) { + throw new IllegalStateException("Cancellation callback set after cancel()"); + } + + this.periodicExecutionCancellation = Optional.of(periodicExecutionCancellation); + } + } + + /** + * Cancel the execution of the {@link Runlet}. + * + * <ul> + * <li>Either the runlet will not execute any more {@link Runlet#run()}s, and {@link Runlet#close()} + * and then {@code periodicExecutionCancellation} will be called synchronously, or + * <li>{@link #run()} is executing concurrently by another thread {@code T}. The last call to + * {@link Runlet#run()} will be called by {@code T} shortly, is in progress, or has completed. + * Then {@code T} will call {@link Runlet#close()} followed by {@code periodicExecutionCancellation}, + * before the return of {@link #run()}. + * </ul> + * + * <p>{@link #setPeriodicExecutionCancellationCallback(Runnable)} must be called happens-before this method. + */ + @Override + public void cancel() { + synchronized (monitor) { + if (!periodicExecutionCancellation.isPresent()) { + throw new IllegalStateException("setPeriodicExecutionCancellationCallback has not been called before cancel"); + } + + cancelled = true; + if (running) return; + } + + runlet.close(); + periodicExecutionCancellation.get().run(); + } + + /** + * Must be called periodically in happens-before order, but may be called concurrently with + * {@link #setPeriodicExecutionCancellationCallback(Runnable)} and {@link #cancel()}. + */ + @Override + public void run() { + try { + synchronized (monitor) { + if (cancelled) return; + running = true; + } + + runlet.run(); + + synchronized (monitor) { + running = false; + if (!cancelled) return; + + if (!periodicExecutionCancellation.isPresent()) { + // This should be impossible given the implementation of cancel() + throw new IllegalStateException("Cancelled before cancellation callback was set"); + } + } + + runlet.close(); + periodicExecutionCancellation.get().run(); + } catch (Throwable e) { + logger.log(LogLevel.ERROR, "Failed run of periodic execution", e); + } + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java new file mode 100644 index 00000000000..a41bd0f777e --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +/** + * A {@code Runlet} joins {@link AutoCloseable} with {@link Runnable} with the following semantics: + * + * <ul> + * <li>The {@link #run()} method may be called any number of times, followed by a single call to {@link #close()}. + * <li>The caller must ensure the calls are ordered by {@code happens-before}, i.e. the class can be thread-unsafe. + * </ul> + * + * @author hakonhall + */ +public interface Runlet extends AutoCloseable, Runnable { + void run(); + + @Override + void close(); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java new file mode 100644 index 00000000000..4d6dc4b316f --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import java.time.Duration; + +/** + * @author hakonhall + */ +public interface RunletExecutor extends AutoCloseable { + /** + * Execute the task periodically with a fixed delay. + * + * <p>If the execution is {@link Cancellable#cancel() cancelled}, the runlet is {@link Runlet#close() closed} + * as soon as possible. + */ + Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay); + + /** Shuts down and waits for all execution to wind down. */ + @Override + void close(); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java new file mode 100644 index 00000000000..1f647a7fb31 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java @@ -0,0 +1,71 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import com.yahoo.log.LogLevel; + +import java.time.Duration; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +/** + * @author hakonhall + */ +public class RunletExecutorImpl implements RunletExecutor { + private static Logger logger = Logger.getLogger(RunletExecutorImpl.class.getName()); + + // About 'static': Javadoc says "Instances of java.util.Random are threadsafe." + private static final Random random = new Random(); + + private final AtomicInteger executionId = new AtomicInteger(0); + private final ConcurrentHashMap<Integer, CancellableImpl> cancellables = new ConcurrentHashMap<>(); + private final ScheduledThreadPoolExecutor executor; + + public RunletExecutorImpl(int threadPoolSize) { + executor = new ScheduledThreadPoolExecutor(threadPoolSize); + } + + public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) { + Duration initialDelay = Duration.ofMillis((long) random.nextInt((int) delay.toMillis())); + CancellableImpl cancellable = new CancellableImpl(runlet); + ScheduledFuture<?> future = executor.scheduleWithFixedDelay(cancellable, initialDelay.toMillis(), delay.toMillis(), TimeUnit.MILLISECONDS); + cancellable.setPeriodicExecutionCancellationCallback(() -> future.cancel(false)); + Integer id = executionId.incrementAndGet(); + cancellables.put(id, cancellable); + return () -> cancelRunlet(id); + } + + private void cancelRunlet(Integer id) { + CancellableImpl cancellable = cancellables.remove(id); + if (cancellable != null) { + cancellable.cancel(); + } + } + + @Override + public void close() { + // At this point no-one should be scheduling new runlets, so this ought to clear the map. + cancellables.keySet().forEach(this::cancelRunlet); + + if (cancellables.size() > 0) { + throw new IllegalStateException("Runlets scheduled while closing the executor"); + } + + // The cancellables will cancel themselves from the executor only after up-to delay time, + // so wait until all have drained. + while (executor.getQueue().size() > 0) { + try { Thread.sleep(200); } catch (InterruptedException ignored) { } + } + + executor.shutdown(); + try { + executor.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.log(LogLevel.WARNING, "Timed out waiting for termination of executor", e); + } + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java new file mode 100644 index 00000000000..4a382ee8d94 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java @@ -0,0 +1,87 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.BasicHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; + +import java.io.IOException; +import java.net.URL; +import java.time.Duration; + +/** + * @author hakonhall + */ +class ApacheHttpClient implements AutoCloseable { + private final URL url; + private final CloseableHttpClient client; + + @FunctionalInterface + interface Handler<T> { + T handle(CloseableHttpResponse httpResponse) throws Exception; + } + + static CloseableHttpClient makeCloseableHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) { + Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() + .register(url.getProtocol(),socketFactory) + .build(); + + HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry); + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout((int) timeout.toMillis()) // establishment of connection + .setConnectionRequestTimeout((int) timeout.toMillis()) // connection from connection manager + .setSocketTimeout((int) timeout.toMillis()) // waiting for data + .build(); + + ConnectionKeepAliveStrategy keepAliveStrategy = + new DefaultConnectionKeepAliveStrategy() { + @Override + public long getKeepAliveDuration(HttpResponse response, HttpContext context) { + long keepAliveMillis = super.getKeepAliveDuration(response, context); + if (keepAliveMillis == -1) { + keepAliveMillis = keepAlive.toMillis(); + } + return keepAliveMillis; + } + }; + + return HttpClients.custom() + .setKeepAliveStrategy(keepAliveStrategy) + .setConnectionManager(connectionManager) + .disableAutomaticRetries() + .setDefaultRequestConfig(requestConfig) + .build(); + } + + ApacheHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) { + this(url, makeCloseableHttpClient(url, timeout, keepAlive, socketFactory)); + } + + ApacheHttpClient(URL url, CloseableHttpClient client) { + this.url = url; + this.client = client; + } + + <T> T get(Handler<T> handler) throws Exception { + try (CloseableHttpResponse httpResponse = client.execute(new HttpGet(url.toString()))) { + return handler.handle(httpResponse); + } + } + + @Override + public void close() throws IOException { + client.close(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index 2d81474853c..5fab8ac8591 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -2,53 +2,48 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.PortInfo; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.model.ServiceId; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.List; +import java.util.HashSet; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; +import java.util.Set; /** * Responsible for monitoring a whole application using /state/v1/health. * * @author hakon */ -public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { - public static final String PORT_TAG_STATE = "STATE"; - public static final String PORT_TAG_HTTP = "HTTP"; - /** Port tags implying /state/v1/health is served */ - public static final List<String> PORT_TAGS_HEALTH = - Collections.unmodifiableList(Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE)); +class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { + private final ApplicationId applicationId; + private final StateV1HealthModel healthModel; + private final Map<ServiceId, HealthMonitor> monitors = new HashMap<>(); - private final Map<ServiceId, HealthMonitor> healthMonitors; - - public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) { - return startMonitoring(application, HealthMonitor::new); + ApplicationHealthMonitor(ApplicationId applicationId, StateV1HealthModel healthModel) { + this.applicationId = applicationId; + this.healthModel = healthModel; } - /** For testing. */ - static ApplicationHealthMonitor startMonitoring(ApplicationInfo application, - Function<HealthEndpoint, HealthMonitor> mapper) { - return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper)); - } + void monitor(ApplicationInfo applicationInfo) { + if (!applicationInfo.getApplicationId().equals(applicationId)) { + throw new IllegalArgumentException("Monitors " + applicationId + " but was asked to monitor " + applicationInfo.getApplicationId()); + } + + Map<ServiceId, HealthEndpoint> endpoints = healthModel.extractHealthEndpoints(applicationInfo); + + // Remove obsolete monitors + Set<ServiceId> removed = new HashSet<>(monitors.keySet()); + removed.removeAll(endpoints.keySet()); + removed.stream().map(monitors::remove).forEach(HealthMonitor::close); - private ApplicationHealthMonitor(Map<ServiceId, HealthMonitor> healthMonitors) { - this.healthMonitors = healthMonitors; + // Add new monitors. + endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring())); } @Override @@ -62,7 +57,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos ServiceType serviceType, ConfigId configId) { ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId); - HealthMonitor monitor = healthMonitors.get(serviceId); + HealthMonitor monitor = monitors.get(serviceId); if (monitor == null) { return ServiceStatus.NOT_CHECKED; } @@ -72,45 +67,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos @Override public void close() { - healthMonitors.values().forEach(HealthMonitor::close); - healthMonitors.clear(); - } - - private static Map<ServiceId, HealthMonitor> makeHealthMonitors( - ApplicationInfo application, Function<HealthEndpoint, HealthMonitor> monitorFactory) { - Map<ServiceId, HealthMonitor> healthMonitors = new HashMap<>(); - for (HostInfo hostInfo : application.getModel().getHosts()) { - for (ServiceInfo serviceInfo : hostInfo.getServices()) { - for (PortInfo portInfo : serviceInfo.getPorts()) { - maybeCreateHealthMonitor( - application, - hostInfo, - serviceInfo, - portInfo, - monitorFactory) - .ifPresent(healthMonitor -> healthMonitors.put( - ApplicationInstanceGenerator.getServiceId(application, serviceInfo), - healthMonitor)); - } - } - } - return healthMonitors; - } - - private static Optional<HealthMonitor> maybeCreateHealthMonitor( - ApplicationInfo applicationInfo, - HostInfo hostInfo, - ServiceInfo serviceInfo, - PortInfo portInfo, - Function<HealthEndpoint, HealthMonitor> monitorFactory) { - if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) { - HostName hostname = HostName.from(hostInfo.getHostname()); - HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort()); - HealthMonitor healthMonitor = monitorFactory.apply(endpoint); - healthMonitor.startMonitoring(); - return Optional.of(healthMonitor); - } - - return Optional.empty(); + monitors.values().forEach(HealthMonitor::close); + monitors.clear(); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java index 43be236268c..a747753160e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.ApplicationId; /** * @author hakonhall */ @FunctionalInterface interface ApplicationHealthMonitorFactory { - ApplicationHealthMonitor create(ApplicationInfo applicationInfo); + ApplicationHealthMonitor create(ApplicationId applicationId); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java deleted file mode 100644 index 129cc799a25..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.health; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; - -import java.util.function.Function; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Health client - * - * NOT thread-safe. - * - * @author hakon - */ -public class HealthClient implements AutoCloseable { - private static final ObjectMapper mapper = new ObjectMapper(); - private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB - private static final int DEFAULT_TIMEOUT_MILLIS = 1_000; - - private static final ConnectionKeepAliveStrategy KEEP_ALIVE_STRATEGY = - new DefaultConnectionKeepAliveStrategy() { - @Override - public long getKeepAliveDuration(HttpResponse response, HttpContext context) { - long keepAlive = super.getKeepAliveDuration(response, context); - if (keepAlive == -1) { - // Keep connections alive 60 seconds if a keep-alive value - // has not be explicitly set by the server - keepAlive = 60000; - } - return keepAlive; - } - }; - - private final HealthEndpoint endpoint; - private final CloseableHttpClient httpClient; - private final Function<HttpEntity, String> getContentFunction; - - public HealthClient(HealthEndpoint endpoint) { - this(endpoint, - makeCloseableHttpClient(endpoint), - entity -> uncheck(() -> EntityUtils.toString(entity))); - } - - /** For testing. */ - HealthClient(HealthEndpoint endpoint, - CloseableHttpClient httpClient, - Function<HttpEntity, String> getContentFunction) { - this.endpoint = endpoint; - this.httpClient = httpClient; - this.getContentFunction = getContentFunction; - } - - public HealthEndpoint getEndpoint() { - return endpoint; - } - - public HealthInfo getHealthInfo() { - try { - return probeHealth(); - } catch (Exception e) { - return HealthInfo.fromException(e); - } - } - - @Override - public void close() { - try { - httpClient.close(); - } catch (Exception e) { - // ignore - } - } - - private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) { - Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() - .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory()) - .build(); - - HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry); - - RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(DEFAULT_TIMEOUT_MILLIS) // establishment of connection - .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MILLIS) // connection from connection manager - .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data - .build(); - - return HttpClients.custom() - .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY) - .setConnectionManager(connectionManager) - .disableAutomaticRetries() - .setDefaultRequestConfig(requestConfig) - .build(); - } - - private HealthInfo probeHealth() throws Exception { - HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString()); - - CloseableHttpClient httpClient = this.httpClient; - if (httpClient == null) { - throw new IllegalStateException("HTTP client never started or has closed"); - } - - try (CloseableHttpResponse httpResponse = httpClient.execute(httpget)) { - int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode < 200 || httpStatusCode >= 300) { - return HealthInfo.fromBadHttpStatusCode(httpStatusCode); - } - - HttpEntity bodyEntity = httpResponse.getEntity(); - long contentLength = bodyEntity.getContentLength(); - if (contentLength > MAX_CONTENT_LENGTH) { - throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); - } - String body = getContentFunction.apply(bodyEntity); - HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class); - - if (healthResponse.status == null || healthResponse.status.code == null) { - return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); - } else { - return HealthInfo.fromHealthStatusCode(healthResponse.status.code); - } - } - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java index e15d82ea70b..8c4997634a0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java @@ -1,38 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import org.apache.http.conn.socket.ConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import java.net.URL; -import java.util.Collections; - -import static com.yahoo.yolean.Exceptions.uncheck; +import com.yahoo.vespa.service.model.ServiceId; /** + * An endpoint 1-1 with a service and that can be health monitored. + * * @author hakon */ -public interface HealthEndpoint { - - static HealthEndpoint forHttp(HostName hostname, int port) { - URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health")); - return new HttpHealthEndpoint(url); - } - - static HealthEndpoint forHttps(HostName hostname, - int port, - ServiceIdentityProvider serviceIdentityProvider, - AthenzIdentity remoteIdentity) { - URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health")); - HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity)); - return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier); - } - - URL getStateV1HealthUrl(); - ConnectionSocketFactory getConnectionSocketFactory(); +interface HealthEndpoint { + ServiceId getServiceId(); String description(); + HealthMonitor startMonitoring(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java index d6dc1942404..f0e13548f58 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java @@ -1,90 +1,14 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import java.time.Duration; -import java.util.Random; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - /** - * Used to monitor the health of a single URL endpoint. - * - * <p>Must be closed on successful start of monitoring ({} - * - * <p>Thread-safe - * - * @author hakon + * @author hakonhall */ -public class HealthMonitor implements AutoCloseable { - private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName()); - - /** The duration between each health request. */ - private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10); - - // About 'static': Javadoc says "Instances of java.util.Random are threadsafe." - private static final Random random = new Random(); - - private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); - private final HealthClient healthClient; - private final Duration delay; - - private volatile HealthInfo lastHealthInfo = HealthInfo.empty(); - - public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) { - this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY); - } - - /** For testing. */ - HealthMonitor(HealthClient healthClient, Duration delay) { - this.healthClient = healthClient; - this.delay = delay; - } - - public void startMonitoring() { - executor.scheduleWithFixedDelay( - this::updateSynchronously, - initialDelayInMillis(delay.toMillis()), - delay.toMillis(), - TimeUnit.MILLISECONDS); - } - - public ServiceStatus getStatus() { - return lastHealthInfo.toServiceStatus(); - } +interface HealthMonitor extends AutoCloseable { + ServiceStatus getStatus(); @Override - public void close() { - executor.shutdown(); - - try { - executor.awaitTermination(2, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.log(LogLevel.INFO, "Interrupted while waiting for health monitor termination: " + - e.getMessage()); - } - - healthClient.close(); - } - - private long initialDelayInMillis(long maxInitialDelayInMillis) { - if (maxInitialDelayInMillis >= Integer.MAX_VALUE) { - throw new IllegalArgumentException("Max initial delay is out of bounds: " + maxInitialDelayInMillis); - } - - return (long) random.nextInt((int) maxInitialDelayInMillis); - } - - private void updateSynchronously() { - try { - lastHealthInfo = healthClient.getHealthInfo(); - } catch (Throwable t) { - // An uncaught exception will kill the executor.scheduleWithFixedDelay thread! - logger.log(LogLevel.WARNING, "Failed to get health info for " + - healthClient.getEndpoint().description(), t); - } - } + void close(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index e9a5ec314f6..2ad37faf593 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -12,8 +12,10 @@ import com.yahoo.vespa.flags.FeatureFlag; import com.yahoo.vespa.flags.FileFlagSource; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; +import com.yahoo.vespa.service.executor.RunletExecutorImpl; import com.yahoo.vespa.service.manager.MonitorManager; +import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; /** @@ -22,6 +24,28 @@ import java.util.concurrent.ConcurrentHashMap; * @author hakon */ public class HealthMonitorManager implements MonitorManager { + // Weight the following against each other: + // - The number of threads N working on health checking + // - The health request timeout T + // - The max staleness S of the health of an endpoint + // - The ideal staleness I of the health of an endpoint + // + // The largest zone is main.prod.us-west-1: + // - 314 tenant host admins + // - 7 proxy host admins + // - 3 config host admins + // - 3 config servers + // for a total of E = 327 endpoints + private static final int MAX_ENDPOINTS = 500; + private static final Duration HEALTH_REQUEST_TIMEOUT = Duration.ofSeconds(1); + private static final Duration TARGET_HEALTH_STALENESS = Duration.ofSeconds(10); + private static final Duration MAX_HEALTH_STALENESS = Duration.ofSeconds(60); + static final int THREAD_POOL_SIZE = (int) Math.ceil(MAX_ENDPOINTS * HEALTH_REQUEST_TIMEOUT.toMillis() / (double) MAX_HEALTH_STALENESS.toMillis()); + + // Keep connections alive 60 seconds (>=MAX_HEALTH_STALENESS) if a keep-alive value has not be + // explicitly set by the server. + private static final Duration KEEP_ALIVE = Duration.ofSeconds(60); + private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>(); private final DuperModelManager duperModel; private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory; @@ -31,22 +55,29 @@ public class HealthMonitorManager implements MonitorManager { public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) { this(duperModel, new FeatureFlag("healthmonitor-monitorinfra", true, flagSource), - ApplicationHealthMonitor::startMonitoring); + new StateV1HealthModel(TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE))); + } + + private HealthMonitorManager(DuperModelManager duperModel, + FeatureFlag monitorInfra, + StateV1HealthModel healthModel) { + this(duperModel, monitorInfra, id -> new ApplicationHealthMonitor(id, healthModel)); } HealthMonitorManager(DuperModelManager duperModel, FeatureFlag monitorInfra, ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { this.duperModel = duperModel; - this.applicationHealthMonitorFactory = applicationHealthMonitorFactory; this.monitorInfra = monitorInfra; + this.applicationHealthMonitorFactory = applicationHealthMonitorFactory; } @Override public void applicationActivated(ApplicationInfo application) { if (wouldMonitor(application.getApplicationId())) { - ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application); - healthMonitors.put(application.getApplicationId(), monitor); + healthMonitors + .computeIfAbsent(application.getApplicationId(), applicationHealthMonitorFactory::create) + .monitor(application); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java new file mode 100644 index 00000000000..4ed49e17e9f --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.vespa.service.executor.Runlet; + +/** + * A {@link HealthUpdater} will probe the health with {@link #run()}, whose result can be fetched with the + * thread-safe method {@link #getLatestHealthInfo()}. + * + * @author hakonhall + */ +interface HealthUpdater extends Runlet { + HealthInfo getLatestHealthInfo(); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java deleted file mode 100644 index 793c1a93379..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.health; - -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; - -import java.net.URL; - -/** - * @author hakon - */ -class HttpHealthEndpoint implements HealthEndpoint { - private final URL url; - private final ConnectionSocketFactory socketFactory; - - HttpHealthEndpoint(URL url) { - this.url = url; - this.socketFactory = PlainConnectionSocketFactory.getSocketFactory(); - } - - @Override - public URL getStateV1HealthUrl() { - return url; - } - - @Override - public ConnectionSocketFactory getConnectionSocketFactory() { - return socketFactory; - } - - @Override - public String description() { - return url.toString(); - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java deleted file mode 100644 index 42e408256c5..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.health; - -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import java.net.URL; - -/** - * @author hakon - */ -public class HttpsHealthEndpoint implements HealthEndpoint { - private final URL url; - private final HostnameVerifier hostnameVerifier; - private final ServiceIdentityProvider serviceIdentityProvider; - - HttpsHealthEndpoint(URL url, - ServiceIdentityProvider serviceIdentityProvider, - HostnameVerifier hostnameVerifier) { - this.url = url; - this.serviceIdentityProvider = serviceIdentityProvider; - this.hostnameVerifier = hostnameVerifier; - } - - @Override - public URL getStateV1HealthUrl() { - return url; - } - - @Override - public ConnectionSocketFactory getConnectionSocketFactory() { - return new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(serviceIdentityProvider), hostnameVerifier); - } - - @Override - public String description() { - return url.toString(); - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java new file mode 100644 index 00000000000..88aefe42a14 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -0,0 +1,74 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.log.LogLevel; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.net.URL; +import java.time.Duration; +import java.util.function.Function; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * A thread-unsafe /state/v1/health endpoint client. + * + * @author hakonhall + */ +public class StateV1HealthClient implements AutoCloseable { + private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Logger logger = Logger.getLogger(StateV1HealthClient.class.getName()); + private final ApacheHttpClient httpClient; + private final Function<HttpEntity, String> getContentFunction; + + StateV1HealthClient(URL url, Duration requestTimeout, Duration connectionKeepAlive) { + this(new ApacheHttpClient(url, requestTimeout, connectionKeepAlive, PlainConnectionSocketFactory.getSocketFactory()), + entity -> uncheck(() -> EntityUtils.toString(entity))); + } + + StateV1HealthClient(ApacheHttpClient apacheHttpClient, Function<HttpEntity, String> getContentFunction) { + httpClient = apacheHttpClient; + this.getContentFunction = getContentFunction; + } + + HealthInfo get() throws Exception { + return httpClient.get(this::handle); + } + + private HealthInfo handle(CloseableHttpResponse httpResponse) throws IOException { + int httpStatusCode = httpResponse.getStatusLine().getStatusCode(); + if (httpStatusCode < 200 || httpStatusCode >= 300) { + return HealthInfo.fromBadHttpStatusCode(httpStatusCode); + } + + HttpEntity bodyEntity = httpResponse.getEntity(); + long contentLength = bodyEntity.getContentLength(); + if (contentLength > MAX_CONTENT_LENGTH) { + throw new IllegalArgumentException("Content too long: " + contentLength + " bytes"); + } + String body = getContentFunction.apply(bodyEntity); + HealthResponse healthResponse = MAPPER.readValue(body, HealthResponse.class); + + if (healthResponse.status == null || healthResponse.status.code == null) { + return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS); + } else { + return HealthInfo.fromHealthStatusCode(healthResponse.status.code); + } + } + + @Override + public void close() { + try { + httpClient.close(); + } catch (Exception e) { + logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e); + } + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java new file mode 100644 index 00000000000..8eca03c616f --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java @@ -0,0 +1,59 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.service.executor.RunletExecutor; +import com.yahoo.vespa.service.model.ServiceId; + +import java.net.URL; +import java.time.Duration; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +class StateV1HealthEndpoint implements HealthEndpoint { + private final ServiceId serviceId; + private final URL url; + private final Duration requestTimeout; + private final Duration connectionKeepAlive; + private final Duration delay; + private final RunletExecutor executor; + + StateV1HealthEndpoint(ServiceId serviceId, + HostName hostname, + int port, + Duration delay, + Duration requestTimeout, + Duration connectionKeepAlive, + RunletExecutor executor) { + this.serviceId = serviceId; + this.delay = delay; + this.executor = executor; + this.url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health")); + this.requestTimeout = requestTimeout; + this.connectionKeepAlive = connectionKeepAlive; + } + + @Override + public ServiceId getServiceId() { + return serviceId; + } + + @Override + public HealthMonitor startMonitoring() { + StateV1HealthUpdater updater = new StateV1HealthUpdater(url, requestTimeout, connectionKeepAlive); + return new StateV1HealthMonitor(updater, executor, delay); + } + + @Override + public String description() { + return url.toString(); + } + + @Override + public String toString() { + return description(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java new file mode 100644 index 00000000000..5e8979deb9f --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -0,0 +1,74 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.service.executor.RunletExecutor; +import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; +import com.yahoo.vespa.service.model.ServiceId; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author hakonhall + */ +public class StateV1HealthModel implements AutoCloseable { + private static final String PORT_TAG_STATE = "STATE"; + private static final String PORT_TAG_HTTP = "HTTP"; + + /** Port tags implying /state/v1/health is served on HTTP. */ + public static final List<String> HTTP_HEALTH_PORT_TAGS = Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE); + private final Duration targetHealthStaleness; + private final Duration requestTimeout; + private final Duration connectionKeepAlive; + private final RunletExecutor executor; + + StateV1HealthModel(Duration targetHealthStaleness, + Duration requestTimeout, + Duration connectionKeepAlive, + RunletExecutor executor) { + this.targetHealthStaleness = targetHealthStaleness; + this.requestTimeout = requestTimeout; + this.connectionKeepAlive = connectionKeepAlive; + this.executor = executor; + } + + Map<ServiceId, HealthEndpoint> extractHealthEndpoints(ApplicationInfo application) { + Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>(); + + for (HostInfo hostInfo : application.getModel().getHosts()) { + HostName hostname = HostName.from(hostInfo.getHostname()); + for (ServiceInfo serviceInfo : hostInfo.getServices()) { + ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); + for (PortInfo portInfo : serviceInfo.getPorts()) { + if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) { + StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint( + serviceId, + hostname, + portInfo.getPort(), + targetHealthStaleness, + requestTimeout, + connectionKeepAlive, + executor); + endpoints.put(serviceId, endpoint); + break; // Avoid >1 endpoints per serviceId + } + } + } + } + + return endpoints; + } + + @Override + public void close() { + executor.close(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java new file mode 100644 index 00000000000..d37797c7be9 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.executor.Cancellable; +import com.yahoo.vespa.service.executor.RunletExecutor; + +import java.time.Duration; + +/** + * Used to monitor the health of a single URL endpoint. + * + * @author hakon + */ +class StateV1HealthMonitor implements HealthMonitor { + private final StateV1HealthUpdater updater; + private final Cancellable periodicExecution; + + StateV1HealthMonitor(StateV1HealthUpdater updater, RunletExecutor executor, Duration delay) { + this.updater = updater; + this.periodicExecution = executor.scheduleWithFixedDelay(updater, delay); + } + + @Override + public ServiceStatus getStatus() { + return updater.getLatestHealthInfo().toServiceStatus(); + } + + @Override + public void close() { + periodicExecution.cancel(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java new file mode 100644 index 00000000000..011ec3b3212 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import java.net.URL; +import java.time.Duration; + +/** + * @author hakonhall + */ +class StateV1HealthUpdater implements HealthUpdater { + private final StateV1HealthClient healthClient; + + private volatile HealthInfo lastHealthInfo = HealthInfo.empty(); + + StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) { + this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive)); + } + + StateV1HealthUpdater(StateV1HealthClient healthClient) { + this.healthClient = healthClient; + } + + @Override + public HealthInfo getLatestHealthInfo() { + return lastHealthInfo; + } + + @Override + public void run() { + try { + lastHealthInfo = healthClient.get(); + } catch (Exception e) { + lastHealthInfo = HealthInfo.fromException(e); + } + } + + @Override + public void close() { + healthClient.close(); + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java new file mode 100644 index 00000000000..eb6f92d928c --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java @@ -0,0 +1,79 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class CancellableImplTest { + private final TestExecutor executor = new TestExecutor(); + private final TestRunlet runlet = new TestRunlet(); + private final Cancellable cancellable = executor.scheduleWithFixedDelay(runlet, Duration.ofSeconds(1)); + + @After + public void tearDown() { + executor.close(); + } + + @Before + public void setUp() { + assertEquals(0, runlet.getRunsStarted()); + executor.runToCompletion(1); + assertEquals(1, runlet.getRunsStarted()); + executor.runToCompletion(2); + assertEquals(2, runlet.getRunsStarted()); + assertTrue(executor.isExecutionRunning()); + assertFalse(runlet.isClosed()); + assertTrue(executor.isExecutionRunning()); + assertFalse(runlet.isClosed()); + } + + @Test + public void testCancelWhileIdle() { + // Cancel while runlet is not running and verify closure and executor cancellation + cancellable.cancel(); + assertFalse(executor.isExecutionRunning()); + assertTrue(runlet.isClosed()); + + // Ensure a spurious run is ignored. + executor.runAsync(); + executor.runToCompletion(3); + assertEquals(2, runlet.getRunsStarted()); + } + + @Test + public void testCancelWhileRunning() { + // halt execution in runlet + runlet.shouldWaitInRun(true); + executor.runAsync(); + runlet.waitUntilInRun(); + assertEquals(3, runlet.getRunsStarted()); + assertEquals(2, runlet.getRunsCompleted()); + assertTrue(executor.isExecutionRunning()); + assertFalse(runlet.isClosed()); + + // Cancel now + cancellable.cancel(); + assertTrue(executor.isExecutionRunning()); + assertFalse(runlet.isClosed()); + + // Complete the runlet.run(), and verify the close and executor cancellation takes effect + runlet.shouldWaitInRun(false); + executor.waitUntilRunCompleted(3); + assertFalse(executor.isExecutionRunning()); + assertTrue(runlet.isClosed()); + + // Ensure a spurious run is ignored. + executor.runToCompletion(4); + assertEquals(3, runlet.getRunsStarted()); + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java new file mode 100644 index 00000000000..9828d6300ed --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java @@ -0,0 +1,71 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import org.junit.After; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class RunletExecutorImplTest { + private final RunletExecutorImpl executor = new RunletExecutorImpl(2); + + @After + public void tearDown() { + executor.close(); + } + + @Test + public void testAFewCancellations() { + for (int i = 0; i < 10; ++i) { + TestRunlet runlet = new TestRunlet(); + Cancellable cancellable = schedule(runlet); + runlet.waitUntilCompleted(5); + cancellable.cancel(); + runlet.waitUntilClosed(); + } + } + + @Test + public void testCongestedThreadPool() { + TestRunlet runlet1 = new TestRunlet(); + runlet1.shouldWaitInRun(true); + Cancellable cancellable1 = schedule(runlet1); + runlet1.waitUntilInRun(); + + TestRunlet runlet2 = new TestRunlet(); + runlet2.shouldWaitInRun(true); + Cancellable cancellable2 = schedule(runlet2); + runlet2.waitUntilInRun(); + + TestRunlet runlet3 = new TestRunlet(); + Cancellable cancellable3 = schedule(runlet3); + try { Thread.sleep(10); } catch (InterruptedException ignored) { } + assertEquals(0, runlet3.getRunsStarted()); + + cancellable3.cancel(); + assertTrue(runlet3.isClosed()); + assertEquals(0, runlet3.getRunsStarted()); + + runlet1.shouldWaitInRun(false); + runlet2.shouldWaitInRun(false); + cancellable1.cancel(); + cancellable2.cancel(); + } + + @Test + public void testWithoutCancellation() { + TestRunlet runlet = new TestRunlet(); + Cancellable toBeIgnored = schedule(runlet); + runlet.waitUntilCompleted(2); + } + + private Cancellable schedule(Runlet runlet) { + return executor.scheduleWithFixedDelay(runlet, Duration.ofMillis(20)); + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java new file mode 100644 index 00000000000..c40fc03ea00 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java @@ -0,0 +1,105 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * @author hakonhall + */ +public class TestExecutor implements RunletExecutor { + private List<Thread> threads = new ArrayList<>(); + + private Runlet runlet; + private CancellableImpl cancellable; + + private final Object monitor = new Object(); + private boolean afterRun = false; + private boolean waitAfterRun = false; + private int runsCompleted = 0; + + private final Runnable cancelExecution = () -> executionRunning = false; + private volatile boolean executionRunning = true; + + @Override + public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) { + if (this.runlet != null) { + throw new IllegalStateException("TestExecutor only supports execution of one runlet"); + } + + this.runlet = runlet; + this.cancellable = new CancellableImpl(runlet); + this.cancellable.setPeriodicExecutionCancellationCallback(cancelExecution); + return this::cancel; + } + + private void cancel() { + cancellable.cancel(); + } + + boolean isExecutionRunning() { + return executionRunning; + } + + void runAsync() { + Thread thread = new Thread(this::threadMain); + thread.start(); + threads.add(thread); + } + + void runToCompletion(int run) { + runAsync(); + waitUntilRunCompleted(run); + } + + private void threadMain() { + cancellable.run(); + + synchronized (monitor) { + ++runsCompleted; + afterRun = true; + monitor.notifyAll(); + + while (waitAfterRun) { + monitor.notifyAll(); + } + afterRun = false; + } + } + + void setWaitAfterRun(boolean waitAfterRun) { + synchronized (monitor) { + this.waitAfterRun = waitAfterRun; + } + } + + void waitUntilAfterRun() { + synchronized (monitor) { + while (!afterRun) { + uncheckedWait(); + } + } + } + + void waitUntilRunCompleted(int run) { + synchronized (monitor) { + while (runsCompleted < run) { + uncheckedWait(); + } + } + } + + void uncheckedWait() { + try { + monitor.wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + threads.forEach(thread -> { try { thread.join(); } catch (InterruptedException ignored) {} }); + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java new file mode 100644 index 00000000000..7e671dccd96 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java @@ -0,0 +1,98 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.executor; + +/** + * @author hakonhall + */ +public class TestRunlet implements Runlet { + private final Object monitor = new Object(); + private boolean running = false; + private boolean shouldWaitInRun = false; + private boolean closed = false; + private int runsStarted = 0; + private int runsCompleted = 0; + + int getRunsStarted() { + synchronized (monitor) { + return runsStarted; + } + } + + int getRunsCompleted() { + return runsCompleted; + } + + boolean isClosed() { + synchronized (monitor) { + return closed; + } + } + + void shouldWaitInRun(boolean value) { + synchronized (monitor) { + shouldWaitInRun = value; + monitor.notifyAll(); + } + } + + void waitUntilInRun() { + synchronized (monitor) { + while (!running) { + uncheckedWait(); + } + } + } + + void waitUntilCompleted(int runsCompleted) { + synchronized (monitor) { + while (this.runsCompleted < runsCompleted) { + uncheckedWait(); + } + } + } + + void waitUntilClosed() { + synchronized (monitor) { + while (!closed) { + uncheckedWait(); + } + } + } + + @Override + public void run() { + synchronized (monitor) { + if (closed) { + throw new IllegalStateException("run after close"); + } + + ++runsStarted; + running = true; + monitor.notifyAll(); + + while (shouldWaitInRun) { + uncheckedWait(); + } + + ++runsCompleted; + running = false; + monitor.notifyAll(); + } + } + + @Override + public void close() { + synchronized (monitor) { + closed = true; + monitor.notifyAll(); + } + } + + private void uncheckedWait() { + try { + monitor.wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index 0dfca12099e..821f5282998 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -1,47 +1,92 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.model.ServiceId; import com.yahoo.vespa.service.monitor.ConfigserverUtil; import org.junit.Test; import java.util.HashMap; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ApplicationHealthMonitorTest { private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); @Test - public void sanityCheck() { - MonitorFactory monitorFactory = new MonitorFactory(); - + public void activationAndRemoval() { HealthMonitor monitor1 = mock(HealthMonitor.class); HealthMonitor monitor2 = mock(HealthMonitor.class); HealthMonitor monitor3 = mock(HealthMonitor.class); - monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1); - monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2); - monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3); + ApplicationInfo configServer = ConfigserverUtil.makeExampleConfigServer(); + StateV1HealthModel model = mock(StateV1HealthModel.class); + ApplicationHealthMonitor applicationMonitor = new ApplicationHealthMonitor(configServer.getApplicationId(), model); + + // Activate with cfg1-2 + HealthEndpoint endpoint1 = mock(HealthEndpoint.class); + HealthEndpoint endpoint2 = mock(HealthEndpoint.class); + Map<ServiceId, HealthEndpoint> initialEndpoints = new HashMap<>(); + initialEndpoints.put(serviceIdOf("cfg1"), endpoint1); + initialEndpoints.put(serviceIdOf("cfg2"), endpoint2); + + when(model.extractHealthEndpoints(configServer)).thenReturn(initialEndpoints); + when(endpoint1.startMonitoring()).thenReturn(monitor1); + when(endpoint2.startMonitoring()).thenReturn(monitor2); + applicationMonitor.monitor(configServer); + + verify(endpoint1, times(1)).startMonitoring(); + verify(endpoint2, times(1)).startMonitoring(); when(monitor1.getStatus()).thenReturn(ServiceStatus.UP); when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN); - when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED); - - ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring( - ConfigserverUtil.makeExampleConfigServer(), - monitorFactory); + when(monitor3.getStatus()).thenReturn(ServiceStatus.UP); assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1")); assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2")); assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg3")); + + // Update application to contain cfg2-3 + HealthEndpoint endpoint3 = mock(HealthEndpoint.class); + when(endpoint3.startMonitoring()).thenReturn(monitor3); + Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>(); + endpoints.put(serviceIdOf("cfg2"), endpoint2); + endpoints.put(serviceIdOf("cfg3"), endpoint3); + when(model.extractHealthEndpoints(configServer)).thenReturn(endpoints); + applicationMonitor.monitor(configServer); + + // Only monitor1 has been removed and had its close called + verify(monitor1, times(1)).close(); + verify(monitor2, never()).close(); + verify(monitor3, never()).close(); + + // Only endpoint3 started monitoring from last monitor() + verify(endpoint1, times(1)).startMonitoring(); + verify(endpoint2, times(1)).startMonitoring(); + verify(endpoint3, times(1)).startMonitoring(); + + // Now cfg1 will be NOT_CHECKED, while cfg3 should be UP. + assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg1")); + assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2")); + assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg3")); + + applicationMonitor.close(); + } + + private ServiceId serviceIdOf(String hostname) { + return new ServiceId(configServerApplication.getApplicationId(), + configServerApplication.getClusterId(), + configServerApplication.getServiceType(), + configServerApplication.configIdFor(HostName.from(hostname))); } private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) { @@ -51,70 +96,4 @@ public class ApplicationHealthMonitorTest { configServerApplication.getServiceType(), configServerApplication.configIdFor(HostName.from(hostname))); } - - private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> { - private Map<String, EndpointInfo> endpointMonitors = new HashMap<>(); - - public void expectEndpoint(String url, HealthMonitor monitorToReturn) { - endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn)); - } - - @Override - public HealthMonitor apply(HealthEndpoint endpoint) { - String url = endpoint.getStateV1HealthUrl().toString(); - EndpointInfo info = endpointMonitors.get(url); - if (info == null) { - throw new IllegalArgumentException("Endpoint not expected: " + url); - } - - if (info.isEndpointDiscovered()) { - throw new IllegalArgumentException("A HealthMonitor has already been created to " + url); - } - - info.setEndpointDiscovered(true); - - return info.getMonitorToReturn(); - } - } - - private static class EndpointInfo { - private final String url; - private final HealthMonitor monitorToReturn; - - private boolean endpointDiscovered = false; - - private EndpointInfo(String url, HealthMonitor monitorToReturn) { - this.url = url; - this.monitorToReturn = monitorToReturn; - } - - public String getUrl() { - return url; - } - - public boolean isEndpointDiscovered() { - return endpointDiscovered; - } - - public void setEndpointDiscovered(boolean endpointDiscovered) { - this.endpointDiscovered = endpointDiscovered; - } - - public HealthMonitor getMonitorToReturn() { - return monitorToReturn; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - EndpointInfo that = (EndpointInfo) o; - return Objects.equals(url, that.url); - } - - @Override - public int hashCode() { - return Objects.hash(url); - } - } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index f420f5c1284..86b0ee4a8f3 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -49,7 +49,9 @@ public class HealthMonitorManagerTest { when(monitorInfra.value()).thenReturn(false); ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer(); manager.applicationActivated(applicationInfo); + verify(monitor, times(1)).monitor(applicationInfo); manager.applicationRemoved(applicationInfo.getApplicationId()); + verify(monitor, times(1)).close(); } @Test @@ -73,7 +75,7 @@ public class HealthMonitorManagerTest { ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); manager.applicationActivated(proxyHostApplicationInfo); - verify(monitorFactory, never()).create(proxyHostApplicationInfo); + verify(monitorFactory, never()).create(proxyHostApplicationInfo.getApplicationId()); assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1"); } @@ -88,7 +90,7 @@ public class HealthMonitorManagerTest { ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); manager.applicationActivated(proxyHostApplicationInfo); - verify(monitorFactory, times(1)).create(proxyHostApplicationInfo); + verify(monitorFactory, times(1)).create(proxyHostApplicationInfo.getApplicationId()); when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP); assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1"); @@ -98,6 +100,11 @@ public class HealthMonitorManagerTest { assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1"); } + @Test + public void threadPoolSize() { + assertEquals(9, HealthMonitorManager.THREAD_POOL_SIZE); + } + private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) { ServiceStatus actual = manager.getStatus( infraApplication.getApplicationId(), diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java deleted file mode 100644 index 94ba4726ad0..00000000000 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.health; - -import com.yahoo.vespa.applicationmodel.ServiceStatus; -import org.junit.Test; - -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class HealthMonitorTest { - @Test - public void initiallyDown() { - HealthClient healthClient = mock(HealthClient.class); - try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) { - monitor.startMonitoring(); - assertEquals(ServiceStatus.DOWN, monitor.getStatus()); - } - } - - @Test - public void eventuallyUp() { - HealthClient healthClient = mock(HealthClient.class); - when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE)); - try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) { - monitor.startMonitoring(); - - while (monitor.getStatus() != ServiceStatus.UP) { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - // ignore - } - } - } - } -}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java new file mode 100644 index 00000000000..480691772bb --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -0,0 +1,66 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.duper.ProxyHostApplication; +import com.yahoo.vespa.service.executor.Cancellable; +import com.yahoo.vespa.service.executor.RunletExecutor; +import com.yahoo.vespa.service.model.ServiceId; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class StateV1HealthModelTest { + private RunletExecutor executor = mock(RunletExecutor.class); + private Duration healthStaleness = Duration.ofSeconds(1); + private Duration requestTimeout = Duration.ofSeconds(2); + private Duration keepAlive = Duration.ofSeconds(3); + private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor); + private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::from).collect(Collectors.toList()); + private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); + private final Map<ServiceId, HealthEndpoint> endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo); + + @Test + public void test() { + assertEquals(2, endpoints.size()); + + ApplicationId applicationId = ApplicationId.from("hosted-vespa", "proxy-host", "default"); + ClusterId clusterId = new ClusterId("proxy-host"); + ServiceId hostAdmin1 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host1")); + ServiceId hostAdmin2 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host2")); + + HealthEndpoint endpoint1 = endpoints.get(hostAdmin1); + assertNotNull(endpoint1); + assertEquals("http://host1:8080/state/v1/health", endpoint1.description()); + + HealthEndpoint endpoint2 = endpoints.get(hostAdmin2); + assertNotNull(endpoint2); + assertEquals("http://host2:8080/state/v1/health", endpoint2.description()); + + Cancellable cancellable = mock(Cancellable.class); + when(executor.scheduleWithFixedDelay(any(), any())).thenReturn(cancellable); + try (HealthMonitor healthMonitor = endpoint1.startMonitoring()) { + assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus()); + } + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java new file mode 100644 index 00000000000..c892118990f --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java @@ -0,0 +1,37 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.executor.RunletExecutor; +import com.yahoo.vespa.service.executor.RunletExecutorImpl; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StateV1HealthMonitorTest { + @Test + public void downThenUpThenDown() throws Exception { + StateV1HealthClient client = mock(StateV1HealthClient.class); + when(client.get()).thenReturn(HealthInfo.empty()); + + StateV1HealthUpdater updater = new StateV1HealthUpdater(client); + RunletExecutor executor = new RunletExecutorImpl(2); + try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) { + assertEquals(ServiceStatus.DOWN, monitor.getStatus()); + + when(client.get()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE)); + while (monitor.getStatus() != ServiceStatus.UP) { + try { Thread.sleep(2); } catch (InterruptedException ignored) { } + } + + when(client.get()).thenReturn(HealthInfo.fromException(new IllegalStateException("foo"))); + while (monitor.getStatus() != ServiceStatus.DOWN) { + try { Thread.sleep(2); } catch (InterruptedException ignored) { } + } + } + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java index 157b5565071..e7b7a829dac 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java @@ -1,16 +1,18 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.net.URL; +import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -19,7 +21,14 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class HealthClientTest { +public class StateV1HealthUpdaterTest { + private URL url; + + @Before + public void setUp() throws Exception{ + url = new URL("http://host.com:19071"); + } + @Test public void successfulRequestResponse() throws IOException { HealthInfo info = getHealthInfoFromJsonResponse("{\n" + @@ -96,7 +105,6 @@ public class HealthClientTest { private HealthInfo getHealthInfoFromJsonResponse(String content) throws IOException { - HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); CloseableHttpClient client = mock(CloseableHttpClient.class); CloseableHttpResponse response = mock(CloseableHttpResponse.class); @@ -110,22 +118,22 @@ public class HealthClientTest { HttpEntity httpEntity = mock(HttpEntity.class); when(response.getEntity()).thenReturn(httpEntity); - try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) { - + try (StateV1HealthUpdater updater = makeUpdater(client, entry -> content)) { when(httpEntity.getContentLength()).thenReturn((long) content.length()); - return healthClient.getHealthInfo(); + updater.run(); + return updater.getLatestHealthInfo(); } } @Test public void testRequestException() throws IOException { - HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); CloseableHttpClient client = mock(CloseableHttpClient.class); when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string")); - try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> "")) { - HealthInfo info = healthClient.getHealthInfo(); + try (StateV1HealthUpdater updater = makeUpdater(client, entry -> "")) { + updater.run(); + HealthInfo info = updater.getLatestHealthInfo(); assertFalse(info.isHealthy()); assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); assertEquals("Exception: exception string", info.toString()); @@ -135,7 +143,6 @@ public class HealthClientTest { @Test public void testBadHttpResponseCode() throws IOException { - HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071); CloseableHttpClient client = mock(CloseableHttpClient.class); CloseableHttpResponse response = mock(CloseableHttpResponse.class); @@ -150,13 +157,19 @@ public class HealthClientTest { when(response.getEntity()).thenReturn(httpEntity); String content = "{}"; - try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) { - + try (HealthUpdater updater = makeUpdater(client, entry -> content)) { when(httpEntity.getContentLength()).thenReturn((long) content.length()); - HealthInfo info = healthClient.getHealthInfo(); + updater.run(); + HealthInfo info = updater.getLatestHealthInfo(); assertFalse(info.isHealthy()); assertEquals(ServiceStatus.DOWN, info.toServiceStatus()); assertEquals("Bad HTTP response status code 500", info.toString()); } } + + private StateV1HealthUpdater makeUpdater(CloseableHttpClient client, Function<HttpEntity, String> getContentFunction) { + ApacheHttpClient apacheHttpClient = new ApacheHttpClient(url, client); + StateV1HealthClient healthClient = new StateV1HealthClient(apacheHttpClient, getContentFunction); + return new StateV1HealthUpdater(healthClient); + } }
\ No newline at end of file |