Use thread pool for health monitoring in service-monitor

This is necessary to avoid using too many threads when monitoring the
host-admin on the tenant Docker hosts.

22 files changed, 682 insertions, 406 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index 024282d3d21..f559e9336c8 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -13,7 +13,7 @@ import java.util.TreeMap;
 import java.util.logging.Logger;
 
 /**
- * A non-thread-safe mutable container of ApplicationInfo in the DuperModel, also taking care of listeners on changes.
+ * A non-thread-safe mutable container of ApplicationInfo, also taking care of listeners on changes.
  *
  * @author hakonhall
  */
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
index 8c74fe0396e..09140423010 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
@@ -17,7 +17,7 @@ import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
-import com.yahoo.vespa.service.health.ApplicationHealthMonitor;
+import com.yahoo.vespa.service.health.StateV1HealthModel;
 import com.yahoo.vespa.service.model.ModelGenerator;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
@@ -107,7 +107,7 @@ public abstract class InfraApplication implements InfraApplicationApi {
     }
 
     private HostInfo makeHostInfo(HostName hostname) {
-        PortInfo portInfo = new PortInfo(healthPort, ApplicationHealthMonitor.PORT_TAGS_HEALTH);
+        PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS);
 
         Map<String, String> properties = new HashMap<>();
         properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, getClusterId().s());
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java
new file mode 100644
index 00000000000..80c35851fa5
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java
@@ -0,0 +1,10 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * @author hakonhall
+ */
+@FunctionalInterface
+public interface Cancellable {
+    void cancel();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java
new file mode 100644
index 00000000000..316b810c682
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java
@@ -0,0 +1,103 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import com.yahoo.log.LogLevel;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Provides the {@link Cancellable} returned by {@link RunletExecutorImpl#scheduleWithFixedDelay(Runlet, Duration)},
+ * and ensuring the correct semantic execution of the {@link Runlet}.
+ *
+ * @author hakonhall
+ */
+class CancellableImpl implements Cancellable, Runnable {
+    private static final Logger logger = Logger.getLogger(CancellableImpl.class.getName());
+
+    private final Object monitor = new Object();
+    private Runlet runlet;
+    private Optional<Runnable> periodicExecutionCancellation = Optional.empty();
+    private boolean running = false;
+    private boolean cancelled = false;
+
+    public CancellableImpl(Runlet runlet) {
+        this.runlet = runlet;
+    }
+
+    /**
+     * Provide a way for {@code this} to cancel the periodic execution of {@link #run()}.
+     *
+     * <p>Must be called happens-before {@link #cancel()}.
+     */
+    void setPeriodicExecutionCancellationCallback(Runnable periodicExecutionCancellation) {
+        synchronized (monitor) {
+            if (cancelled) {
+                throw new IllegalStateException("Cancellation callback set after cancel()");
+            }
+
+            this.periodicExecutionCancellation = Optional.of(periodicExecutionCancellation);
+        }
+    }
+
+    /**
+     * Cancel the execution of the {@link Runlet}.
+     *
+     * <ul>
+     *     <li>Either the runlet will not execute any more {@link Runlet#run()}s, and {@link Runlet#close()}
+     *     and then {@code periodicExecutionCancellation} will be called synchronously, or
+     *     <li>{@link #run()} is executing concurrently by another thread {@code T}. The last call to
+     *     {@link Runlet#run()} will be called by {@code T} shortly, is in progress, or has completed.
+     *     Then {@code T} will call {@link Runlet#close()} followed by {@code periodicExecutionCancellation},
+     *     before the return of {@link #run()}.
+     * </ul>
+     *
+     * <p>{@link #setPeriodicExecutionCancellationCallback(Runnable)} must be called happens-before this method.
+     */
+    @Override
+    public void cancel() {
+        synchronized (monitor) {
+            if (!periodicExecutionCancellation.isPresent()) {
+                throw new IllegalStateException("setPeriodicExecutionCancellationCallback has not been called before cancel");
+            }
+
+            cancelled = true;
+            if (running) return;
+        }
+
+        runlet.close();
+        periodicExecutionCancellation.get().run();
+    }
+
+    /**
+     * Must be called periodically in happens-before order, but may be called concurrently with
+     * {@link #setPeriodicExecutionCancellationCallback(Runnable)} and {@link #cancel()}.
+     */
+    @Override
+    public void run() {
+        try {
+            synchronized (monitor) {
+                if (cancelled) return;
+                running = true;
+            }
+
+            runlet.run();
+
+            synchronized (monitor) {
+                running = false;
+                if (!cancelled) return;
+
+                if (!periodicExecutionCancellation.isPresent()) {
+                    // This should be impossible given the implementation of cancel()
+                    throw new IllegalStateException("Cancelled before cancellation callback was set");
+                }
+            }
+
+            runlet.close();
+            periodicExecutionCancellation.get().run();
+        } catch (Throwable e) {
+            logger.log(LogLevel.ERROR, "Failed run of periodic execution", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java
new file mode 100644
index 00000000000..a41bd0f777e
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java
@@ -0,0 +1,19 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * A {@code Runlet} joins {@link AutoCloseable} with {@link Runnable} with the following semantics:
+ *
+ * <ul>
+ *     <li>The {@link #run()} method may be called any number of times, followed by a single call to {@link #close()}.
+ *     <li>The caller must ensure the calls are ordered by {@code happens-before}, i.e. the class can be thread-unsafe.
+ * </ul>
+ *
+ * @author hakonhall
+ */
+public interface Runlet extends AutoCloseable, Runnable {
+    void run();
+
+    @Override
+    void close();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java
new file mode 100644
index 00000000000..4d6dc4b316f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java
@@ -0,0 +1,21 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+public interface RunletExecutor extends AutoCloseable {
+    /**
+     * Execute the task periodically with a fixed delay.
+     *
+     * <p>If the execution is {@link Cancellable#cancel() cancelled}, the runlet is {@link Runlet#close() closed}
+     * as soon as possible.
+     */
+    Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay);
+
+    /** Shuts down and waits for all execution to wind down. */
+    @Override
+    void close();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java
new file mode 100644
index 00000000000..1f647a7fb31
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java
@@ -0,0 +1,71 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import com.yahoo.log.LogLevel;
+
+import java.time.Duration;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+/**
+ * @author hakonhall
+ */
+public class RunletExecutorImpl implements RunletExecutor {
+    private static Logger logger = Logger.getLogger(RunletExecutorImpl.class.getName());
+
+    // About 'static': Javadoc says "Instances of java.util.Random are threadsafe."
+    private static final Random random = new Random();
+
+    private final AtomicInteger executionId = new AtomicInteger(0);
+    private final ConcurrentHashMap<Integer, CancellableImpl> cancellables = new ConcurrentHashMap<>();
+    private final ScheduledThreadPoolExecutor executor;
+
+    public RunletExecutorImpl(int threadPoolSize) {
+        executor = new ScheduledThreadPoolExecutor(threadPoolSize);
+    }
+
+    public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) {
+        Duration initialDelay = Duration.ofMillis((long) random.nextInt((int) delay.toMillis()));
+        CancellableImpl cancellable = new CancellableImpl(runlet);
+        ScheduledFuture<?> future = executor.scheduleWithFixedDelay(cancellable, initialDelay.toMillis(), delay.toMillis(), TimeUnit.MILLISECONDS);
+        cancellable.setPeriodicExecutionCancellationCallback(() -> future.cancel(false));
+        Integer id = executionId.incrementAndGet();
+        cancellables.put(id, cancellable);
+        return () -> cancelRunlet(id);
+    }
+
+    private void cancelRunlet(Integer id) {
+        CancellableImpl cancellable = cancellables.remove(id);
+        if (cancellable != null) {
+            cancellable.cancel();
+        }
+    }
+
+    @Override
+    public void close() {
+        // At this point no-one should be scheduling new runlets, so this ought to clear the map.
+        cancellables.keySet().forEach(this::cancelRunlet);
+
+        if (cancellables.size() > 0) {
+            throw new IllegalStateException("Runlets scheduled while closing the executor");
+        }
+
+        // The cancellables will cancel themselves from the executor only after up-to delay time,
+        // so wait until all have drained.
+        while (executor.getQueue().size() > 0) {
+            try { Thread.sleep(200); } catch (InterruptedException ignored) { }
+        }
+
+        executor.shutdown();
+        try {
+            executor.awaitTermination(10, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            logger.log(LogLevel.WARNING, "Timed out waiting for termination of executor", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java
new file mode 100644
index 00000000000..4a382ee8d94
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java
@@ -0,0 +1,87 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.ConnectionKeepAliveStrategy;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.protocol.HttpContext;
+
+import java.io.IOException;
+import java.net.URL;
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+class ApacheHttpClient implements AutoCloseable {
+    private final URL url;
+    private final CloseableHttpClient client;
+
+    @FunctionalInterface
+    interface Handler<T> {
+        T handle(CloseableHttpResponse httpResponse) throws Exception;
+    }
+
+    static CloseableHttpClient makeCloseableHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) {
+        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
+                .register(url.getProtocol(),socketFactory)
+                .build();
+
+        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry);
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout((int) timeout.toMillis()) // establishment of connection
+                .setConnectionRequestTimeout((int) timeout.toMillis())  // connection from connection manager
+                .setSocketTimeout((int) timeout.toMillis()) // waiting for data
+                .build();
+
+        ConnectionKeepAliveStrategy keepAliveStrategy =
+                new DefaultConnectionKeepAliveStrategy() {
+                    @Override
+                    public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
+                        long keepAliveMillis = super.getKeepAliveDuration(response, context);
+                        if (keepAliveMillis == -1) {
+                            keepAliveMillis = keepAlive.toMillis();
+                        }
+                        return keepAliveMillis;
+                    }
+                };
+
+        return HttpClients.custom()
+                .setKeepAliveStrategy(keepAliveStrategy)
+                .setConnectionManager(connectionManager)
+                .disableAutomaticRetries()
+                .setDefaultRequestConfig(requestConfig)
+                .build();
+    }
+
+    ApacheHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) {
+        this(url, makeCloseableHttpClient(url, timeout, keepAlive, socketFactory));
+    }
+
+    ApacheHttpClient(URL url, CloseableHttpClient client) {
+        this.url = url;
+        this.client = client;
+    }
+
+    <T> T get(Handler<T> handler) throws Exception {
+        try (CloseableHttpResponse httpResponse = client.execute(new HttpGet(url.toString()))) {
+            return handler.handle(httpResponse);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        client.close();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
index 2d81474853c..5fab8ac8591 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
@@ -2,53 +2,48 @@
 package com.yahoo.vespa.service.health;
 
 import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.model.api.HostInfo;
-import com.yahoo.config.model.api.PortInfo;
-import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
-import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
-import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
 import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
+import java.util.HashSet;
 import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
+import java.util.Set;
 
 /**
  * Responsible for monitoring a whole application using /state/v1/health.
  *
  * @author hakon
  */
-public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
-    public static final String PORT_TAG_STATE = "STATE";
-    public static final String PORT_TAG_HTTP = "HTTP";
-    /** Port tags implying /state/v1/health is served */
-    public static final List<String> PORT_TAGS_HEALTH =
-            Collections.unmodifiableList(Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE));
+class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
+    private final ApplicationId applicationId;
+    private final StateV1HealthModel healthModel;
+    private final Map<ServiceId, HealthMonitor> monitors = new HashMap<>();
 
-    private final Map<ServiceId, HealthMonitor> healthMonitors;
-
-    public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) {
-        return startMonitoring(application, HealthMonitor::new);
+    ApplicationHealthMonitor(ApplicationId applicationId, StateV1HealthModel healthModel) {
+        this.applicationId = applicationId;
+        this.healthModel = healthModel;
     }
 
-    /** For testing. */
-    static ApplicationHealthMonitor startMonitoring(ApplicationInfo application,
-                                                    Function<HealthEndpoint, HealthMonitor> mapper) {
-        return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper));
-    }
+    void monitor(ApplicationInfo applicationInfo) {
+        if (!applicationInfo.getApplicationId().equals(applicationId)) {
+            throw new IllegalArgumentException("Monitors " + applicationId + " but was asked to monitor " + applicationInfo.getApplicationId());
+        }
+
+        Map<ServiceId, HealthEndpoint> endpoints = healthModel.extractHealthEndpoints(applicationInfo);
+
+        // Remove obsolete monitors
+        Set<ServiceId> removed = new HashSet<>(monitors.keySet());
+        removed.removeAll(endpoints.keySet());
+        removed.stream().map(monitors::remove).forEach(HealthMonitor::close);
 
-    private ApplicationHealthMonitor(Map<ServiceId, HealthMonitor> healthMonitors) {
-        this.healthMonitors = healthMonitors;
+        // Add new monitors.
+        endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring()));
     }
 
     @Override
@@ -62,7 +57,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
                                    ServiceType serviceType,
                                    ConfigId configId) {
         ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId);
-        HealthMonitor monitor = healthMonitors.get(serviceId);
+        HealthMonitor monitor = monitors.get(serviceId);
         if (monitor == null) {
             return ServiceStatus.NOT_CHECKED;
         }
@@ -72,45 +67,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
 
     @Override
     public void close() {
-        healthMonitors.values().forEach(HealthMonitor::close);
-        healthMonitors.clear();
-    }
-
-    private static Map<ServiceId, HealthMonitor> makeHealthMonitors(
-            ApplicationInfo application, Function<HealthEndpoint, HealthMonitor> monitorFactory) {
-        Map<ServiceId, HealthMonitor> healthMonitors = new HashMap<>();
-        for (HostInfo hostInfo : application.getModel().getHosts()) {
-            for (ServiceInfo serviceInfo : hostInfo.getServices()) {
-                for (PortInfo portInfo : serviceInfo.getPorts()) {
-                    maybeCreateHealthMonitor(
-                            application,
-                            hostInfo,
-                            serviceInfo,
-                            portInfo,
-                            monitorFactory)
-                            .ifPresent(healthMonitor -> healthMonitors.put(
-                                    ApplicationInstanceGenerator.getServiceId(application, serviceInfo),
-                                    healthMonitor));
-                }
-            }
-        }
-        return healthMonitors;
-    }
-
-    private static Optional<HealthMonitor> maybeCreateHealthMonitor(
-            ApplicationInfo applicationInfo,
-            HostInfo hostInfo,
-            ServiceInfo serviceInfo,
-            PortInfo portInfo,
-            Function<HealthEndpoint, HealthMonitor> monitorFactory) {
-        if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) {
-            HostName hostname = HostName.from(hostInfo.getHostname());
-            HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort());
-            HealthMonitor healthMonitor = monitorFactory.apply(endpoint);
-            healthMonitor.startMonitoring();
-            return Optional.of(healthMonitor);
-        }
-
-        return Optional.empty();
+        monitors.values().forEach(HealthMonitor::close);
+        monitors.clear();
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
index 43be236268c..a747753160e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
@@ -1,12 +1,12 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
 
 /**
  * @author hakonhall
  */
 @FunctionalInterface
 interface ApplicationHealthMonitorFactory {
-    ApplicationHealthMonitor create(ApplicationInfo applicationInfo);
+    ApplicationHealthMonitor create(ApplicationId applicationId);
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java
deleted file mode 100644
index 129cc799a25..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java
+++ /dev/null
@@ -1,142 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.ConnectionKeepAliveStrategy;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-
-import java.util.function.Function;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
-
-/**
- * Health client
- *
- * NOT thread-safe.
- *
- * @author hakon
- */
-public class HealthClient implements AutoCloseable {
-    private static final ObjectMapper mapper = new ObjectMapper();
-    private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB
-    private static final int DEFAULT_TIMEOUT_MILLIS = 1_000;
-
-    private static final ConnectionKeepAliveStrategy KEEP_ALIVE_STRATEGY =
-            new DefaultConnectionKeepAliveStrategy() {
-                @Override
-                public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
-                    long keepAlive = super.getKeepAliveDuration(response, context);
-                    if (keepAlive == -1) {
-                        // Keep connections alive 60 seconds if a keep-alive value
-                        // has not be explicitly set by the server
-                        keepAlive = 60000;
-                    }
-                    return keepAlive;
-                }
-            };
-
-    private final HealthEndpoint endpoint;
-    private final CloseableHttpClient httpClient;
-    private final Function<HttpEntity, String> getContentFunction;
-
-    public HealthClient(HealthEndpoint endpoint) {
-        this(endpoint,
-                makeCloseableHttpClient(endpoint),
-                entity -> uncheck(() -> EntityUtils.toString(entity)));
-    }
-
-    /** For testing. */
-    HealthClient(HealthEndpoint endpoint,
-                 CloseableHttpClient httpClient,
-                 Function<HttpEntity, String> getContentFunction) {
-        this.endpoint = endpoint;
-        this.httpClient = httpClient;
-        this.getContentFunction = getContentFunction;
-    }
-
-    public HealthEndpoint getEndpoint() {
-        return endpoint;
-    }
-
-    public HealthInfo getHealthInfo() {
-        try {
-            return probeHealth();
-        } catch (Exception e) {
-            return HealthInfo.fromException(e);
-        }
-    }
-
-    @Override
-    public void close() {
-        try {
-            httpClient.close();
-        } catch (Exception e) {
-            // ignore
-        }
-    }
-
-    private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) {
-        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
-                .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory())
-                .build();
-
-        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry);
-
-        RequestConfig requestConfig = RequestConfig.custom()
-                .setConnectTimeout(DEFAULT_TIMEOUT_MILLIS) // establishment of connection
-                .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MILLIS)  // connection from connection manager
-                .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data
-                .build();
-
-        return HttpClients.custom()
-                .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY)
-                .setConnectionManager(connectionManager)
-                .disableAutomaticRetries()
-                .setDefaultRequestConfig(requestConfig)
-                .build();
-    }
-
-    private HealthInfo probeHealth() throws Exception {
-        HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString());
-
-        CloseableHttpClient httpClient = this.httpClient;
-        if (httpClient == null) {
-            throw new IllegalStateException("HTTP client never started or has closed");
-        }
-
-        try (CloseableHttpResponse httpResponse = httpClient.execute(httpget)) {
-            int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
-            if (httpStatusCode < 200 || httpStatusCode >= 300) {
-                return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
-            }
-
-            HttpEntity bodyEntity = httpResponse.getEntity();
-            long contentLength = bodyEntity.getContentLength();
-            if (contentLength > MAX_CONTENT_LENGTH) {
-                throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
-            }
-            String body = getContentFunction.apply(bodyEntity);
-            HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class);
-
-            if (healthResponse.status == null || healthResponse.status.code == null) {
-                return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
-            } else {
-                return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
-            }
-        }
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
index e15d82ea70b..8c4997634a0 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
@@ -1,38 +1,15 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.provision.HostName;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
-import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import java.net.URL;
-import java.util.Collections;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
+import com.yahoo.vespa.service.model.ServiceId;
 
 /**
+ * An endpoint 1-1 with a service and that can be health monitored.
+ *
  * @author hakon
  */
-public interface HealthEndpoint {
-
-    static HealthEndpoint forHttp(HostName hostname, int port) {
-        URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health"));
-        return new HttpHealthEndpoint(url);
-    }
-
-    static HealthEndpoint forHttps(HostName hostname,
-                                   int port,
-                                   ServiceIdentityProvider serviceIdentityProvider,
-                                   AthenzIdentity remoteIdentity) {
-        URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health"));
-        HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity));
-        return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier);
-    }
-
-    URL getStateV1HealthUrl();
-    ConnectionSocketFactory getConnectionSocketFactory();
+interface HealthEndpoint {
+    ServiceId getServiceId();
     String description();
+    HealthMonitor startMonitoring();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
index d6dc1942404..f0e13548f58 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
@@ -1,90 +1,14 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 
-import java.time.Duration;
-import java.util.Random;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-
 /**
- * Used to monitor the health of a single URL endpoint.
- *
- * <p>Must be closed on successful start of monitoring ({}
- *
- * <p>Thread-safe
- *
- * @author hakon
+ * @author hakonhall
  */
-public class HealthMonitor implements AutoCloseable {
-    private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName());
-
-    /** The duration between each health request. */
-    private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10);
-
-    // About 'static': Javadoc says "Instances of java.util.Random are threadsafe."
-    private static final Random random = new Random();
-
-    private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
-    private final HealthClient healthClient;
-    private final Duration delay;
-
-    private volatile HealthInfo lastHealthInfo = HealthInfo.empty();
-
-    public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) {
-        this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY);
-    }
-
-    /** For testing. */
-    HealthMonitor(HealthClient healthClient, Duration delay) {
-        this.healthClient = healthClient;
-        this.delay = delay;
-    }
-
-    public void startMonitoring() {
-        executor.scheduleWithFixedDelay(
-                this::updateSynchronously,
-                initialDelayInMillis(delay.toMillis()),
-                delay.toMillis(),
-                TimeUnit.MILLISECONDS);
-    }
-
-    public ServiceStatus getStatus() {
-        return lastHealthInfo.toServiceStatus();
-    }
+interface HealthMonitor extends AutoCloseable {
+    ServiceStatus getStatus();
 
     @Override
-    public void close() {
-        executor.shutdown();
-
-        try {
-            executor.awaitTermination(2, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            logger.log(LogLevel.INFO, "Interrupted while waiting for health monitor termination: " +
-                    e.getMessage());
-        }
-
-        healthClient.close();
-    }
-
-    private long initialDelayInMillis(long maxInitialDelayInMillis) {
-        if (maxInitialDelayInMillis >= Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Max initial delay is out of bounds: " + maxInitialDelayInMillis);
-        }
-
-        return (long) random.nextInt((int) maxInitialDelayInMillis);
-    }
-
-    private void updateSynchronously() {
-        try {
-            lastHealthInfo = healthClient.getHealthInfo();
-        } catch (Throwable t) {
-            // An uncaught exception will kill the executor.scheduleWithFixedDelay thread!
-            logger.log(LogLevel.WARNING, "Failed to get health info for " +
-                    healthClient.getEndpoint().description(), t);
-        }
-    }
+    void close();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index e9a5ec314f6..2ad37faf593 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -12,8 +12,10 @@ import com.yahoo.vespa.flags.FeatureFlag;
 import com.yahoo.vespa.flags.FileFlagSource;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.ZoneApplication;
+import com.yahoo.vespa.service.executor.RunletExecutorImpl;
 import com.yahoo.vespa.service.manager.MonitorManager;
 
+import java.time.Duration;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -22,6 +24,28 @@ import java.util.concurrent.ConcurrentHashMap;
  * @author hakon
  */
 public class HealthMonitorManager implements MonitorManager {
+    // Weight the following against each other:
+    //  - The number of threads N working on health checking
+    //  - The health request timeout T
+    //  - The max staleness S of the health of an endpoint
+    //  - The ideal staleness I of the health of an endpoint
+    //
+    // The largest zone is main.prod.us-west-1:
+    //  - 314 tenant host admins
+    //  - 7 proxy host admins
+    //  - 3 config host admins
+    //  - 3 config servers
+    // for a total of E = 327 endpoints
+    private static final int MAX_ENDPOINTS = 500;
+    private static final Duration HEALTH_REQUEST_TIMEOUT = Duration.ofSeconds(1);
+    private static final Duration TARGET_HEALTH_STALENESS = Duration.ofSeconds(10);
+    private static final Duration MAX_HEALTH_STALENESS = Duration.ofSeconds(60);
+    static final int THREAD_POOL_SIZE = (int) Math.ceil(MAX_ENDPOINTS * HEALTH_REQUEST_TIMEOUT.toMillis() / (double) MAX_HEALTH_STALENESS.toMillis());
+
+    // Keep connections alive 60 seconds (>=MAX_HEALTH_STALENESS) if a keep-alive value has not be
+    // explicitly set by the server.
+    private static final Duration KEEP_ALIVE = Duration.ofSeconds(60);
+
     private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>();
     private final DuperModelManager duperModel;
     private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory;
@@ -31,22 +55,29 @@ public class HealthMonitorManager implements MonitorManager {
     public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) {
         this(duperModel,
                 new FeatureFlag("healthmonitor-monitorinfra", true, flagSource),
-                ApplicationHealthMonitor::startMonitoring);
+                new StateV1HealthModel(TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE)));
+    }
+
+    private HealthMonitorManager(DuperModelManager duperModel,
+                                 FeatureFlag monitorInfra,
+                                 StateV1HealthModel healthModel) {
+        this(duperModel, monitorInfra, id -> new ApplicationHealthMonitor(id, healthModel));
     }
 
     HealthMonitorManager(DuperModelManager duperModel,
                          FeatureFlag monitorInfra,
                          ApplicationHealthMonitorFactory applicationHealthMonitorFactory) {
         this.duperModel = duperModel;
-        this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
         this.monitorInfra = monitorInfra;
+        this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
     }
 
     @Override
     public void applicationActivated(ApplicationInfo application) {
         if (wouldMonitor(application.getApplicationId())) {
-            ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application);
-            healthMonitors.put(application.getApplicationId(), monitor);
+            healthMonitors
+                    .computeIfAbsent(application.getApplicationId(), applicationHealthMonitorFactory::create)
+                    .monitor(application);
         }
     }
 
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java
new file mode 100644
index 00000000000..4ed49e17e9f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java
@@ -0,0 +1,14 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.service.executor.Runlet;
+
+/**
+ * A {@link HealthUpdater} will probe the health with {@link #run()}, whose result can be fetched with the
+ * thread-safe method {@link #getLatestHealthInfo()}.
+ *
+ * @author hakonhall
+ */
+interface HealthUpdater extends Runlet {
+    HealthInfo getLatestHealthInfo();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java
deleted file mode 100644
index 793c1a93379..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-
-import java.net.URL;
-
-/**
- * @author hakon
- */
-class HttpHealthEndpoint implements HealthEndpoint {
-    private final URL url;
-    private final ConnectionSocketFactory socketFactory;
-
-    HttpHealthEndpoint(URL url) {
-        this.url = url;
-        this.socketFactory = PlainConnectionSocketFactory.getSocketFactory();
-    }
-
-    @Override
-    public URL getStateV1HealthUrl() {
-        return url;
-    }
-
-    @Override
-    public ConnectionSocketFactory getConnectionSocketFactory() {
-        return socketFactory;
-    }
-
-    @Override
-    public String description() {
-        return url.toString();
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java
deleted file mode 100644
index 42e408256c5..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
-import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import java.net.URL;
-
-/**
- * @author hakon
- */
-public class HttpsHealthEndpoint implements HealthEndpoint {
-    private final URL url;
-    private final HostnameVerifier hostnameVerifier;
-    private final ServiceIdentityProvider serviceIdentityProvider;
-
-    HttpsHealthEndpoint(URL url,
-                        ServiceIdentityProvider serviceIdentityProvider,
-                        HostnameVerifier hostnameVerifier) {
-        this.url = url;
-        this.serviceIdentityProvider = serviceIdentityProvider;
-        this.hostnameVerifier = hostnameVerifier;
-    }
-
-    @Override
-    public URL getStateV1HealthUrl() {
-        return url;
-    }
-
-    @Override
-    public ConnectionSocketFactory getConnectionSocketFactory() {
-        return new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(serviceIdentityProvider), hostnameVerifier);
-    }
-
-    @Override
-    public String description() {
-        return url.toString();
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java
new file mode 100644
index 00000000000..88aefe42a14
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java
@@ -0,0 +1,74 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.log.LogLevel;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.net.URL;
+import java.time.Duration;
+import java.util.function.Function;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * A thread-unsafe /state/v1/health endpoint client.
+ *
+ * @author hakonhall
+ */
+public class StateV1HealthClient implements AutoCloseable {
+    private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final Logger logger = Logger.getLogger(StateV1HealthClient.class.getName());
+    private final ApacheHttpClient httpClient;
+    private final Function<HttpEntity, String> getContentFunction;
+
+    StateV1HealthClient(URL url, Duration requestTimeout, Duration connectionKeepAlive) {
+        this(new ApacheHttpClient(url, requestTimeout, connectionKeepAlive, PlainConnectionSocketFactory.getSocketFactory()),
+                entity -> uncheck(() -> EntityUtils.toString(entity)));
+    }
+
+    StateV1HealthClient(ApacheHttpClient apacheHttpClient, Function<HttpEntity, String> getContentFunction) {
+        httpClient = apacheHttpClient;
+        this.getContentFunction = getContentFunction;
+    }
+
+    HealthInfo get() throws Exception {
+        return httpClient.get(this::handle);
+    }
+
+    private HealthInfo handle(CloseableHttpResponse httpResponse) throws IOException {
+        int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
+        if (httpStatusCode < 200 || httpStatusCode >= 300) {
+            return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
+        }
+
+        HttpEntity bodyEntity = httpResponse.getEntity();
+        long contentLength = bodyEntity.getContentLength();
+        if (contentLength > MAX_CONTENT_LENGTH) {
+            throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
+        }
+        String body = getContentFunction.apply(bodyEntity);
+        HealthResponse healthResponse = MAPPER.readValue(body, HealthResponse.class);
+
+        if (healthResponse.status == null || healthResponse.status.code == null) {
+            return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
+        } else {
+            return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
+        }
+    }
+
+    @Override
+    public void close() {
+        try {
+            httpClient.close();
+        } catch (Exception e) {
+            logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
new file mode 100644
index 00000000000..8eca03c616f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
@@ -0,0 +1,59 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ServiceId;
+
+import java.net.URL;
+import java.time.Duration;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+class StateV1HealthEndpoint implements HealthEndpoint {
+    private final ServiceId serviceId;
+    private final URL url;
+    private final Duration requestTimeout;
+    private final Duration connectionKeepAlive;
+    private final Duration delay;
+    private final RunletExecutor executor;
+
+    StateV1HealthEndpoint(ServiceId serviceId,
+                          HostName hostname,
+                          int port,
+                          Duration delay,
+                          Duration requestTimeout,
+                          Duration connectionKeepAlive,
+                          RunletExecutor executor) {
+        this.serviceId = serviceId;
+        this.delay = delay;
+        this.executor = executor;
+        this.url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health"));
+        this.requestTimeout = requestTimeout;
+        this.connectionKeepAlive = connectionKeepAlive;
+    }
+
+    @Override
+    public ServiceId getServiceId() {
+        return serviceId;
+    }
+
+    @Override
+    public HealthMonitor startMonitoring() {
+        StateV1HealthUpdater updater = new StateV1HealthUpdater(url, requestTimeout, connectionKeepAlive);
+        return new StateV1HealthMonitor(updater, executor, delay);
+    }
+
+    @Override
+    public String description() {
+        return url.toString();
+    }
+
+    @Override
+    public String toString() {
+        return description();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
new file mode 100644
index 00000000000..5e8979deb9f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
@@ -0,0 +1,74 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
+import com.yahoo.vespa.service.model.ServiceId;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author hakonhall
+ */
+public class StateV1HealthModel implements AutoCloseable {
+    private static final String PORT_TAG_STATE = "STATE";
+    private static final String PORT_TAG_HTTP = "HTTP";
+
+    /** Port tags implying /state/v1/health is served on HTTP. */
+    public static final List<String> HTTP_HEALTH_PORT_TAGS = Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE);
+    private final Duration targetHealthStaleness;
+    private final Duration requestTimeout;
+    private final Duration connectionKeepAlive;
+    private final RunletExecutor executor;
+
+    StateV1HealthModel(Duration targetHealthStaleness,
+                       Duration requestTimeout,
+                       Duration connectionKeepAlive,
+                       RunletExecutor executor) {
+        this.targetHealthStaleness = targetHealthStaleness;
+        this.requestTimeout = requestTimeout;
+        this.connectionKeepAlive = connectionKeepAlive;
+        this.executor = executor;
+    }
+
+    Map<ServiceId, HealthEndpoint> extractHealthEndpoints(ApplicationInfo application) {
+        Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
+
+        for (HostInfo hostInfo : application.getModel().getHosts()) {
+            HostName hostname = HostName.from(hostInfo.getHostname());
+            for (ServiceInfo serviceInfo : hostInfo.getServices()) {
+                ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo);
+                for (PortInfo portInfo : serviceInfo.getPorts()) {
+                    if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) {
+                        StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint(
+                                serviceId,
+                                hostname,
+                                portInfo.getPort(),
+                                targetHealthStaleness,
+                                requestTimeout,
+                                connectionKeepAlive,
+                                executor);
+                        endpoints.put(serviceId, endpoint);
+                        break; // Avoid >1 endpoints per serviceId
+                    }
+                }
+            }
+        }
+
+        return endpoints;
+    }
+
+    @Override
+    public void close() {
+        executor.close();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java
new file mode 100644
index 00000000000..d37797c7be9
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.service.executor.Cancellable;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+
+import java.time.Duration;
+
+/**
+ * Used to monitor the health of a single URL endpoint.
+ *
+ * @author hakon
+ */
+class StateV1HealthMonitor implements HealthMonitor {
+    private final StateV1HealthUpdater updater;
+    private final Cancellable periodicExecution;
+
+    StateV1HealthMonitor(StateV1HealthUpdater updater, RunletExecutor executor, Duration delay) {
+        this.updater = updater;
+        this.periodicExecution = executor.scheduleWithFixedDelay(updater, delay);
+    }
+
+    @Override
+    public ServiceStatus getStatus() {
+        return updater.getLatestHealthInfo().toServiceStatus();
+    }
+
+    @Override
+    public void close() {
+        periodicExecution.cancel();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
new file mode 100644
index 00000000000..011ec3b3212
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
@@ -0,0 +1,41 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import java.net.URL;
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+class StateV1HealthUpdater implements HealthUpdater {
+    private final StateV1HealthClient healthClient;
+
+    private volatile HealthInfo lastHealthInfo = HealthInfo.empty();
+
+    StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) {
+        this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive));
+    }
+
+    StateV1HealthUpdater(StateV1HealthClient healthClient) {
+        this.healthClient = healthClient;
+    }
+
+    @Override
+    public HealthInfo getLatestHealthInfo() {
+        return lastHealthInfo;
+    }
+
+    @Override
+    public void run() {
+        try {
+            lastHealthInfo = healthClient.get();
+        } catch (Exception e) {
+            lastHealthInfo = HealthInfo.fromException(e);
+        }
+    }
+
+    @Override
+    public void close() {
+        healthClient.close();
+    }
+}