Use thread pool for health monitoring in service-monitor

This is necessary to avoid using too many threads when monitoring the
host-admin on the tenant Docker hosts.

32 files changed, 1231 insertions, 539 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index 024282d3d21..f559e9336c8 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -13,7 +13,7 @@ import java.util.TreeMap;
 import java.util.logging.Logger;
 
 /**
- * A non-thread-safe mutable container of ApplicationInfo in the DuperModel, also taking care of listeners on changes.
+ * A non-thread-safe mutable container of ApplicationInfo, also taking care of listeners on changes.
  *
  * @author hakonhall
  */
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
index 8c74fe0396e..09140423010 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
@@ -17,7 +17,7 @@ import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
-import com.yahoo.vespa.service.health.ApplicationHealthMonitor;
+import com.yahoo.vespa.service.health.StateV1HealthModel;
 import com.yahoo.vespa.service.model.ModelGenerator;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
@@ -107,7 +107,7 @@ public abstract class InfraApplication implements InfraApplicationApi {
     }
 
     private HostInfo makeHostInfo(HostName hostname) {
-        PortInfo portInfo = new PortInfo(healthPort, ApplicationHealthMonitor.PORT_TAGS_HEALTH);
+        PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS);
 
         Map<String, String> properties = new HashMap<>();
         properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, getClusterId().s());
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java
new file mode 100644
index 00000000000..80c35851fa5
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Cancellable.java
@@ -0,0 +1,10 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * @author hakonhall
+ */
+@FunctionalInterface
+public interface Cancellable {
+    void cancel();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java
new file mode 100644
index 00000000000..316b810c682
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java
@@ -0,0 +1,103 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import com.yahoo.log.LogLevel;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Provides the {@link Cancellable} returned by {@link RunletExecutorImpl#scheduleWithFixedDelay(Runlet, Duration)},
+ * and ensuring the correct semantic execution of the {@link Runlet}.
+ *
+ * @author hakonhall
+ */
+class CancellableImpl implements Cancellable, Runnable {
+    private static final Logger logger = Logger.getLogger(CancellableImpl.class.getName());
+
+    private final Object monitor = new Object();
+    private Runlet runlet;
+    private Optional<Runnable> periodicExecutionCancellation = Optional.empty();
+    private boolean running = false;
+    private boolean cancelled = false;
+
+    public CancellableImpl(Runlet runlet) {
+        this.runlet = runlet;
+    }
+
+    /**
+     * Provide a way for {@code this} to cancel the periodic execution of {@link #run()}.
+     *
+     * <p>Must be called happens-before {@link #cancel()}.
+     */
+    void setPeriodicExecutionCancellationCallback(Runnable periodicExecutionCancellation) {
+        synchronized (monitor) {
+            if (cancelled) {
+                throw new IllegalStateException("Cancellation callback set after cancel()");
+            }
+
+            this.periodicExecutionCancellation = Optional.of(periodicExecutionCancellation);
+        }
+    }
+
+    /**
+     * Cancel the execution of the {@link Runlet}.
+     *
+     * <ul>
+     *     <li>Either the runlet will not execute any more {@link Runlet#run()}s, and {@link Runlet#close()}
+     *     and then {@code periodicExecutionCancellation} will be called synchronously, or
+     *     <li>{@link #run()} is executing concurrently by another thread {@code T}. The last call to
+     *     {@link Runlet#run()} will be called by {@code T} shortly, is in progress, or has completed.
+     *     Then {@code T} will call {@link Runlet#close()} followed by {@code periodicExecutionCancellation},
+     *     before the return of {@link #run()}.
+     * </ul>
+     *
+     * <p>{@link #setPeriodicExecutionCancellationCallback(Runnable)} must be called happens-before this method.
+     */
+    @Override
+    public void cancel() {
+        synchronized (monitor) {
+            if (!periodicExecutionCancellation.isPresent()) {
+                throw new IllegalStateException("setPeriodicExecutionCancellationCallback has not been called before cancel");
+            }
+
+            cancelled = true;
+            if (running) return;
+        }
+
+        runlet.close();
+        periodicExecutionCancellation.get().run();
+    }
+
+    /**
+     * Must be called periodically in happens-before order, but may be called concurrently with
+     * {@link #setPeriodicExecutionCancellationCallback(Runnable)} and {@link #cancel()}.
+     */
+    @Override
+    public void run() {
+        try {
+            synchronized (monitor) {
+                if (cancelled) return;
+                running = true;
+            }
+
+            runlet.run();
+
+            synchronized (monitor) {
+                running = false;
+                if (!cancelled) return;
+
+                if (!periodicExecutionCancellation.isPresent()) {
+                    // This should be impossible given the implementation of cancel()
+                    throw new IllegalStateException("Cancelled before cancellation callback was set");
+                }
+            }
+
+            runlet.close();
+            periodicExecutionCancellation.get().run();
+        } catch (Throwable e) {
+            logger.log(LogLevel.ERROR, "Failed run of periodic execution", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java
new file mode 100644
index 00000000000..a41bd0f777e
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/Runlet.java
@@ -0,0 +1,19 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * A {@code Runlet} joins {@link AutoCloseable} with {@link Runnable} with the following semantics:
+ *
+ * <ul>
+ *     <li>The {@link #run()} method may be called any number of times, followed by a single call to {@link #close()}.
+ *     <li>The caller must ensure the calls are ordered by {@code happens-before}, i.e. the class can be thread-unsafe.
+ * </ul>
+ *
+ * @author hakonhall
+ */
+public interface Runlet extends AutoCloseable, Runnable {
+    void run();
+
+    @Override
+    void close();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java
new file mode 100644
index 00000000000..4d6dc4b316f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutor.java
@@ -0,0 +1,21 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+public interface RunletExecutor extends AutoCloseable {
+    /**
+     * Execute the task periodically with a fixed delay.
+     *
+     * <p>If the execution is {@link Cancellable#cancel() cancelled}, the runlet is {@link Runlet#close() closed}
+     * as soon as possible.
+     */
+    Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay);
+
+    /** Shuts down and waits for all execution to wind down. */
+    @Override
+    void close();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java
new file mode 100644
index 00000000000..1f647a7fb31
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java
@@ -0,0 +1,71 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import com.yahoo.log.LogLevel;
+
+import java.time.Duration;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+/**
+ * @author hakonhall
+ */
+public class RunletExecutorImpl implements RunletExecutor {
+    private static Logger logger = Logger.getLogger(RunletExecutorImpl.class.getName());
+
+    // About 'static': Javadoc says "Instances of java.util.Random are threadsafe."
+    private static final Random random = new Random();
+
+    private final AtomicInteger executionId = new AtomicInteger(0);
+    private final ConcurrentHashMap<Integer, CancellableImpl> cancellables = new ConcurrentHashMap<>();
+    private final ScheduledThreadPoolExecutor executor;
+
+    public RunletExecutorImpl(int threadPoolSize) {
+        executor = new ScheduledThreadPoolExecutor(threadPoolSize);
+    }
+
+    public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) {
+        Duration initialDelay = Duration.ofMillis((long) random.nextInt((int) delay.toMillis()));
+        CancellableImpl cancellable = new CancellableImpl(runlet);
+        ScheduledFuture<?> future = executor.scheduleWithFixedDelay(cancellable, initialDelay.toMillis(), delay.toMillis(), TimeUnit.MILLISECONDS);
+        cancellable.setPeriodicExecutionCancellationCallback(() -> future.cancel(false));
+        Integer id = executionId.incrementAndGet();
+        cancellables.put(id, cancellable);
+        return () -> cancelRunlet(id);
+    }
+
+    private void cancelRunlet(Integer id) {
+        CancellableImpl cancellable = cancellables.remove(id);
+        if (cancellable != null) {
+            cancellable.cancel();
+        }
+    }
+
+    @Override
+    public void close() {
+        // At this point no-one should be scheduling new runlets, so this ought to clear the map.
+        cancellables.keySet().forEach(this::cancelRunlet);
+
+        if (cancellables.size() > 0) {
+            throw new IllegalStateException("Runlets scheduled while closing the executor");
+        }
+
+        // The cancellables will cancel themselves from the executor only after up-to delay time,
+        // so wait until all have drained.
+        while (executor.getQueue().size() > 0) {
+            try { Thread.sleep(200); } catch (InterruptedException ignored) { }
+        }
+
+        executor.shutdown();
+        try {
+            executor.awaitTermination(10, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            logger.log(LogLevel.WARNING, "Timed out waiting for termination of executor", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java
new file mode 100644
index 00000000000..4a382ee8d94
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java
@@ -0,0 +1,87 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.ConnectionKeepAliveStrategy;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.protocol.HttpContext;
+
+import java.io.IOException;
+import java.net.URL;
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+class ApacheHttpClient implements AutoCloseable {
+    private final URL url;
+    private final CloseableHttpClient client;
+
+    @FunctionalInterface
+    interface Handler<T> {
+        T handle(CloseableHttpResponse httpResponse) throws Exception;
+    }
+
+    static CloseableHttpClient makeCloseableHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) {
+        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
+                .register(url.getProtocol(),socketFactory)
+                .build();
+
+        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry);
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout((int) timeout.toMillis()) // establishment of connection
+                .setConnectionRequestTimeout((int) timeout.toMillis())  // connection from connection manager
+                .setSocketTimeout((int) timeout.toMillis()) // waiting for data
+                .build();
+
+        ConnectionKeepAliveStrategy keepAliveStrategy =
+                new DefaultConnectionKeepAliveStrategy() {
+                    @Override
+                    public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
+                        long keepAliveMillis = super.getKeepAliveDuration(response, context);
+                        if (keepAliveMillis == -1) {
+                            keepAliveMillis = keepAlive.toMillis();
+                        }
+                        return keepAliveMillis;
+                    }
+                };
+
+        return HttpClients.custom()
+                .setKeepAliveStrategy(keepAliveStrategy)
+                .setConnectionManager(connectionManager)
+                .disableAutomaticRetries()
+                .setDefaultRequestConfig(requestConfig)
+                .build();
+    }
+
+    ApacheHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) {
+        this(url, makeCloseableHttpClient(url, timeout, keepAlive, socketFactory));
+    }
+
+    ApacheHttpClient(URL url, CloseableHttpClient client) {
+        this.url = url;
+        this.client = client;
+    }
+
+    <T> T get(Handler<T> handler) throws Exception {
+        try (CloseableHttpResponse httpResponse = client.execute(new HttpGet(url.toString()))) {
+            return handler.handle(httpResponse);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        client.close();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
index 2d81474853c..5fab8ac8591 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
@@ -2,53 +2,48 @@
 package com.yahoo.vespa.service.health;
 
 import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.model.api.HostInfo;
-import com.yahoo.config.model.api.PortInfo;
-import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
-import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
-import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
 import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
+import java.util.HashSet;
 import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
+import java.util.Set;
 
 /**
  * Responsible for monitoring a whole application using /state/v1/health.
  *
  * @author hakon
  */
-public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
-    public static final String PORT_TAG_STATE = "STATE";
-    public static final String PORT_TAG_HTTP = "HTTP";
-    /** Port tags implying /state/v1/health is served */
-    public static final List<String> PORT_TAGS_HEALTH =
-            Collections.unmodifiableList(Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE));
+class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
+    private final ApplicationId applicationId;
+    private final StateV1HealthModel healthModel;
+    private final Map<ServiceId, HealthMonitor> monitors = new HashMap<>();
 
-    private final Map<ServiceId, HealthMonitor> healthMonitors;
-
-    public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) {
-        return startMonitoring(application, HealthMonitor::new);
+    ApplicationHealthMonitor(ApplicationId applicationId, StateV1HealthModel healthModel) {
+        this.applicationId = applicationId;
+        this.healthModel = healthModel;
     }
 
-    /** For testing. */
-    static ApplicationHealthMonitor startMonitoring(ApplicationInfo application,
-                                                    Function<HealthEndpoint, HealthMonitor> mapper) {
-        return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper));
-    }
+    void monitor(ApplicationInfo applicationInfo) {
+        if (!applicationInfo.getApplicationId().equals(applicationId)) {
+            throw new IllegalArgumentException("Monitors " + applicationId + " but was asked to monitor " + applicationInfo.getApplicationId());
+        }
+
+        Map<ServiceId, HealthEndpoint> endpoints = healthModel.extractHealthEndpoints(applicationInfo);
+
+        // Remove obsolete monitors
+        Set<ServiceId> removed = new HashSet<>(monitors.keySet());
+        removed.removeAll(endpoints.keySet());
+        removed.stream().map(monitors::remove).forEach(HealthMonitor::close);
 
-    private ApplicationHealthMonitor(Map<ServiceId, HealthMonitor> healthMonitors) {
-        this.healthMonitors = healthMonitors;
+        // Add new monitors.
+        endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring()));
     }
 
     @Override
@@ -62,7 +57,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
                                    ServiceType serviceType,
                                    ConfigId configId) {
         ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId);
-        HealthMonitor monitor = healthMonitors.get(serviceId);
+        HealthMonitor monitor = monitors.get(serviceId);
         if (monitor == null) {
             return ServiceStatus.NOT_CHECKED;
         }
@@ -72,45 +67,7 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
 
     @Override
     public void close() {
-        healthMonitors.values().forEach(HealthMonitor::close);
-        healthMonitors.clear();
-    }
-
-    private static Map<ServiceId, HealthMonitor> makeHealthMonitors(
-            ApplicationInfo application, Function<HealthEndpoint, HealthMonitor> monitorFactory) {
-        Map<ServiceId, HealthMonitor> healthMonitors = new HashMap<>();
-        for (HostInfo hostInfo : application.getModel().getHosts()) {
-            for (ServiceInfo serviceInfo : hostInfo.getServices()) {
-                for (PortInfo portInfo : serviceInfo.getPorts()) {
-                    maybeCreateHealthMonitor(
-                            application,
-                            hostInfo,
-                            serviceInfo,
-                            portInfo,
-                            monitorFactory)
-                            .ifPresent(healthMonitor -> healthMonitors.put(
-                                    ApplicationInstanceGenerator.getServiceId(application, serviceInfo),
-                                    healthMonitor));
-                }
-            }
-        }
-        return healthMonitors;
-    }
-
-    private static Optional<HealthMonitor> maybeCreateHealthMonitor(
-            ApplicationInfo applicationInfo,
-            HostInfo hostInfo,
-            ServiceInfo serviceInfo,
-            PortInfo portInfo,
-            Function<HealthEndpoint, HealthMonitor> monitorFactory) {
-        if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) {
-            HostName hostname = HostName.from(hostInfo.getHostname());
-            HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort());
-            HealthMonitor healthMonitor = monitorFactory.apply(endpoint);
-            healthMonitor.startMonitoring();
-            return Optional.of(healthMonitor);
-        }
-
-        return Optional.empty();
+        monitors.values().forEach(HealthMonitor::close);
+        monitors.clear();
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
index 43be236268c..a747753160e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
@@ -1,12 +1,12 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
 
 /**
  * @author hakonhall
  */
 @FunctionalInterface
 interface ApplicationHealthMonitorFactory {
-    ApplicationHealthMonitor create(ApplicationInfo applicationInfo);
+    ApplicationHealthMonitor create(ApplicationId applicationId);
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java
deleted file mode 100644
index 129cc799a25..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthClient.java
+++ /dev/null
@@ -1,142 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.ConnectionKeepAliveStrategy;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-
-import java.util.function.Function;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
-
-/**
- * Health client
- *
- * NOT thread-safe.
- *
- * @author hakon
- */
-public class HealthClient implements AutoCloseable {
-    private static final ObjectMapper mapper = new ObjectMapper();
-    private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB
-    private static final int DEFAULT_TIMEOUT_MILLIS = 1_000;
-
-    private static final ConnectionKeepAliveStrategy KEEP_ALIVE_STRATEGY =
-            new DefaultConnectionKeepAliveStrategy() {
-                @Override
-                public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
-                    long keepAlive = super.getKeepAliveDuration(response, context);
-                    if (keepAlive == -1) {
-                        // Keep connections alive 60 seconds if a keep-alive value
-                        // has not be explicitly set by the server
-                        keepAlive = 60000;
-                    }
-                    return keepAlive;
-                }
-            };
-
-    private final HealthEndpoint endpoint;
-    private final CloseableHttpClient httpClient;
-    private final Function<HttpEntity, String> getContentFunction;
-
-    public HealthClient(HealthEndpoint endpoint) {
-        this(endpoint,
-                makeCloseableHttpClient(endpoint),
-                entity -> uncheck(() -> EntityUtils.toString(entity)));
-    }
-
-    /** For testing. */
-    HealthClient(HealthEndpoint endpoint,
-                 CloseableHttpClient httpClient,
-                 Function<HttpEntity, String> getContentFunction) {
-        this.endpoint = endpoint;
-        this.httpClient = httpClient;
-        this.getContentFunction = getContentFunction;
-    }
-
-    public HealthEndpoint getEndpoint() {
-        return endpoint;
-    }
-
-    public HealthInfo getHealthInfo() {
-        try {
-            return probeHealth();
-        } catch (Exception e) {
-            return HealthInfo.fromException(e);
-        }
-    }
-
-    @Override
-    public void close() {
-        try {
-            httpClient.close();
-        } catch (Exception e) {
-            // ignore
-        }
-    }
-
-    private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) {
-        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
-                .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory())
-                .build();
-
-        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry);
-
-        RequestConfig requestConfig = RequestConfig.custom()
-                .setConnectTimeout(DEFAULT_TIMEOUT_MILLIS) // establishment of connection
-                .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MILLIS)  // connection from connection manager
-                .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data
-                .build();
-
-        return HttpClients.custom()
-                .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY)
-                .setConnectionManager(connectionManager)
-                .disableAutomaticRetries()
-                .setDefaultRequestConfig(requestConfig)
-                .build();
-    }
-
-    private HealthInfo probeHealth() throws Exception {
-        HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString());
-
-        CloseableHttpClient httpClient = this.httpClient;
-        if (httpClient == null) {
-            throw new IllegalStateException("HTTP client never started or has closed");
-        }
-
-        try (CloseableHttpResponse httpResponse = httpClient.execute(httpget)) {
-            int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
-            if (httpStatusCode < 200 || httpStatusCode >= 300) {
-                return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
-            }
-
-            HttpEntity bodyEntity = httpResponse.getEntity();
-            long contentLength = bodyEntity.getContentLength();
-            if (contentLength > MAX_CONTENT_LENGTH) {
-                throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
-            }
-            String body = getContentFunction.apply(bodyEntity);
-            HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class);
-
-            if (healthResponse.status == null || healthResponse.status.code == null) {
-                return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
-            } else {
-                return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
-            }
-        }
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
index e15d82ea70b..8c4997634a0 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
@@ -1,38 +1,15 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.provision.HostName;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
-import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import java.net.URL;
-import java.util.Collections;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
+import com.yahoo.vespa.service.model.ServiceId;
 
 /**
+ * An endpoint 1-1 with a service and that can be health monitored.
+ *
  * @author hakon
  */
-public interface HealthEndpoint {
-
-    static HealthEndpoint forHttp(HostName hostname, int port) {
-        URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health"));
-        return new HttpHealthEndpoint(url);
-    }
-
-    static HealthEndpoint forHttps(HostName hostname,
-                                   int port,
-                                   ServiceIdentityProvider serviceIdentityProvider,
-                                   AthenzIdentity remoteIdentity) {
-        URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health"));
-        HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity));
-        return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier);
-    }
-
-    URL getStateV1HealthUrl();
-    ConnectionSocketFactory getConnectionSocketFactory();
+interface HealthEndpoint {
+    ServiceId getServiceId();
     String description();
+    HealthMonitor startMonitoring();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
index d6dc1942404..f0e13548f58 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitor.java
@@ -1,90 +1,14 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 
-import java.time.Duration;
-import java.util.Random;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-
 /**
- * Used to monitor the health of a single URL endpoint.
- *
- * <p>Must be closed on successful start of monitoring ({}
- *
- * <p>Thread-safe
- *
- * @author hakon
+ * @author hakonhall
  */
-public class HealthMonitor implements AutoCloseable {
-    private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName());
-
-    /** The duration between each health request. */
-    private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10);
-
-    // About 'static': Javadoc says "Instances of java.util.Random are threadsafe."
-    private static final Random random = new Random();
-
-    private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
-    private final HealthClient healthClient;
-    private final Duration delay;
-
-    private volatile HealthInfo lastHealthInfo = HealthInfo.empty();
-
-    public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) {
-        this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY);
-    }
-
-    /** For testing. */
-    HealthMonitor(HealthClient healthClient, Duration delay) {
-        this.healthClient = healthClient;
-        this.delay = delay;
-    }
-
-    public void startMonitoring() {
-        executor.scheduleWithFixedDelay(
-                this::updateSynchronously,
-                initialDelayInMillis(delay.toMillis()),
-                delay.toMillis(),
-                TimeUnit.MILLISECONDS);
-    }
-
-    public ServiceStatus getStatus() {
-        return lastHealthInfo.toServiceStatus();
-    }
+interface HealthMonitor extends AutoCloseable {
+    ServiceStatus getStatus();
 
     @Override
-    public void close() {
-        executor.shutdown();
-
-        try {
-            executor.awaitTermination(2, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            logger.log(LogLevel.INFO, "Interrupted while waiting for health monitor termination: " +
-                    e.getMessage());
-        }
-
-        healthClient.close();
-    }
-
-    private long initialDelayInMillis(long maxInitialDelayInMillis) {
-        if (maxInitialDelayInMillis >= Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Max initial delay is out of bounds: " + maxInitialDelayInMillis);
-        }
-
-        return (long) random.nextInt((int) maxInitialDelayInMillis);
-    }
-
-    private void updateSynchronously() {
-        try {
-            lastHealthInfo = healthClient.getHealthInfo();
-        } catch (Throwable t) {
-            // An uncaught exception will kill the executor.scheduleWithFixedDelay thread!
-            logger.log(LogLevel.WARNING, "Failed to get health info for " +
-                    healthClient.getEndpoint().description(), t);
-        }
-    }
+    void close();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index e9a5ec314f6..2ad37faf593 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -12,8 +12,10 @@ import com.yahoo.vespa.flags.FeatureFlag;
 import com.yahoo.vespa.flags.FileFlagSource;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.ZoneApplication;
+import com.yahoo.vespa.service.executor.RunletExecutorImpl;
 import com.yahoo.vespa.service.manager.MonitorManager;
 
+import java.time.Duration;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -22,6 +24,28 @@ import java.util.concurrent.ConcurrentHashMap;
  * @author hakon
  */
 public class HealthMonitorManager implements MonitorManager {
+    // Weight the following against each other:
+    //  - The number of threads N working on health checking
+    //  - The health request timeout T
+    //  - The max staleness S of the health of an endpoint
+    //  - The ideal staleness I of the health of an endpoint
+    //
+    // The largest zone is main.prod.us-west-1:
+    //  - 314 tenant host admins
+    //  - 7 proxy host admins
+    //  - 3 config host admins
+    //  - 3 config servers
+    // for a total of E = 327 endpoints
+    private static final int MAX_ENDPOINTS = 500;
+    private static final Duration HEALTH_REQUEST_TIMEOUT = Duration.ofSeconds(1);
+    private static final Duration TARGET_HEALTH_STALENESS = Duration.ofSeconds(10);
+    private static final Duration MAX_HEALTH_STALENESS = Duration.ofSeconds(60);
+    static final int THREAD_POOL_SIZE = (int) Math.ceil(MAX_ENDPOINTS * HEALTH_REQUEST_TIMEOUT.toMillis() / (double) MAX_HEALTH_STALENESS.toMillis());
+
+    // Keep connections alive 60 seconds (>=MAX_HEALTH_STALENESS) if a keep-alive value has not be
+    // explicitly set by the server.
+    private static final Duration KEEP_ALIVE = Duration.ofSeconds(60);
+
     private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>();
     private final DuperModelManager duperModel;
     private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory;
@@ -31,22 +55,29 @@ public class HealthMonitorManager implements MonitorManager {
     public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) {
         this(duperModel,
                 new FeatureFlag("healthmonitor-monitorinfra", true, flagSource),
-                ApplicationHealthMonitor::startMonitoring);
+                new StateV1HealthModel(TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE)));
+    }
+
+    private HealthMonitorManager(DuperModelManager duperModel,
+                                 FeatureFlag monitorInfra,
+                                 StateV1HealthModel healthModel) {
+        this(duperModel, monitorInfra, id -> new ApplicationHealthMonitor(id, healthModel));
     }
 
     HealthMonitorManager(DuperModelManager duperModel,
                          FeatureFlag monitorInfra,
                          ApplicationHealthMonitorFactory applicationHealthMonitorFactory) {
         this.duperModel = duperModel;
-        this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
         this.monitorInfra = monitorInfra;
+        this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
     }
 
     @Override
     public void applicationActivated(ApplicationInfo application) {
         if (wouldMonitor(application.getApplicationId())) {
-            ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application);
-            healthMonitors.put(application.getApplicationId(), monitor);
+            healthMonitors
+                    .computeIfAbsent(application.getApplicationId(), applicationHealthMonitorFactory::create)
+                    .monitor(application);
         }
     }
 
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java
new file mode 100644
index 00000000000..4ed49e17e9f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthUpdater.java
@@ -0,0 +1,14 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.service.executor.Runlet;
+
+/**
+ * A {@link HealthUpdater} will probe the health with {@link #run()}, whose result can be fetched with the
+ * thread-safe method {@link #getLatestHealthInfo()}.
+ *
+ * @author hakonhall
+ */
+interface HealthUpdater extends Runlet {
+    HealthInfo getLatestHealthInfo();
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java
deleted file mode 100644
index 793c1a93379..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpHealthEndpoint.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-
-import java.net.URL;
-
-/**
- * @author hakon
- */
-class HttpHealthEndpoint implements HealthEndpoint {
-    private final URL url;
-    private final ConnectionSocketFactory socketFactory;
-
-    HttpHealthEndpoint(URL url) {
-        this.url = url;
-        this.socketFactory = PlainConnectionSocketFactory.getSocketFactory();
-    }
-
-    @Override
-    public URL getStateV1HealthUrl() {
-        return url;
-    }
-
-    @Override
-    public ConnectionSocketFactory getConnectionSocketFactory() {
-        return socketFactory;
-    }
-
-    @Override
-    public String description() {
-        return url.toString();
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java
deleted file mode 100644
index 42e408256c5..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HttpsHealthEndpoint.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
-import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import java.net.URL;
-
-/**
- * @author hakon
- */
-public class HttpsHealthEndpoint implements HealthEndpoint {
-    private final URL url;
-    private final HostnameVerifier hostnameVerifier;
-    private final ServiceIdentityProvider serviceIdentityProvider;
-
-    HttpsHealthEndpoint(URL url,
-                        ServiceIdentityProvider serviceIdentityProvider,
-                        HostnameVerifier hostnameVerifier) {
-        this.url = url;
-        this.serviceIdentityProvider = serviceIdentityProvider;
-        this.hostnameVerifier = hostnameVerifier;
-    }
-
-    @Override
-    public URL getStateV1HealthUrl() {
-        return url;
-    }
-
-    @Override
-    public ConnectionSocketFactory getConnectionSocketFactory() {
-        return new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(serviceIdentityProvider), hostnameVerifier);
-    }
-
-    @Override
-    public String description() {
-        return url.toString();
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java
new file mode 100644
index 00000000000..88aefe42a14
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java
@@ -0,0 +1,74 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.log.LogLevel;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.net.URL;
+import java.time.Duration;
+import java.util.function.Function;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * A thread-unsafe /state/v1/health endpoint client.
+ *
+ * @author hakonhall
+ */
+public class StateV1HealthClient implements AutoCloseable {
+    private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final Logger logger = Logger.getLogger(StateV1HealthClient.class.getName());
+    private final ApacheHttpClient httpClient;
+    private final Function<HttpEntity, String> getContentFunction;
+
+    StateV1HealthClient(URL url, Duration requestTimeout, Duration connectionKeepAlive) {
+        this(new ApacheHttpClient(url, requestTimeout, connectionKeepAlive, PlainConnectionSocketFactory.getSocketFactory()),
+                entity -> uncheck(() -> EntityUtils.toString(entity)));
+    }
+
+    StateV1HealthClient(ApacheHttpClient apacheHttpClient, Function<HttpEntity, String> getContentFunction) {
+        httpClient = apacheHttpClient;
+        this.getContentFunction = getContentFunction;
+    }
+
+    HealthInfo get() throws Exception {
+        return httpClient.get(this::handle);
+    }
+
+    private HealthInfo handle(CloseableHttpResponse httpResponse) throws IOException {
+        int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
+        if (httpStatusCode < 200 || httpStatusCode >= 300) {
+            return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
+        }
+
+        HttpEntity bodyEntity = httpResponse.getEntity();
+        long contentLength = bodyEntity.getContentLength();
+        if (contentLength > MAX_CONTENT_LENGTH) {
+            throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
+        }
+        String body = getContentFunction.apply(bodyEntity);
+        HealthResponse healthResponse = MAPPER.readValue(body, HealthResponse.class);
+
+        if (healthResponse.status == null || healthResponse.status.code == null) {
+            return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
+        } else {
+            return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
+        }
+    }
+
+    @Override
+    public void close() {
+        try {
+            httpClient.close();
+        } catch (Exception e) {
+            logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e);
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
new file mode 100644
index 00000000000..8eca03c616f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
@@ -0,0 +1,59 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ServiceId;
+
+import java.net.URL;
+import java.time.Duration;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+class StateV1HealthEndpoint implements HealthEndpoint {
+    private final ServiceId serviceId;
+    private final URL url;
+    private final Duration requestTimeout;
+    private final Duration connectionKeepAlive;
+    private final Duration delay;
+    private final RunletExecutor executor;
+
+    StateV1HealthEndpoint(ServiceId serviceId,
+                          HostName hostname,
+                          int port,
+                          Duration delay,
+                          Duration requestTimeout,
+                          Duration connectionKeepAlive,
+                          RunletExecutor executor) {
+        this.serviceId = serviceId;
+        this.delay = delay;
+        this.executor = executor;
+        this.url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health"));
+        this.requestTimeout = requestTimeout;
+        this.connectionKeepAlive = connectionKeepAlive;
+    }
+
+    @Override
+    public ServiceId getServiceId() {
+        return serviceId;
+    }
+
+    @Override
+    public HealthMonitor startMonitoring() {
+        StateV1HealthUpdater updater = new StateV1HealthUpdater(url, requestTimeout, connectionKeepAlive);
+        return new StateV1HealthMonitor(updater, executor, delay);
+    }
+
+    @Override
+    public String description() {
+        return url.toString();
+    }
+
+    @Override
+    public String toString() {
+        return description();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
new file mode 100644
index 00000000000..5e8979deb9f
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
@@ -0,0 +1,74 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
+import com.yahoo.vespa.service.model.ServiceId;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author hakonhall
+ */
+public class StateV1HealthModel implements AutoCloseable {
+    private static final String PORT_TAG_STATE = "STATE";
+    private static final String PORT_TAG_HTTP = "HTTP";
+
+    /** Port tags implying /state/v1/health is served on HTTP. */
+    public static final List<String> HTTP_HEALTH_PORT_TAGS = Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE);
+    private final Duration targetHealthStaleness;
+    private final Duration requestTimeout;
+    private final Duration connectionKeepAlive;
+    private final RunletExecutor executor;
+
+    StateV1HealthModel(Duration targetHealthStaleness,
+                       Duration requestTimeout,
+                       Duration connectionKeepAlive,
+                       RunletExecutor executor) {
+        this.targetHealthStaleness = targetHealthStaleness;
+        this.requestTimeout = requestTimeout;
+        this.connectionKeepAlive = connectionKeepAlive;
+        this.executor = executor;
+    }
+
+    Map<ServiceId, HealthEndpoint> extractHealthEndpoints(ApplicationInfo application) {
+        Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
+
+        for (HostInfo hostInfo : application.getModel().getHosts()) {
+            HostName hostname = HostName.from(hostInfo.getHostname());
+            for (ServiceInfo serviceInfo : hostInfo.getServices()) {
+                ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo);
+                for (PortInfo portInfo : serviceInfo.getPorts()) {
+                    if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) {
+                        StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint(
+                                serviceId,
+                                hostname,
+                                portInfo.getPort(),
+                                targetHealthStaleness,
+                                requestTimeout,
+                                connectionKeepAlive,
+                                executor);
+                        endpoints.put(serviceId, endpoint);
+                        break; // Avoid >1 endpoints per serviceId
+                    }
+                }
+            }
+        }
+
+        return endpoints;
+    }
+
+    @Override
+    public void close() {
+        executor.close();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java
new file mode 100644
index 00000000000..d37797c7be9
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.service.executor.Cancellable;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+
+import java.time.Duration;
+
+/**
+ * Used to monitor the health of a single URL endpoint.
+ *
+ * @author hakon
+ */
+class StateV1HealthMonitor implements HealthMonitor {
+    private final StateV1HealthUpdater updater;
+    private final Cancellable periodicExecution;
+
+    StateV1HealthMonitor(StateV1HealthUpdater updater, RunletExecutor executor, Duration delay) {
+        this.updater = updater;
+        this.periodicExecution = executor.scheduleWithFixedDelay(updater, delay);
+    }
+
+    @Override
+    public ServiceStatus getStatus() {
+        return updater.getLatestHealthInfo().toServiceStatus();
+    }
+
+    @Override
+    public void close() {
+        periodicExecution.cancel();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
new file mode 100644
index 00000000000..011ec3b3212
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
@@ -0,0 +1,41 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import java.net.URL;
+import java.time.Duration;
+
+/**
+ * @author hakonhall
+ */
+class StateV1HealthUpdater implements HealthUpdater {
+    private final StateV1HealthClient healthClient;
+
+    private volatile HealthInfo lastHealthInfo = HealthInfo.empty();
+
+    StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) {
+        this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive));
+    }
+
+    StateV1HealthUpdater(StateV1HealthClient healthClient) {
+        this.healthClient = healthClient;
+    }
+
+    @Override
+    public HealthInfo getLatestHealthInfo() {
+        return lastHealthInfo;
+    }
+
+    @Override
+    public void run() {
+        try {
+            lastHealthInfo = healthClient.get();
+        } catch (Exception e) {
+            lastHealthInfo = HealthInfo.fromException(e);
+        }
+    }
+
+    @Override
+    public void close() {
+        healthClient.close();
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java
new file mode 100644
index 00000000000..eb6f92d928c
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java
@@ -0,0 +1,79 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class CancellableImplTest {
+    private final TestExecutor executor = new TestExecutor();
+    private final TestRunlet runlet = new TestRunlet();
+    private final Cancellable cancellable = executor.scheduleWithFixedDelay(runlet, Duration.ofSeconds(1));
+
+    @After
+    public void tearDown() {
+        executor.close();
+    }
+
+    @Before
+    public void setUp() {
+        assertEquals(0, runlet.getRunsStarted());
+        executor.runToCompletion(1);
+        assertEquals(1, runlet.getRunsStarted());
+        executor.runToCompletion(2);
+        assertEquals(2, runlet.getRunsStarted());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+    }
+
+    @Test
+    public void testCancelWhileIdle() {
+        // Cancel while runlet is not running and verify closure and executor cancellation
+        cancellable.cancel();
+        assertFalse(executor.isExecutionRunning());
+        assertTrue(runlet.isClosed());
+
+        // Ensure a spurious run is ignored.
+        executor.runAsync();
+        executor.runToCompletion(3);
+        assertEquals(2, runlet.getRunsStarted());
+    }
+
+    @Test
+    public void testCancelWhileRunning() {
+        // halt execution in runlet
+        runlet.shouldWaitInRun(true);
+        executor.runAsync();
+        runlet.waitUntilInRun();
+        assertEquals(3, runlet.getRunsStarted());
+        assertEquals(2, runlet.getRunsCompleted());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+
+        // Cancel now
+        cancellable.cancel();
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+
+        // Complete the runlet.run(), and verify the close and executor cancellation takes effect
+        runlet.shouldWaitInRun(false);
+        executor.waitUntilRunCompleted(3);
+        assertFalse(executor.isExecutionRunning());
+        assertTrue(runlet.isClosed());
+
+        // Ensure a spurious run is ignored.
+        executor.runToCompletion(4);
+        assertEquals(3, runlet.getRunsStarted());
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java
new file mode 100644
index 00000000000..9828d6300ed
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java
@@ -0,0 +1,71 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import org.junit.After;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class RunletExecutorImplTest {
+    private final RunletExecutorImpl executor = new RunletExecutorImpl(2);
+
+    @After
+    public void tearDown() {
+        executor.close();
+    }
+
+    @Test
+    public void testAFewCancellations() {
+        for (int i = 0; i < 10; ++i) {
+            TestRunlet runlet = new TestRunlet();
+            Cancellable cancellable = schedule(runlet);
+            runlet.waitUntilCompleted(5);
+            cancellable.cancel();
+            runlet.waitUntilClosed();
+        }
+    }
+
+    @Test
+    public void testCongestedThreadPool() {
+        TestRunlet runlet1 = new TestRunlet();
+        runlet1.shouldWaitInRun(true);
+        Cancellable cancellable1 = schedule(runlet1);
+        runlet1.waitUntilInRun();
+
+        TestRunlet runlet2 = new TestRunlet();
+        runlet2.shouldWaitInRun(true);
+        Cancellable cancellable2 = schedule(runlet2);
+        runlet2.waitUntilInRun();
+
+        TestRunlet runlet3 = new TestRunlet();
+        Cancellable cancellable3 = schedule(runlet3);
+        try { Thread.sleep(10); } catch (InterruptedException ignored) { }
+        assertEquals(0, runlet3.getRunsStarted());
+
+        cancellable3.cancel();
+        assertTrue(runlet3.isClosed());
+        assertEquals(0, runlet3.getRunsStarted());
+
+        runlet1.shouldWaitInRun(false);
+        runlet2.shouldWaitInRun(false);
+        cancellable1.cancel();
+        cancellable2.cancel();
+    }
+
+    @Test
+    public void testWithoutCancellation() {
+        TestRunlet runlet = new TestRunlet();
+        Cancellable toBeIgnored = schedule(runlet);
+        runlet.waitUntilCompleted(2);
+    }
+
+    private Cancellable schedule(Runlet runlet) {
+        return executor.scheduleWithFixedDelay(runlet, Duration.ofMillis(20));
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java
new file mode 100644
index 00000000000..c40fc03ea00
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java
@@ -0,0 +1,105 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author hakonhall
+ */
+public class TestExecutor implements RunletExecutor {
+    private List<Thread> threads = new ArrayList<>();
+
+    private Runlet runlet;
+    private CancellableImpl cancellable;
+
+    private final Object monitor = new Object();
+    private boolean afterRun = false;
+    private boolean waitAfterRun = false;
+    private int runsCompleted = 0;
+
+    private final Runnable cancelExecution = () -> executionRunning = false;
+    private volatile boolean executionRunning = true;
+
+    @Override
+    public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) {
+        if (this.runlet != null) {
+            throw new IllegalStateException("TestExecutor only supports execution of one runlet");
+        }
+
+        this.runlet = runlet;
+        this.cancellable = new CancellableImpl(runlet);
+        this.cancellable.setPeriodicExecutionCancellationCallback(cancelExecution);
+        return this::cancel;
+    }
+
+    private void cancel() {
+        cancellable.cancel();
+    }
+
+    boolean isExecutionRunning() {
+        return executionRunning;
+    }
+
+    void runAsync() {
+        Thread thread = new Thread(this::threadMain);
+        thread.start();
+        threads.add(thread);
+    }
+
+    void runToCompletion(int run) {
+        runAsync();
+        waitUntilRunCompleted(run);
+    }
+
+    private void threadMain() {
+        cancellable.run();
+
+        synchronized (monitor) {
+            ++runsCompleted;
+            afterRun = true;
+            monitor.notifyAll();
+
+            while (waitAfterRun) {
+                monitor.notifyAll();
+            }
+            afterRun = false;
+        }
+    }
+
+    void setWaitAfterRun(boolean waitAfterRun) {
+        synchronized (monitor) {
+            this.waitAfterRun = waitAfterRun;
+        }
+    }
+
+    void waitUntilAfterRun() {
+        synchronized (monitor) {
+            while (!afterRun) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilRunCompleted(int run) {
+        synchronized (monitor) {
+            while (runsCompleted < run) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void uncheckedWait() {
+        try {
+            monitor.wait();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void close() {
+        threads.forEach(thread -> { try { thread.join(); } catch (InterruptedException ignored) {} });
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java
new file mode 100644
index 00000000000..7e671dccd96
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java
@@ -0,0 +1,98 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * @author hakonhall
+ */
+public class TestRunlet implements Runlet {
+    private final Object monitor = new Object();
+    private boolean running = false;
+    private boolean shouldWaitInRun = false;
+    private boolean closed = false;
+    private int runsStarted = 0;
+    private int runsCompleted = 0;
+
+    int getRunsStarted() {
+        synchronized (monitor) {
+            return runsStarted;
+        }
+    }
+
+    int getRunsCompleted() {
+        return runsCompleted;
+    }
+
+    boolean isClosed() {
+        synchronized (monitor) {
+            return closed;
+        }
+    }
+
+    void shouldWaitInRun(boolean value) {
+        synchronized (monitor) {
+            shouldWaitInRun = value;
+            monitor.notifyAll();
+        }
+    }
+
+    void waitUntilInRun() {
+        synchronized (monitor) {
+            while (!running) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilCompleted(int runsCompleted) {
+        synchronized (monitor) {
+            while (this.runsCompleted < runsCompleted) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilClosed() {
+        synchronized (monitor) {
+            while (!closed) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    @Override
+    public void run() {
+        synchronized (monitor) {
+            if (closed) {
+                throw new IllegalStateException("run after close");
+            }
+
+            ++runsStarted;
+            running = true;
+            monitor.notifyAll();
+
+            while (shouldWaitInRun) {
+                uncheckedWait();
+            }
+
+            ++runsCompleted;
+            running = false;
+            monitor.notifyAll();
+        }
+    }
+
+    @Override
+    public void close() {
+        synchronized (monitor) {
+            closed = true;
+            monitor.notifyAll();
+        }
+    }
+
+    private void uncheckedWait() {
+        try {
+            monitor.wait();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
index 0dfca12099e..821f5282998 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
@@ -1,47 +1,92 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
+import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.model.ServiceId;
 import com.yahoo.vespa.service.monitor.ConfigserverUtil;
 import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
-import java.util.function.Function;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class ApplicationHealthMonitorTest {
     private final ConfigServerApplication configServerApplication = new ConfigServerApplication();
 
     @Test
-    public void sanityCheck() {
-        MonitorFactory monitorFactory = new MonitorFactory();
-
+    public void activationAndRemoval() {
         HealthMonitor monitor1 = mock(HealthMonitor.class);
         HealthMonitor monitor2 = mock(HealthMonitor.class);
         HealthMonitor monitor3 = mock(HealthMonitor.class);
 
-        monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1);
-        monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2);
-        monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3);
+        ApplicationInfo configServer = ConfigserverUtil.makeExampleConfigServer();
+        StateV1HealthModel model = mock(StateV1HealthModel.class);
+        ApplicationHealthMonitor applicationMonitor = new ApplicationHealthMonitor(configServer.getApplicationId(), model);
+
+        // Activate with cfg1-2
+        HealthEndpoint endpoint1 = mock(HealthEndpoint.class);
+        HealthEndpoint endpoint2 = mock(HealthEndpoint.class);
+        Map<ServiceId, HealthEndpoint> initialEndpoints = new HashMap<>();
+        initialEndpoints.put(serviceIdOf("cfg1"), endpoint1);
+        initialEndpoints.put(serviceIdOf("cfg2"), endpoint2);
+
+        when(model.extractHealthEndpoints(configServer)).thenReturn(initialEndpoints);
+        when(endpoint1.startMonitoring()).thenReturn(monitor1);
+        when(endpoint2.startMonitoring()).thenReturn(monitor2);
+        applicationMonitor.monitor(configServer);
+
+        verify(endpoint1, times(1)).startMonitoring();
+        verify(endpoint2, times(1)).startMonitoring();
 
         when(monitor1.getStatus()).thenReturn(ServiceStatus.UP);
         when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN);
-        when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED);
-
-        ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring(
-                ConfigserverUtil.makeExampleConfigServer(),
-                monitorFactory);
+        when(monitor3.getStatus()).thenReturn(ServiceStatus.UP);
 
         assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1"));
         assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2"));
         assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg3"));
+
+        // Update application to contain cfg2-3
+        HealthEndpoint endpoint3 = mock(HealthEndpoint.class);
+        when(endpoint3.startMonitoring()).thenReturn(monitor3);
+        Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
+        endpoints.put(serviceIdOf("cfg2"), endpoint2);
+        endpoints.put(serviceIdOf("cfg3"), endpoint3);
+        when(model.extractHealthEndpoints(configServer)).thenReturn(endpoints);
+        applicationMonitor.monitor(configServer);
+
+        // Only monitor1 has been removed and had its close called
+        verify(monitor1, times(1)).close();
+        verify(monitor2, never()).close();
+        verify(monitor3, never()).close();
+
+        // Only endpoint3 started monitoring from last monitor()
+        verify(endpoint1, times(1)).startMonitoring();
+        verify(endpoint2, times(1)).startMonitoring();
+        verify(endpoint3, times(1)).startMonitoring();
+
+        // Now cfg1 will be NOT_CHECKED, while cfg3 should be UP.
+        assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg1"));
+        assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2"));
+        assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg3"));
+
+        applicationMonitor.close();
+    }
+
+    private ServiceId serviceIdOf(String hostname) {
+        return new ServiceId(configServerApplication.getApplicationId(),
+                configServerApplication.getClusterId(),
+                configServerApplication.getServiceType(),
+                configServerApplication.configIdFor(HostName.from(hostname)));
     }
 
     private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) {
@@ -51,70 +96,4 @@ public class ApplicationHealthMonitorTest {
                 configServerApplication.getServiceType(),
                 configServerApplication.configIdFor(HostName.from(hostname)));
     }
-
-    private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> {
-        private Map<String, EndpointInfo> endpointMonitors = new HashMap<>();
-
-        public void expectEndpoint(String url, HealthMonitor monitorToReturn) {
-            endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn));
-        }
-
-        @Override
-        public HealthMonitor apply(HealthEndpoint endpoint) {
-            String url = endpoint.getStateV1HealthUrl().toString();
-            EndpointInfo info = endpointMonitors.get(url);
-            if (info == null) {
-                throw new IllegalArgumentException("Endpoint not expected: " + url);
-            }
-
-            if (info.isEndpointDiscovered()) {
-                throw new IllegalArgumentException("A HealthMonitor has already been created to " + url);
-            }
-
-            info.setEndpointDiscovered(true);
-
-            return info.getMonitorToReturn();
-        }
-    }
-
-    private static class EndpointInfo {
-        private final String url;
-        private final HealthMonitor monitorToReturn;
-
-        private boolean endpointDiscovered = false;
-
-        private EndpointInfo(String url, HealthMonitor monitorToReturn) {
-            this.url = url;
-            this.monitorToReturn = monitorToReturn;
-        }
-
-        public String getUrl() {
-            return url;
-        }
-
-        public boolean isEndpointDiscovered() {
-            return endpointDiscovered;
-        }
-
-        public void setEndpointDiscovered(boolean endpointDiscovered) {
-            this.endpointDiscovered = endpointDiscovered;
-        }
-
-        public HealthMonitor getMonitorToReturn() {
-            return monitorToReturn;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            EndpointInfo that = (EndpointInfo) o;
-            return Objects.equals(url, that.url);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(url);
-        }
-    }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
index f420f5c1284..86b0ee4a8f3 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
@@ -49,7 +49,9 @@ public class HealthMonitorManagerTest {
         when(monitorInfra.value()).thenReturn(false);
         ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer();
         manager.applicationActivated(applicationInfo);
+        verify(monitor, times(1)).monitor(applicationInfo);
         manager.applicationRemoved(applicationInfo.getApplicationId());
+        verify(monitor, times(1)).close();
     }
 
     @Test
@@ -73,7 +75,7 @@ public class HealthMonitorManagerTest {
         ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
 
         manager.applicationActivated(proxyHostApplicationInfo);
-        verify(monitorFactory, never()).create(proxyHostApplicationInfo);
+        verify(monitorFactory, never()).create(proxyHostApplicationInfo.getApplicationId());
 
         assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1");
     }
@@ -88,7 +90,7 @@ public class HealthMonitorManagerTest {
         ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
 
         manager.applicationActivated(proxyHostApplicationInfo);
-        verify(monitorFactory, times(1)).create(proxyHostApplicationInfo);
+        verify(monitorFactory, times(1)).create(proxyHostApplicationInfo.getApplicationId());
 
         when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP);
         assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1");
@@ -98,6 +100,11 @@ public class HealthMonitorManagerTest {
         assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1");
     }
 
+    @Test
+    public void threadPoolSize() {
+        assertEquals(9, HealthMonitorManager.THREAD_POOL_SIZE);
+    }
+
     private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) {
         ServiceStatus actual = manager.getStatus(
                 infraApplication.getApplicationId(),
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java
deleted file mode 100644
index 94ba4726ad0..00000000000
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.yahoo.vespa.applicationmodel.ServiceStatus;
-import org.junit.Test;
-
-import java.time.Duration;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class HealthMonitorTest {
-    @Test
-    public void initiallyDown() {
-        HealthClient healthClient = mock(HealthClient.class);
-        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) {
-            monitor.startMonitoring();
-            assertEquals(ServiceStatus.DOWN, monitor.getStatus());
-        }
-    }
-
-    @Test
-    public void eventuallyUp() {
-        HealthClient healthClient = mock(HealthClient.class);
-        when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE));
-        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) {
-            monitor.startMonitoring();
-
-            while (monitor.getStatus() != ServiceStatus.UP) {
-                try {
-                    Thread.sleep(1);
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
new file mode 100644
index 00000000000..480691772bb
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
@@ -0,0 +1,66 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.service.duper.ProxyHostApplication;
+import com.yahoo.vespa.service.executor.Cancellable;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ServiceId;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class StateV1HealthModelTest {
+    private RunletExecutor executor = mock(RunletExecutor.class);
+    private Duration healthStaleness = Duration.ofSeconds(1);
+    private Duration requestTimeout = Duration.ofSeconds(2);
+    private Duration keepAlive = Duration.ofSeconds(3);
+    private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor);
+    private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
+    private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::from).collect(Collectors.toList());
+    private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
+    private final Map<ServiceId, HealthEndpoint> endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo);
+
+    @Test
+    public void test() {
+        assertEquals(2, endpoints.size());
+
+        ApplicationId applicationId = ApplicationId.from("hosted-vespa", "proxy-host", "default");
+        ClusterId clusterId = new ClusterId("proxy-host");
+        ServiceId hostAdmin1 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host1"));
+        ServiceId hostAdmin2 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host2"));
+
+        HealthEndpoint endpoint1 = endpoints.get(hostAdmin1);
+        assertNotNull(endpoint1);
+        assertEquals("http://host1:8080/state/v1/health", endpoint1.description());
+
+        HealthEndpoint endpoint2 = endpoints.get(hostAdmin2);
+        assertNotNull(endpoint2);
+        assertEquals("http://host2:8080/state/v1/health", endpoint2.description());
+
+        Cancellable cancellable = mock(Cancellable.class);
+        when(executor.scheduleWithFixedDelay(any(), any())).thenReturn(cancellable);
+        try (HealthMonitor healthMonitor = endpoint1.startMonitoring()) {
+            assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus());
+        }
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
new file mode 100644
index 00000000000..c892118990f
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
@@ -0,0 +1,37 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.executor.RunletExecutorImpl;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class StateV1HealthMonitorTest {
+    @Test
+    public void downThenUpThenDown() throws Exception {
+        StateV1HealthClient client = mock(StateV1HealthClient.class);
+        when(client.get()).thenReturn(HealthInfo.empty());
+
+        StateV1HealthUpdater updater = new StateV1HealthUpdater(client);
+        RunletExecutor executor = new RunletExecutorImpl(2);
+        try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) {
+            assertEquals(ServiceStatus.DOWN, monitor.getStatus());
+
+            when(client.get()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE));
+            while (monitor.getStatus() != ServiceStatus.UP) {
+                try { Thread.sleep(2); } catch (InterruptedException ignored) { }
+            }
+
+            when(client.get()).thenReturn(HealthInfo.fromException(new IllegalStateException("foo")));
+            while (monitor.getStatus() != ServiceStatus.DOWN) {
+                try { Thread.sleep(2); } catch (InterruptedException ignored) { }
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
index 157b5565071..e7b7a829dac 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
@@ -1,16 +1,18 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import org.apache.http.HttpEntity;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.net.URL;
+import java.util.function.Function;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -19,7 +21,14 @@ import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class HealthClientTest {
+public class StateV1HealthUpdaterTest {
+    private URL url;
+
+    @Before
+    public void setUp() throws Exception{
+        url = new URL("http://host.com:19071");
+    }
+
     @Test
     public void successfulRequestResponse() throws IOException {
         HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
@@ -96,7 +105,6 @@ public class HealthClientTest {
 
     private HealthInfo getHealthInfoFromJsonResponse(String content)
             throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         CloseableHttpResponse response = mock(CloseableHttpResponse.class);
@@ -110,22 +118,22 @@ public class HealthClientTest {
         HttpEntity httpEntity = mock(HttpEntity.class);
         when(response.getEntity()).thenReturn(httpEntity);
 
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) {
-
+        try (StateV1HealthUpdater updater = makeUpdater(client, entry -> content)) {
             when(httpEntity.getContentLength()).thenReturn((long) content.length());
-            return healthClient.getHealthInfo();
+            updater.run();
+            return updater.getLatestHealthInfo();
         }
     }
 
     @Test
     public void testRequestException() throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string"));
 
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> "")) {
-            HealthInfo info = healthClient.getHealthInfo();
+        try (StateV1HealthUpdater updater = makeUpdater(client, entry -> "")) {
+            updater.run();
+            HealthInfo info = updater.getLatestHealthInfo();
             assertFalse(info.isHealthy());
             assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
             assertEquals("Exception: exception string", info.toString());
@@ -135,7 +143,6 @@ public class HealthClientTest {
     @Test
     public void testBadHttpResponseCode()
             throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         CloseableHttpResponse response = mock(CloseableHttpResponse.class);
@@ -150,13 +157,19 @@ public class HealthClientTest {
         when(response.getEntity()).thenReturn(httpEntity);
 
         String content = "{}";
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) {
-
+        try (HealthUpdater updater = makeUpdater(client, entry -> content)) {
             when(httpEntity.getContentLength()).thenReturn((long) content.length());
-            HealthInfo info = healthClient.getHealthInfo();
+            updater.run();
+            HealthInfo info = updater.getLatestHealthInfo();
             assertFalse(info.isHealthy());
             assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
             assertEquals("Bad HTTP response status code 500", info.toString());
         }
     }
+
+    private StateV1HealthUpdater makeUpdater(CloseableHttpClient client, Function<HttpEntity, String> getContentFunction) {
+        ApacheHttpClient apacheHttpClient = new ApacheHttpClient(url, client);
+        StateV1HealthClient healthClient = new StateV1HealthClient(apacheHttpClient, getContentFunction);
+        return new StateV1HealthUpdater(healthClient);
+    }
 }
\ No newline at end of file