Monitor cfg app health if activated

11 files changed, 560 insertions, 118 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java
index bd2658db8aa..7c78d61da30 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitor.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
 
 /**
  * Responsible for monitoring a whole application using /state/v1/health.
@@ -37,7 +38,13 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
     private final Map<ServiceId, HealthMonitor> healthMonitors;
 
     public static ApplicationHealthMonitor startMonitoring(ApplicationInfo application) {
-        return new ApplicationHealthMonitor(makeHealthMonitors(application));
+        return startMonitoring(application, HealthMonitor::new);
+    }
+
+    /** For testing. */
+    static ApplicationHealthMonitor startMonitoring(ApplicationInfo application,
+                                                    Function<HealthEndpoint, HealthMonitor> mapper) {
+        return new ApplicationHealthMonitor(makeHealthMonitors(application, mapper));
     }
 
     private ApplicationHealthMonitor(Map<ServiceId, HealthMonitor> healthMonitors) {
@@ -64,7 +71,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
         healthMonitors.clear();
     }
 
-    private static Map<ServiceId, HealthMonitor> makeHealthMonitors(ApplicationInfo application) {
+    private static Map<ServiceId, HealthMonitor> makeHealthMonitors(
+            ApplicationInfo application, Function<HealthEndpoint, HealthMonitor> monitorFactory) {
         Map<ServiceId, HealthMonitor> healthMonitors = new HashMap<>();
         for (HostInfo hostInfo : application.getModel().getHosts()) {
             for (ServiceInfo serviceInfo : hostInfo.getServices()) {
@@ -73,7 +81,8 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
                             application,
                             hostInfo,
                             serviceInfo,
-                            portInfo)
+                            portInfo,
+                            monitorFactory)
                             .ifPresent(healthMonitor -> healthMonitors.put(
                                     ApplicationInstanceGenerator.getServiceId(application, serviceInfo),
                                     healthMonitor));
@@ -87,14 +96,14 @@ public class ApplicationHealthMonitor implements ServiceStatusProvider, AutoClos
             ApplicationInfo applicationInfo,
             HostInfo hostInfo,
             ServiceInfo serviceInfo,
-            PortInfo portInfo) {
+            PortInfo portInfo,
+            Function<HealthEndpoint, HealthMonitor> monitorFactory) {
         if (portInfo.getTags().containsAll(PORT_TAGS_HEALTH)) {
             HostName hostname = HostName.from(hostInfo.getHostname());
             HealthEndpoint endpoint = HealthEndpoint.forHttp(hostname, portInfo.getPort());
-            // todo: make HealthMonitor
-            // HealthMonitor healthMonitor = new HealthMonitor(endpoint);
-            // healthMonitor.startMonitoring();
-            return Optional.empty();
+            HealthMonitor healthMonitor = monitorFactory.apply(endpoint);
+            healthMonitor.startMonitoring();
+            return Optional.of(healthMonitor);
         }
 
         return Optional.empty();
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java
index 43a02a385be..128ca1b5d18 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthClient.java
@@ -14,7 +14,6 @@ import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.ConnectionKeepAliveStrategy;
 import org.apache.http.conn.HttpClientConnectionManager;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.client.HttpClients;
@@ -23,8 +22,16 @@ import org.apache.http.protocol.HttpContext;
 import org.apache.http.util.EntityUtils;
 
 import javax.net.ssl.SSLContext;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
 
 /**
+ * Health client
+ *
+ * NOT thread-safe.
+ *
  * @author hakon
  */
 public class HealthClient implements AutoCloseable, ServiceIdentityProvider.Listener {
@@ -47,27 +54,76 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List
             };
 
     private final HealthEndpoint endpoint;
+    private final Supplier<CloseableHttpClient> clientSupplier;
+    private final Function<HttpEntity, String> getContentFunction;
 
-    private volatile CloseableHttpClient httpClient;
+    private CloseableHttpClient httpClient = null;
 
     public HealthClient(HealthEndpoint endpoint) {
+        this(endpoint,
+                () -> makeCloseableHttpClient(endpoint),
+                entity -> uncheck(() -> EntityUtils.toString(entity)));
+    }
+
+    /** For testing. */
+    HealthClient(HealthEndpoint endpoint,
+                 Supplier<CloseableHttpClient> clientSupplier,
+                 Function<HttpEntity, String> getContentFunction) {
         this.endpoint = endpoint;
+        this.clientSupplier = clientSupplier;
+        this.getContentFunction = getContentFunction;
     }
 
     public void start() {
-        endpoint.getServiceIdentityProvider().ifPresent(provider -> {
-            onCredentialsUpdate(provider.getIdentitySslContext(), null);
-            provider.addIdentityListener(this);
-        });
+        updateHttpClient();
+        endpoint.registerListener(this);
     }
 
     @Override
     public void onCredentialsUpdate(SSLContext sslContext, AthenzService ignored) {
-        SSLConnectionSocketFactory socketFactory =
-                new SSLConnectionSocketFactory(sslContext, endpoint.getHostnameVerifier().orElse(null));
+        updateHttpClient();
+    }
+
+    public HealthEndpoint getEndpoint() {
+        return endpoint;
+    }
 
+    public HealthInfo getHealthInfo() {
+        try {
+            return probeHealth();
+        } catch (Exception e) {
+            return HealthInfo.fromException(e);
+        }
+    }
+
+    @Override
+    public void close() {
+        endpoint.removeListener(this);
+
+        if (httpClient != null) {
+            try {
+                httpClient.close();
+            } catch (Exception e) {
+                // ignore
+            }
+            httpClient = null;
+        }
+    }
+
+    private void updateHttpClient() {
+        CloseableHttpClient httpClient = clientSupplier.get();
+
+        if (this.httpClient != null) {
+            // Note: close() can be called any number of times.
+            uncheck(() -> this.httpClient.close());
+        }
+
+        this.httpClient = httpClient;
+    }
+
+    private static CloseableHttpClient makeCloseableHttpClient(HealthEndpoint endpoint) {
         Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
-                .register("https", socketFactory)
+                .register(endpoint.getStateV1HealthUrl().getProtocol(), endpoint.getConnectionSocketFactory())
                 .build();
 
         HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry);
@@ -78,7 +134,7 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List
                 .setSocketTimeout(DEFAULT_TIMEOUT_MILLIS) // waiting for data
                 .build();
 
-        this.httpClient = HttpClients.custom()
+        return HttpClients.custom()
                 .setKeepAliveStrategy(KEEP_ALIVE_STRATEGY)
                 .setConnectionManager(connectionManager)
                 .disableAutomaticRetries()
@@ -86,54 +142,37 @@ public class HealthClient implements AutoCloseable, ServiceIdentityProvider.List
                 .build();
     }
 
-    public HealthInfo getHealthInfo() {
-        try {
-            return probeHealth();
-        } catch (Exception e) {
-            return HealthInfo.fromException(e);
-        }
-    }
-
-    @Override
-    public void close() {
-        endpoint.getServiceIdentityProvider().ifPresent(provider -> provider.removeIdentityListener(this));
-
-        try {
-            httpClient.close();
-        } catch (Exception e) {
-            // ignore
-        }
-        httpClient = null;
-    }
-
     private HealthInfo probeHealth() throws Exception {
         HttpGet httpget = new HttpGet(endpoint.getStateV1HealthUrl().toString());
-        CloseableHttpResponse httpResponse;
 
         CloseableHttpClient httpClient = this.httpClient;
         if (httpClient == null) {
-            throw new IllegalStateException("HTTP client has closed");
+            throw new IllegalStateException("HTTP client never started or has closed");
         }
 
-        httpResponse = httpClient.execute(httpget);
+        CloseableHttpResponse httpResponse = httpClient.execute(httpget);
 
-        int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
-        if (httpStatusCode < 200 || httpStatusCode >= 300) {
-            return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
-        }
-
-        HttpEntity bodyEntity = httpResponse.getEntity();
-        long contentLength = bodyEntity.getContentLength();
-        if (contentLength > MAX_CONTENT_LENGTH) {
-            throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
-        }
-        String body = EntityUtils.toString(bodyEntity);
-        HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class);
-
-        if (healthResponse.status == null || healthResponse.status.code == null) {
-            return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
-        } else {
-            return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
+        try {
+            int httpStatusCode = httpResponse.getStatusLine().getStatusCode();
+            if (httpStatusCode < 200 || httpStatusCode >= 300) {
+                return HealthInfo.fromBadHttpStatusCode(httpStatusCode);
+            }
+
+            HttpEntity bodyEntity = httpResponse.getEntity();
+            long contentLength = bodyEntity.getContentLength();
+            if (contentLength > MAX_CONTENT_LENGTH) {
+                throw new IllegalArgumentException("Content too long: " + contentLength + " bytes");
+            }
+            String body = getContentFunction.apply(bodyEntity);
+            HealthResponse healthResponse = mapper.readValue(body, HealthResponse.class);
+
+            if (healthResponse.status == null || healthResponse.status.code == null) {
+                return HealthInfo.fromHealthStatusCode(HealthResponse.Status.DEFAULT_STATUS);
+            } else {
+                return HealthInfo.fromHealthStatusCode(healthResponse.status.code);
+            }
+        } finally {
+            httpResponse.close();
         }
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java
index e9d17a9ab70..bc667a60bc7 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthEndpoint.java
@@ -5,25 +5,22 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
 
 import javax.net.ssl.HostnameVerifier;
 import java.net.URL;
 import java.util.Collections;
-import java.util.Optional;
 
 import static com.yahoo.yolean.Exceptions.uncheck;
 
 /**
  * @author hakon
  */
-class HealthEndpoint {
-    private final URL url;
-    private final Optional<HostnameVerifier> hostnameVerifier;
-    private final Optional<ServiceIdentityProvider> serviceIdentityProvider;
+public interface HealthEndpoint {
 
     static HealthEndpoint forHttp(HostName hostname, int port) {
         URL url = uncheck(() -> new URL("http", hostname.value(), port, "/state/v1/health"));
-        return new HealthEndpoint(url, Optional.empty(), Optional.empty());
+        return new HttpHealthEndpoint(url);
     }
 
     static HealthEndpoint forHttps(HostName hostname,
@@ -32,26 +29,12 @@ class HealthEndpoint {
                                    AthenzIdentity remoteIdentity) {
         URL url = uncheck(() -> new URL("https", hostname.value(), port, "/state/v1/health"));
         HostnameVerifier peerVerifier = new AthenzIdentityVerifier(Collections.singleton(remoteIdentity));
-        return new HealthEndpoint(url, Optional.of(serviceIdentityProvider), Optional.of(peerVerifier));
+        return new HttpsHealthEndpoint(url, serviceIdentityProvider, peerVerifier);
     }
 
-    private HealthEndpoint(URL url,
-                           Optional<ServiceIdentityProvider> serviceIdentityProvider,
-                           Optional<HostnameVerifier> hostnameVerifier) {
-        this.url = url;
-        this.serviceIdentityProvider = serviceIdentityProvider;
-        this.hostnameVerifier = hostnameVerifier;
-    }
-
-    public URL getStateV1HealthUrl() {
-        return url;
-    }
-
-    public Optional<ServiceIdentityProvider> getServiceIdentityProvider() {
-        return serviceIdentityProvider;
-    }
-
-    public Optional<HostnameVerifier> getHostnameVerifier() {
-        return hostnameVerifier;
-    }
+    URL getStateV1HealthUrl();
+    ConnectionSocketFactory getConnectionSocketFactory();
+    void registerListener(ServiceIdentityProvider.Listener listener);
+    void removeListener(ServiceIdentityProvider.Listener listener);
+    String toString();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java
index a3fe3cb3106..8b724afba5f 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthInfo.java
@@ -50,7 +50,17 @@ public class HealthInfo {
         return healthStatusCode.map(UP_STATUS_CODE::equals).orElse(false);
     }
 
-    public ServiceStatus toSerivceStatus() {
+    public ServiceStatus toServiceStatus() {
+        // Bootstrapping ServiceStatus: To avoid thundering herd problem at startup,
+        // the clients will not fetch the health immediately. What should the ServiceStatus
+        // be before the first health has been fetched?
+        //
+        // NOT_CHECKED: Logically the right thing, but if an Orchestrator gets a suspend request
+        // in this window, and another service within the cluster is down, it ends up allowing
+        // suspension when it shouldn't have done so.
+        //
+        // DOWN: Only safe initial value, possibly except if the first initial delay is long,
+        // as that could indicate it has been down for too long.
         return isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN;
     }
 
@@ -65,7 +75,7 @@ public class HealthInfo {
         } else if (healthStatusCode.isPresent()) {
             return "Bad health status code '" + healthStatusCode.get() + "'";
         } else if (exception.isPresent()) {
-            return Exceptions.toMessageString(exception.get());
+            return "Exception: " + Exceptions.toMessageString(exception.get());
         } else if (httpStatusCode.isPresent()) {
             return "Bad HTTP response status code " + httpStatusCode.getAsInt();
         } else {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java
index fd809b32918..5518f9ddeba 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitor.java
@@ -13,40 +13,48 @@ import java.util.logging.Logger;
 /**
  * Used to monitor the health of a single URL endpoint.
  *
+ * <p>Must be closed on successful start of monitoring ({}
+ *
+ * <p>Thread-safe
+ *
  * @author hakon
  */
 public class HealthMonitor implements AutoCloseable {
     private static final Logger logger = Logger.getLogger(HealthMonitor.class.getName());
-    private static final Duration DELAY = Duration.ofSeconds(20);
+
+    /** The duration between each health request. */
+    private static final Duration DEFAULT_DELAY = Duration.ofSeconds(10);
+
     // About 'static': Javadoc says "Instances of java.util.Random are threadsafe."
     private static final Random random = new Random();
 
     private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
     private final HealthClient healthClient;
+    private final Duration delay;
 
     private volatile HealthInfo lastHealthInfo = HealthInfo.empty();
 
     public HealthMonitor(HealthEndpoint stateV1HealthEndpoint) {
-        this.healthClient = new HealthClient(stateV1HealthEndpoint);
+        this(new HealthClient(stateV1HealthEndpoint), DEFAULT_DELAY);
     }
 
     /** For testing. */
-    HealthMonitor(HealthClient healthClient) {
+    HealthMonitor(HealthClient healthClient, Duration delay) {
         this.healthClient = healthClient;
+        this.delay = delay;
     }
 
     public void startMonitoring() {
         healthClient.start();
         executor.scheduleWithFixedDelay(
                 this::updateSynchronously,
-                initialDelayInSeconds(DELAY.getSeconds()),
-                DELAY.getSeconds(),
-                TimeUnit.SECONDS);
+                initialDelayInMillis(delay.toMillis()),
+                delay.toMillis(),
+                TimeUnit.MILLISECONDS);
     }
 
     public ServiceStatus getStatus() {
-        // todo: return lastHealthInfo.toServiceStatus();
-        return ServiceStatus.NOT_CHECKED;
+        return lastHealthInfo.toServiceStatus();
     }
 
     @Override
@@ -63,11 +71,17 @@ public class HealthMonitor implements AutoCloseable {
         healthClient.close();
     }
 
-    private long initialDelayInSeconds(long maxInitialDelayInSeconds) {
+    private long initialDelayInMillis(long maxInitialDelayInSeconds) {
         return random.nextLong() % maxInitialDelayInSeconds;
     }
 
     private void updateSynchronously() {
-        lastHealthInfo = healthClient.getHealthInfo();
+        try {
+            lastHealthInfo = healthClient.getHealthInfo();
+        } catch (Throwable t) {
+            // An uncaught exception will kill the executor.scheduleWithFixedDelay thread!
+            logger.log(LogLevel.WARNING, "Failed to get health info for " +
+                    healthClient.getEndpoint(), t);
+        }
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java
index 473ef5e3a94..383cb6961a7 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java
@@ -9,17 +9,20 @@ import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
 import com.yahoo.vespa.service.monitor.application.ZoneApplication;
 import com.yahoo.vespa.service.monitor.internal.MonitorManager;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
+ * Manages all /state/v1/health related monitoring.
+ *
  * @author hakon
  */
 public class HealthMonitorManager implements MonitorManager {
-    private final Map<ApplicationId, ApplicationHealthMonitor> healthMonitors = new HashMap<>();
+    private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors =
+            new ConcurrentHashMap<>();
     private final ConfigserverConfig configserverConfig;
 
     @Inject
@@ -29,7 +32,7 @@ public class HealthMonitorManager implements MonitorManager {
 
     @Override
     public void applicationActivated(ApplicationInfo application) {
-        if (applicationMonitored(application.getApplicationId())) {
+        if (applicationMonitoredForHealth(application.getApplicationId())) {
             ApplicationHealthMonitor monitor =
                     ApplicationHealthMonitor.startMonitoring(application);
             healthMonitors.put(application.getApplicationId(), monitor);
@@ -38,11 +41,9 @@ public class HealthMonitorManager implements MonitorManager {
 
     @Override
     public void applicationRemoved(ApplicationId id) {
-        if (applicationMonitored(id)) {
-            ApplicationHealthMonitor monitor = healthMonitors.remove(id);
-            if (monitor != null) {
-                monitor.close();
-            }
+        ApplicationHealthMonitor monitor = healthMonitors.remove(id);
+        if (monitor != null) {
+            monitor.close();
         }
     }
 
@@ -58,11 +59,15 @@ public class HealthMonitorManager implements MonitorManager {
             return ServiceStatus.UP;
         }
 
-        return ServiceStatus.NOT_CHECKED;
+        ApplicationHealthMonitor monitor = healthMonitors.get(applicationId);
+        if (monitor == null) {
+            return ServiceStatus.NOT_CHECKED;
+        }
+
+        return monitor.getStatus(applicationId, clusterId, serviceType, configId);
     }
 
-    private boolean applicationMonitored(ApplicationId id) {
-        // todo: health-check config server
-        return false;
+    private boolean applicationMonitoredForHealth(ApplicationId id) {
+        return id.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId());
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java
new file mode 100644
index 00000000000..5f4a1d1024b
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpHealthEndpoint.java
@@ -0,0 +1,44 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor.internal.health;
+
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+
+import java.net.URL;
+
+/**
+ * @author hakon
+ */
+class HttpHealthEndpoint implements HealthEndpoint {
+    private final URL url;
+    private final ConnectionSocketFactory socketFactory;
+
+    HttpHealthEndpoint(URL url) {
+        this.url = url;
+        this.socketFactory = PlainConnectionSocketFactory.getSocketFactory();
+    }
+
+    @Override
+    public URL getStateV1HealthUrl() {
+        return url;
+    }
+
+    @Override
+    public ConnectionSocketFactory getConnectionSocketFactory() {
+        return socketFactory;
+    }
+
+    @Override
+    public void registerListener(ServiceIdentityProvider.Listener listener) {
+    }
+
+    @Override
+    public void removeListener(ServiceIdentityProvider.Listener listener) {
+    }
+
+    @Override
+    public String toString() {
+        return url.toString();
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java
new file mode 100644
index 00000000000..d1198cda78d
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HttpsHealthEndpoint.java
@@ -0,0 +1,53 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor.internal.health;
+
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import java.net.URL;
+
+/**
+ * @author hakon
+ */
+public class HttpsHealthEndpoint implements HealthEndpoint {
+    private final URL url;
+    private final HostnameVerifier hostnameVerifier;
+    private final ServiceIdentityProvider serviceIdentityProvider;
+
+    HttpsHealthEndpoint(URL url,
+                        ServiceIdentityProvider serviceIdentityProvider,
+                        HostnameVerifier hostnameVerifier) {
+        this.url = url;
+        this.serviceIdentityProvider = serviceIdentityProvider;
+        this.hostnameVerifier = hostnameVerifier;
+    }
+
+    @Override
+    public URL getStateV1HealthUrl() {
+        return url;
+    }
+
+    @Override
+    public ConnectionSocketFactory getConnectionSocketFactory() {
+        SSLContext sslContext = serviceIdentityProvider.getIdentitySslContext();
+        return new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
+    }
+
+    @Override
+    public void registerListener(ServiceIdentityProvider.Listener listener) {
+        serviceIdentityProvider.addIdentityListener(listener);
+    }
+
+    @Override
+    public void removeListener(ServiceIdentityProvider.Listener listener) {
+        serviceIdentityProvider.removeIdentityListener(listener);
+    }
+
+    @Override
+    public String toString() {
+        return url.toString();
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java
index 51b0503565f..b0fdb14726f 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/ApplicationHealthMonitorTest.java
@@ -6,19 +6,121 @@ import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
 import com.yahoo.vespa.service.monitor.internal.ConfigserverUtil;
 import org.junit.Test;
 
-import static com.yahoo.vespa.applicationmodel.ServiceStatus.NOT_CHECKED;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class ApplicationHealthMonitorTest {
     @Test
     public void sanityCheck() {
-        ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(
-                ConfigserverUtil.makeExampleConfigServer());
-        ServiceStatus status = monitor.getStatus(
+        MonitorFactory monitorFactory = new MonitorFactory();
+
+        HealthMonitor monitor1 = mock(HealthMonitor.class);
+        HealthMonitor monitor2 = mock(HealthMonitor.class);
+        HealthMonitor monitor3 = mock(HealthMonitor.class);
+
+        monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1);
+        monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2);
+        monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3);
+
+        when(monitor1.getStatus()).thenReturn(ServiceStatus.UP);
+        when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN);
+        when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED);
+
+        ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring(
+                ConfigserverUtil.makeExampleConfigServer(),
+                monitorFactory);
+
+        ServiceStatus status1 = applicationMonitor.getStatus(
                 ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(),
                 ConfigServerApplication.CLUSTER_ID,
                 ConfigServerApplication.SERVICE_TYPE,
                 ConfigServerApplication.configIdFrom(0));
-        assertEquals(NOT_CHECKED, status);
+        assertEquals(ServiceStatus.UP, status1);
+
+        ServiceStatus status2 = applicationMonitor.getStatus(
+                ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(),
+                ConfigServerApplication.CLUSTER_ID,
+                ConfigServerApplication.SERVICE_TYPE,
+                ConfigServerApplication.configIdFrom(1));
+        assertEquals(ServiceStatus.DOWN, status2);
+
+        ServiceStatus status3 = applicationMonitor.getStatus(
+                ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(),
+                ConfigServerApplication.CLUSTER_ID,
+                ConfigServerApplication.SERVICE_TYPE,
+                ConfigServerApplication.configIdFrom(2));
+        assertEquals(ServiceStatus.NOT_CHECKED, status3);
+    }
+
+    private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> {
+        private Map<String, EndpointInfo> endpointMonitors = new HashMap<>();
+
+        public void expectEndpoint(String url, HealthMonitor monitorToReturn) {
+            endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn));
+        }
+
+        @Override
+        public HealthMonitor apply(HealthEndpoint endpoint) {
+            String url = endpoint.getStateV1HealthUrl().toString();
+            EndpointInfo info = endpointMonitors.get(url);
+            if (info == null) {
+                throw new IllegalArgumentException("Endpoint not expected: " + url);
+            }
+
+            if (info.isEndpointDiscovered()) {
+                throw new IllegalArgumentException("A HealthMonitor has already been created to " + url);
+            }
+
+            info.setEndpointDiscovered(true);
+
+            return info.getMonitorToReturn();
+        }
+    }
+
+    private static class EndpointInfo {
+        private final String url;
+        private final HealthMonitor monitorToReturn;
+
+        private boolean endpointDiscovered = false;
+
+        private EndpointInfo(String url, HealthMonitor monitorToReturn) {
+            this.url = url;
+            this.monitorToReturn = monitorToReturn;
+        }
+
+        public String getUrl() {
+            return url;
+        }
+
+        public boolean isEndpointDiscovered() {
+            return endpointDiscovered;
+        }
+
+        public void setEndpointDiscovered(boolean endpointDiscovered) {
+            this.endpointDiscovered = endpointDiscovered;
+        }
+
+        public HealthMonitor getMonitorToReturn() {
+            return monitorToReturn;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            EndpointInfo that = (EndpointInfo) o;
+            return Objects.equals(url, that.url);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(url);
+        }
     }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java
new file mode 100644
index 00000000000..c3e06faaf92
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthClientTest.java
@@ -0,0 +1,165 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor.internal.health;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class HealthClientTest {
+    @Test
+    public void successfulRequestResponse() throws IOException {
+        HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
+                "    \"metrics\": {\n" +
+                "        \"snapshot\": {\n" +
+                "            \"from\": 1.528789829249E9,\n" +
+                "            \"to\": 1.528789889249E9\n" +
+                "        }\n" +
+                "    },\n" +
+                "    \"status\": {\"code\": \"up\"},\n" +
+                "    \"time\": 1528789889364\n" +
+                "}");
+        assertTrue(info.isHealthy());
+        assertEquals(ServiceStatus.UP, info.toServiceStatus());
+    }
+
+    @Test
+    public void notUpResponse() throws IOException {
+        HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
+                "    \"metrics\": {\n" +
+                "        \"snapshot\": {\n" +
+                "            \"from\": 1.528789829249E9,\n" +
+                "            \"to\": 1.528789889249E9\n" +
+                "        }\n" +
+                "    },\n" +
+                "    \"status\": {\"code\": \"initializing\"},\n" +
+                "    \"time\": 1528789889364\n" +
+                "}");
+        assertFalse(info.isHealthy());
+        assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+        assertEquals("Bad health status code 'initializing'", info.toString());
+    }
+
+    @Test
+    public void noCodeInResponse() throws IOException {
+        HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
+                "    \"metrics\": {\n" +
+                "        \"snapshot\": {\n" +
+                "            \"from\": 1.528789829249E9,\n" +
+                "            \"to\": 1.528789889249E9\n" +
+                "        }\n" +
+                "    },\n" +
+                "    \"status\": {\"foo\": \"bar\"},\n" +
+                "    \"time\": 1528789889364\n" +
+                "}");
+        assertFalse(info.isHealthy());
+        assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+        assertEquals("Bad health status code 'down'", info.toString());
+    }
+
+    @Test
+    public void noStatusInResponse() throws IOException {
+        HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
+                "    \"metrics\": {\n" +
+                "        \"snapshot\": {\n" +
+                "            \"from\": 1.528789829249E9,\n" +
+                "            \"to\": 1.528789889249E9\n" +
+                "        }\n" +
+                "    },\n" +
+                "    \"time\": 1528789889364\n" +
+                "}");
+        assertFalse(info.isHealthy());
+        assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+        assertEquals("Bad health status code 'down'", info.toString());
+    }
+
+    @Test
+    public void badJson() throws IOException {
+        HealthInfo info = getHealthInfoFromJsonResponse("} foo bar");
+        assertFalse(info.isHealthy());
+        assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+        assertTrue(info.toString().startsWith("Exception: Unexpected close marker '}': "));
+    }
+
+    private HealthInfo getHealthInfoFromJsonResponse(String content)
+            throws IOException {
+        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
+        CloseableHttpClient client = mock(CloseableHttpClient.class);
+
+        CloseableHttpResponse response = mock(CloseableHttpResponse.class);
+        when(client.execute(any())).thenReturn(response);
+
+        StatusLine statusLine = mock(StatusLine.class);
+        when(response.getStatusLine()).thenReturn(statusLine);
+
+        when(statusLine.getStatusCode()).thenReturn(200);
+
+        HttpEntity httpEntity = mock(HttpEntity.class);
+        when(response.getEntity()).thenReturn(httpEntity);
+
+        try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) {
+            healthClient.start();
+
+            when(httpEntity.getContentLength()).thenReturn((long) content.length());
+            return healthClient.getHealthInfo();
+        }
+    }
+
+    @Test
+    public void testRequestException() throws IOException {
+        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
+        CloseableHttpClient client = mock(CloseableHttpClient.class);
+
+        when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string"));
+
+        try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> "")) {
+            healthClient.start();
+            HealthInfo info = healthClient.getHealthInfo();
+            assertFalse(info.isHealthy());
+            assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+            assertEquals("Exception: exception string", info.toString());
+        }
+    }
+
+    @Test
+    public void testBadHttpResponseCode()
+            throws IOException {
+        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
+        CloseableHttpClient client = mock(CloseableHttpClient.class);
+
+        CloseableHttpResponse response = mock(CloseableHttpResponse.class);
+        when(client.execute(any())).thenReturn(response);
+
+        StatusLine statusLine = mock(StatusLine.class);
+        when(response.getStatusLine()).thenReturn(statusLine);
+
+        when(statusLine.getStatusCode()).thenReturn(500);
+
+        HttpEntity httpEntity = mock(HttpEntity.class);
+        when(response.getEntity()).thenReturn(httpEntity);
+
+        String content = "{}";
+        try (HealthClient healthClient = new HealthClient(endpoint, () -> client, entry -> content)) {
+            healthClient.start();
+
+            when(httpEntity.getContentLength()).thenReturn((long) content.length());
+            HealthInfo info = healthClient.getHealthInfo();
+            assertFalse(info.isHealthy());
+            assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
+            assertEquals("Bad HTTP response status code 500", info.toString());
+        }
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java
index cca1530ad97..2a203027353 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorTest.java
@@ -4,18 +4,36 @@ package com.yahoo.vespa.service.monitor.internal.health;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import org.junit.Test;
 
-import java.net.MalformedURLException;
+import java.time.Duration;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class HealthMonitorTest {
     @Test
-    public void basicTests() throws MalformedURLException {
+    public void initiallyDown() {
         HealthClient healthClient = mock(HealthClient.class);
-        try (HealthMonitor monitor = new HealthMonitor(healthClient)) {
+        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) {
             monitor.startMonitoring();
-            assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus());
+            assertEquals(ServiceStatus.DOWN, monitor.getStatus());
+        }
+    }
+
+    @Test
+    public void eventuallyUp() {
+        HealthClient healthClient = mock(HealthClient.class);
+        when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE));
+        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) {
+            monitor.startMonitoring();
+
+            while (monitor.getStatus() != ServiceStatus.UP) {
+                try {
+                    Thread.sleep(1);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
         }
     }
 }
\ No newline at end of file