Health rest API

Makes a new REST API /orchestrator/v1/health/<ApplicationId> that shows the
list of services that are monitored for health. This information is currently a
bit difficult to infer from
/orchestrator/v1/instances/<ApplicationInstanceReference> since it is the
combined view of health and Slobrok. There are already APIs for Slobrok.

Example content:

$ curl -s localhost:19071/orchestrator/v1/health/hosted-vespa:zone-config-serve\
rs:default|jq .
  {
    "services": [
      {
        "clusterId": "zone-config-servers",
        "serviceType": "configserver",
        "configId": "zone-config-servers/cfg6",
        "status": {
          "serviceStatus": "UP",
          "lastChecked": 1548939111.708718,
          "since": 1548939051.686223,
          "endpoint": "http://cfg4.prod.cd-us-central-1.vespahosted.ne1.yahoo.com:19071/state/v1/health"
        }
      },
      ...
    ]
  }

This view is slightly different from the application model view, just because
that's exactly how the health monitoring is structured (individual monitors
against endpoints).

The "endpoint" information will also be added to /instances if the status comes
from health and not Slobrok.

13 files changed, 99 insertions, 29 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
index e728d1ea914..873baeae710 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java
@@ -8,13 +8,14 @@ import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.applicationmodel.ServiceType;
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Responsible for monitoring a whole application using /state/v1/health.
@@ -24,6 +25,10 @@ import java.util.Set;
 class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
     private final ApplicationId applicationId;
     private final StateV1HealthModel healthModel;
+
+    // Guards against concurrent access to monitors field w/objects in 1. monitor() (called from e.g. DuperModel),
+    // 2. getStatus() called from caching layer and Orchestrator above, and 3. REST calls to getAllServiceStatuses().
+    private final Object guard = new Object();
     private final Map<ServiceId, HealthMonitor> monitors = new HashMap<>();
 
     ApplicationHealthMonitor(ApplicationId applicationId, StateV1HealthModel healthModel) {
@@ -38,13 +43,15 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
 
         Map<ServiceId, HealthEndpoint> endpoints = healthModel.extractHealthEndpoints(applicationInfo);
 
-        // Remove obsolete monitors
-        Set<ServiceId> removed = new HashSet<>(monitors.keySet());
-        removed.removeAll(endpoints.keySet());
-        removed.stream().map(monitors::remove).forEach(HealthMonitor::close);
+        synchronized (guard) {
+            // Remove obsolete monitors
+            Set<ServiceId> removed = new HashSet<>(monitors.keySet());
+            removed.removeAll(endpoints.keySet());
+            removed.stream().map(monitors::remove).forEach(HealthMonitor::close);
 
-        // Add new monitors.
-        endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring()));
+            // Add new monitors.
+            endpoints.forEach((serviceId, endpoint) -> monitors.computeIfAbsent(serviceId, ignoredId -> endpoint.startMonitoring()));
+        }
     }
 
     @Override
@@ -53,12 +60,23 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable {
                                        ServiceType serviceType,
                                        ConfigId configId) {
         ServiceId serviceId = new ServiceId(applicationId, clusterId, serviceType, configId);
-        HealthMonitor monitor = monitors.get(serviceId);
-        if (monitor == null) {
-            return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED);
+
+        synchronized (guard) {
+            HealthMonitor monitor = monitors.get(serviceId);
+            if (monitor == null) {
+                return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED);
+            }
+
+            return monitor.getStatus();
         }
+    }
 
-        return monitor.getStatus();
+    public Map<ServiceId, ServiceStatusInfo> getAllServiceStatuses() {
+        synchronized (guard) {
+            return monitors.entrySet().stream().collect(Collectors.toMap(
+                    entry -> entry.getKey(),
+                    entry -> entry.getValue().getStatus()));
+        }
     }
 
     @Override
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
index 8c4997634a0..eae5ef3f284 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthEndpoint.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 
 /**
  * An endpoint 1-1 with a service and that can be health monitored.
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index b802c6c5413..ad9a50f8127 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -14,9 +14,14 @@ import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.RunletExecutorImpl;
+import com.yahoo.vespa.service.manager.HealthMonitorApi;
 import com.yahoo.vespa.service.manager.MonitorManager;
+import com.yahoo.vespa.service.monitor.ServiceId;
 
 import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -24,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
  *
  * @author hakon
  */
-public class HealthMonitorManager implements MonitorManager {
+public class HealthMonitorManager implements MonitorManager, HealthMonitorApi {
     // Weight the following against each other:
     //  - The number of threads N working on health checking
     //  - The health request timeout T
@@ -139,4 +144,19 @@ public class HealthMonitorManager implements MonitorManager {
 
         return false;
     }
+
+    @Override
+    public List<ApplicationId> getMonitoredApplicationIds() {
+        return Collections.list(healthMonitors.keys());
+    }
+
+    @Override
+    public Map<ServiceId, ServiceStatusInfo> getServices(ApplicationId applicationId) {
+        ApplicationHealthMonitor applicationHealthMonitor = healthMonitors.get(applicationId);
+        if (applicationHealthMonitor == null) {
+            return Collections.emptyMap();
+        }
+
+        return applicationHealthMonitor.getAllServiceStatuses();
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
index 8eca03c616f..73f0480bf96 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.service.health;
 
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.service.executor.RunletExecutor;
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 
 import java.net.URL;
 import java.time.Duration;
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
index 04943f81478..f015ecadbfc 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
@@ -6,10 +6,11 @@ import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.duper.HostAdminApplication;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.RunletExecutor;
 import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 
 import java.time.Duration;
 import java.util.Arrays;
@@ -53,20 +54,26 @@ public class StateV1HealthModel implements AutoCloseable {
             HostName hostname = HostName.from(hostInfo.getHostname());
             for (ServiceInfo serviceInfo : hostInfo.getServices()) {
 
-                if (monitorTenantHostHealth && isZoneApplication &&
-                        !ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) {
-                    // Only the node admin/host admin cluster of the zone application should be monitored
-                    // TODO: Move the node admin cluster out to a separate infrastructure application
-                    continue;
+                boolean isNodeAdmin = false;
+                if (monitorTenantHostHealth && isZoneApplication) {
+                    if (ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) {
+                        isNodeAdmin = true;
+                    } else {
+                        // Only the node admin/host admin cluster of the zone application should be monitored
+                        // TODO: Move the node admin cluster out to a separate infrastructure application
+                        continue;
+                    }
                 }
 
                 ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo);
                 for (PortInfo portInfo : serviceInfo.getPorts()) {
                     if (portInfo.getTags().containsAll(HTTP_HEALTH_PORT_TAGS)) {
+                        // The host-admin-in-zone-application is one big hack.
+                        int port = isNodeAdmin ? HostAdminApplication.HOST_ADMIN_HEALT_PORT : portInfo.getPort();
                         StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint(
                                 serviceId,
                                 hostname,
-                                portInfo.getPort(),
+                                port,
                                 targetHealthStaleness,
                                 requestTimeout,
                                 connectionKeepAlive,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
index 972e81ce822..6a6768aa78b 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java
@@ -13,15 +13,17 @@ import java.util.Optional;
  * @author hakonhall
  */
 class StateV1HealthUpdater implements HealthUpdater {
+    private final String endpoint;
     private final StateV1HealthClient healthClient;
 
     private volatile ServiceStatusInfo serviceStatusInfo = new ServiceStatusInfo(ServiceStatus.NOT_CHECKED);
 
     StateV1HealthUpdater(URL url, Duration requestTimeout, Duration connectionKeepAlive) {
-        this(new StateV1HealthClient(url, requestTimeout, connectionKeepAlive));
+        this(url.toString(), new StateV1HealthClient(url, requestTimeout, connectionKeepAlive));
     }
 
-    StateV1HealthUpdater(StateV1HealthClient healthClient) {
+    StateV1HealthUpdater(String endpoint, StateV1HealthClient healthClient) {
+        this.endpoint = endpoint;
         this.healthClient = healthClient;
     }
 
@@ -46,7 +48,8 @@ class StateV1HealthUpdater implements HealthUpdater {
         Optional<Instant> newSince = newServiceStatus == serviceStatusInfo.serviceStatus() ?
                 serviceStatusInfo.since() : Optional.of(now);
 
-        serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now), healthInfo.getErrorDescription());
+        serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now),
+                healthInfo.getErrorDescription(), Optional.of(endpoint));
     }
 
     @Override
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java
new file mode 100644
index 00000000000..7bc22db93cd
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/HealthMonitorApi.java
@@ -0,0 +1,21 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.manager;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
+import com.yahoo.vespa.service.health.HealthMonitorManager;
+import com.yahoo.vespa.service.monitor.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The API of {@link HealthMonitorManager} which is exported to other bundles (typically for REST).
+ *
+ * @author hakonhall
+ */
+public interface HealthMonitorApi extends ServiceStatusProvider {
+    List<ApplicationId> getMonitoredApplicationIds();
+    Map<ServiceId, ServiceStatusInfo> getServices(ApplicationId applicationId);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
index 9e6b64f0c91..0a4e8e75adf 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.monitor.ServiceId;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 import java.util.HashMap;
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java
index c0bcff7284c..a97cb1812c1 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java
@@ -1,5 +1,5 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.model;
+package com.yahoo.vespa.service.monitor;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.applicationmodel.ClusterId;
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
index 8861fa7db9f..2ad0eb8f67d 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
@@ -6,7 +6,7 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 import com.yahoo.vespa.service.monitor.ConfigserverUtil;
 import org.junit.Test;
 
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
index b6b88b0c0d3..fcbc3cd9b9f 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
@@ -13,7 +13,7 @@ import com.yahoo.vespa.service.duper.TestZoneApplication;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.executor.Cancellable;
 import com.yahoo.vespa.service.executor.RunletExecutor;
-import com.yahoo.vespa.service.model.ServiceId;
+import com.yahoo.vespa.service.monitor.ServiceId;
 import org.junit.Test;
 
 import java.time.Duration;
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
index 533e5173ccc..e3fb7d08d93 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
@@ -17,7 +17,7 @@ public class StateV1HealthMonitorTest {
     public void downThenUpThenDown() throws Exception {
         StateV1HealthClient client = mock(StateV1HealthClient.class);
 
-        StateV1HealthUpdater updater = new StateV1HealthUpdater(client);
+        StateV1HealthUpdater updater = new StateV1HealthUpdater("https://foo/state/v1/health", client);
         RunletExecutor executor = new RunletExecutorImpl(2);
         try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) {
             assertEquals(ServiceStatus.NOT_CHECKED, monitor.getStatus().serviceStatus());
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
index b8f108a480e..a491307d6dc 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
@@ -165,6 +165,6 @@ public class StateV1HealthUpdaterTest {
     private StateV1HealthUpdater makeUpdater(CloseableHttpClient client, Function<HttpEntity, String> getContentFunction) {
         ApacheHttpClient apacheHttpClient = new ApacheHttpClient(url, client);
         StateV1HealthClient healthClient = new StateV1HealthClient(apacheHttpClient, getContentFunction);
-        return new StateV1HealthUpdater(healthClient);
+        return new StateV1HealthUpdater(url.toString(), healthClient);
     }
 }
\ No newline at end of file