Metadata about /state/v1/health status

The service monitor uses /state/v1/health to monitor config servers and the
host admins (but not yet tenant host admins).

This commit adds some metadata about the status of a service:
 - The time the status was last checked
 - The time the status changed to the current

This can be used to e.g. make more intelligent decisions in the Orchestrator,
e.g. only allowing a service to suspend if it has been DOWN longer than X
seconds (to avoid spurious DOWN to break redundancy and uptime guarantees).

3 files changed, 26 insertions, 14 deletions

diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
index 6984389b688..cd20a01f6af 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
@@ -9,13 +9,15 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.orchestrator.InstanceLookupService;
 import com.yahoo.vespa.orchestrator.OrchestratorUtil;
 import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.service.manager.MonitorManager;
+import com.yahoo.vespa.service.manager.UnionMonitorManager;
 import com.yahoo.vespa.service.monitor.SlobrokApi;
 
 import javax.inject.Inject;
@@ -50,15 +52,18 @@ public class InstanceResource {
 
     private final StatusService statusService;
     private final SlobrokApi slobrokApi;
+    private final MonitorManager rootManager;
     private final InstanceLookupService instanceLookupService;
 
     @Inject
     public InstanceResource(@Component InstanceLookupService instanceLookupService,
                             @Component StatusService statusService,
-                            @Component SlobrokApi slobrokApi) {
+                            @Component SlobrokApi slobrokApi,
+                            @Component UnionMonitorManager rootManager) {
         this.instanceLookupService = instanceLookupService;
         this.statusService = statusService;
         this.slobrokApi = slobrokApi;
+        this.rootManager = rootManager;
     }
 
     @GET
@@ -104,9 +109,9 @@ public class InstanceResource {
     }
 
     @GET
-    @Path("/{instanceId}/serviceStatus")
+    @Path("/{instanceId}/serviceStatusInfo")
     @Produces(MediaType.APPLICATION_JSON)
-    public ServiceStatus getServiceStatus(
+    public ServiceStatusInfo getServiceStatus(
             @PathParam("instanceId") String instanceId,
             @QueryParam("clusterId") String clusterIdString,
             @QueryParam("serviceType") String serviceTypeString,
@@ -130,7 +135,7 @@ public class InstanceResource {
         ServiceType serviceType = new ServiceType(serviceTypeString);
         ConfigId configId = new ConfigId(configIdString);
 
-        return slobrokApi.getStatus(applicationId, clusterId, serviceType, configId);
+        return rootManager.getStatus(applicationId, clusterId, serviceType, configId);
     }
 
     static ApplicationInstanceReference parseInstanceId(String instanceIdString) {
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index ad1ce647a7c..8e11b85241f 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -52,9 +52,12 @@ public class ClusterApiImplTest {
 
         assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo());
         assertFalse(clusterApi.isStorageCluster());
-        assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=DOWN}, "
-                        + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=UP}, "
-                        + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=DOWN}]",
+        assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" +
+                        "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, "
+                        + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" +
+                        "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, "
+                        + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" +
+                        "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]",
                 clusterApi.servicesDownAndNotInGroupDescription());
         assertEquals("[host3, host4]",
                 clusterApi.nodesAllowedToBeDownNotInGroupDescription());
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java
index d7255327ba6..61102461bf0 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java
@@ -7,8 +7,10 @@ import com.yahoo.jrt.slobrok.api.Mirror;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse;
+import com.yahoo.vespa.service.manager.UnionMonitorManager;
 import com.yahoo.vespa.service.monitor.SlobrokApi;
 import org.junit.Test;
 
@@ -31,10 +33,12 @@ public class InstanceResourceTest {
     private static final ClusterId CLUSTER_ID = new ClusterId("cluster-id");
 
     private final SlobrokApi slobrokApi = mock(SlobrokApi.class);
+    private final UnionMonitorManager rootManager = mock(UnionMonitorManager.class);
     private final InstanceResource resource = new InstanceResource(
             null,
             null,
-            slobrokApi);
+            slobrokApi,
+            rootManager);
 
     @Test
     public void testGetSlobrokEntries() throws Exception {
@@ -47,18 +51,18 @@ public class InstanceResourceTest {
     }
 
     @Test
-    public void testGetServiceStatus() {
+    public void testGetServiceStatusInfo() {
         ServiceType serviceType = new ServiceType("serviceType");
         ConfigId configId = new ConfigId("configId");
         ServiceStatus serviceStatus = ServiceStatus.UP;
-        when(slobrokApi.getStatus(APPLICATION_ID, CLUSTER_ID, serviceType, configId))
-                .thenReturn(serviceStatus);
+        when(rootManager.getStatus(APPLICATION_ID, CLUSTER_ID, serviceType, configId))
+                .thenReturn(new ServiceStatusInfo(serviceStatus));
         ServiceStatus actualServiceStatus = resource.getServiceStatus(
                 APPLICATION_INSTANCE_REFERENCE,
                 CLUSTER_ID.s(),
                 serviceType.s(),
-                configId.s());
-        verify(slobrokApi).getStatus(APPLICATION_ID, CLUSTER_ID, serviceType, configId);
+                configId.s()).serviceStatus();
+        verify(rootManager).getStatus(APPLICATION_ID, CLUSTER_ID, serviceType, configId);
         assertEquals(serviceStatus, actualServiceStatus);
     }