Merge pull request #10938 from vespa-engine/mpolden/rotation-status-last-updated

Add lastUpdated field to endpoint status

8 files changed, 84 insertions, 24 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
index e047051558b..ac5612b4e3f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -81,13 +80,14 @@ public class RotationStatusUpdater extends Maintainer {
     }
 
     private RotationStatus getStatus(Instance instance) {
-        var statusMap = new LinkedHashMap<RotationId, Map<ZoneId, RotationState>>();
+        var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>();
         for (var assignedRotation : instance.rotations()) {
             var rotation = applications.rotationRepository().getRotation(assignedRotation.rotationId());
             if (rotation.isEmpty()) continue;
-            var rotationStatus = service.getHealthStatus(rotation.get().name()).entrySet().stream()
-                                        .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue())));
-            statusMap.put(assignedRotation.rotationId(), rotationStatus);
+            var targets = service.getHealthStatus(rotation.get().name()).entrySet().stream()
+                                 .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue())));
+            var lastUpdated = controller().clock().instant();
+            statusMap.put(assignedRotation.rotationId(), new RotationStatus.Targets(targets, lastUpdated));
         }
         return RotationStatus.from(statusMap);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index c15381791e6..356a20f6eba 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -159,6 +159,7 @@ public class ApplicationSerializer {
     // RotationStatus fields
     private static final String rotationStatusField = "rotationStatus2";
     private static final String rotationIdField = "rotationId";
+    private static final String lastUpdatedField = "lastUpdated";
     private static final String rotationStateField = "state";
     private static final String statusField = "status";
 
@@ -321,11 +322,12 @@ public class ApplicationSerializer {
     }
 
     private void toSlime(RotationStatus status, Cursor array) {
-        status.asMap().forEach((rotationId, zoneStatus) -> {
+        status.asMap().forEach((rotationId, targets) -> {
             Cursor rotationObject = array.addObject();
             rotationObject.setString(rotationIdField, rotationId.asString());
+            rotationObject.setLong(lastUpdatedField, targets.lastUpdated().toEpochMilli());
             Cursor statusArray = rotationObject.setArray(statusField);
-            zoneStatus.forEach((zone, state) -> {
+            targets.asMap().forEach((zone, state) -> {
                 Cursor statusObject = statusArray.addObject();
                 zoneIdToSlime(zone, statusObject);
                 statusObject.setString(rotationStateField, state.name());
@@ -442,9 +444,11 @@ public class ApplicationSerializer {
 
     private RotationStatus rotationStatusFromSlime(Inspector parentObject) {
         var object = parentObject.field(rotationStatusField);
-        var statusMap = new LinkedHashMap<RotationId, Map<ZoneId, RotationState>>();
+        var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>();
         object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()),
-                                                                              singleRotationStatusFromSlime(statusObject.field(statusField))));
+                                                                              new RotationStatus.Targets(
+                                                                                      singleRotationStatusFromSlime(statusObject.field(statusField)),
+                                                                                      Instant.ofEpochMilli(statusObject.field(lastUpdatedField).asLong()))));
         return RotationStatus.from(statusMap);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 51cd306767a..f2025235082 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -993,8 +993,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         var array = object.setArray("endpointStatus");
         for (var rotation : rotations) {
             var statusObject = array.addObject();
+            var targets = status.of(rotation.rotationId());
             statusObject.setString("endpointId", rotation.endpointId().id());
+            statusObject.setString("rotationId", rotation.rotationId().asString());
+            statusObject.setString("clusterId", rotation.clusterId().value());
             statusObject.setString("status", rotationStateString(status.of(rotation.rotationId(), deployment)));
+            statusObject.setLong("lastUpdated", targets.lastUpdated().toEpochMilli());
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
index 087d07d9c1b..563f94ef6b8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.rotation;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 
+import java.time.Instant;
 import java.util.Map;
 import java.util.Objects;
 
@@ -16,24 +17,24 @@ public class RotationStatus {
 
     public static final RotationStatus EMPTY = new RotationStatus(Map.of());
 
-    private final Map<RotationId, Map<ZoneId, RotationState>> status;
+    private final Map<RotationId, Targets> status;
 
-    private RotationStatus(Map<RotationId, Map<ZoneId, RotationState>> status) {
+    private RotationStatus(Map<RotationId, Targets> status) {
         this.status = Map.copyOf(Objects.requireNonNull(status));
     }
 
-    public Map<RotationId, Map<ZoneId, RotationState>> asMap() {
+    public Map<RotationId, Targets> asMap() {
         return status;
     }
 
-    /** Get status of given rotation, if any */
-    public Map<ZoneId, RotationState> of(RotationId rotation) {
-        return status.getOrDefault(rotation, Map.of());
+    /** Get targets of given rotation, if any */
+    public Targets of(RotationId rotation) {
+        return status.getOrDefault(rotation, Targets.NONE);
     }
 
     /** Get status of deployment in given rotation, if any */
     public RotationState of(RotationId rotation, Deployment deployment) {
-        return of(rotation).entrySet().stream()
+        return of(rotation).asMap().entrySet().stream()
                            .filter(kv -> kv.getKey().equals(deployment.zone()))
                            .map(Map.Entry::getValue)
                            .findFirst()
@@ -58,8 +59,45 @@ public class RotationStatus {
         return Objects.hash(status);
     }
 
-    public static RotationStatus from(Map<RotationId, Map<ZoneId, RotationState>> status) {
-        return status.isEmpty() ? EMPTY : new RotationStatus(status);
+    public static RotationStatus from(Map<RotationId, Targets> targets) {
+        return targets.isEmpty() ? EMPTY : new RotationStatus(targets);
+    }
+
+    /** Targets of a rotation */
+    public static class Targets {
+
+        public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH);
+
+        private final Map<ZoneId, RotationState> targets;
+        private final Instant lastUpdated;
+
+        public Targets(Map<ZoneId, RotationState> targets, Instant lastUpdated) {
+            this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null"));
+            this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null");
+        }
+
+        public Map<ZoneId, RotationState> asMap() {
+            return targets;
+        }
+
+        public Instant lastUpdated() {
+            return lastUpdated;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Targets targets1 = (Targets) o;
+            return targets.equals(targets1.targets) &&
+                   lastUpdated.equals(targets1.lastUpdated);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(targets, lastUpdated);
+        }
+
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 5c407f8d090..186e982fd75 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -117,8 +117,10 @@ public class ApplicationSerializerTest {
         DeploymentJobs deploymentJobs = new DeploymentJobs(statusList);
 
         var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"),
-                                                        Map.of(ZoneId.from("prod", "us-west-1"), RotationState.in,
-                                                               ZoneId.from("prod", "us-east-3"), RotationState.out)));
+                                                        new RotationStatus.Targets(
+                                                                Map.of(ZoneId.from("prod", "us-west-1"), RotationState.in,
+                                                                       ZoneId.from("prod", "us-east-3"), RotationState.out),
+                                                                Instant.ofEpochMilli(42))));
 
         ApplicationId id1 = ApplicationId.from("t1", "a1", "i1");
         ApplicationId id3 = ApplicationId.from("t1", "a1", "i3");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json
index bcda7992d81..582753d04d6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json
@@ -222,7 +222,10 @@
       "endpointStatus": [
         {
           "endpointId": "default",
-          "status": "IN"
+          "rotationId": "rotation-id-1",
+          "clusterId": "foo",
+          "status": "IN",
+          "lastUpdated": "(ignore)"
         }
       ],
       "environment": "prod",
@@ -237,7 +240,10 @@
       "endpointStatus": [
         {
           "endpointId": "default",
-          "status": "UNKNOWN"
+          "rotationId": "rotation-id-1",
+          "clusterId": "foo",
+          "status": "UNKNOWN",
+          "lastUpdated": "(ignore)"
         }
       ],
       "environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
index 009d6c74dd1..951b42ff07a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
@@ -214,7 +214,10 @@
       "endpointStatus": [
         {
           "endpointId": "default",
-          "status": "IN"
+          "rotationId": "rotation-id-1",
+          "clusterId": "foo",
+          "status": "IN",
+          "lastUpdated":"(ignore)"
         }
       ],
       "environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
index 140be562fe9..143042e3600 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
@@ -5,7 +5,10 @@
   "endpointStatus": [
     {
       "endpointId": "default",
-      "status": "IN"
+      "rotationId": "rotation-id-1",
+      "clusterId": "foo",
+      "status": "IN",
+      "lastUpdated": "(ignore)"
     }
   ],
   "tenant": "tenant1",