Merge pull request #23396 from vespa-engine/olaa/cmr-metrics

Add CMR tracking metric

3 files changed, 26 insertions, 5 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 4aeecdcd4ff..9793cded918 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -70,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer));
         maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer));
         maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer));
-        maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer));
+        maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer, metric));
         maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval));
         maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer));
         maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
index 551f803f368..daba7e74f34 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.text.Text;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -46,26 +47,28 @@ public class VcmrMaintainer extends ControllerMaintainer {
     private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName());
     private static final int DAYS_TO_RETIRE = 2;
     private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7);
+    protected static final String TRACKED_CMRS_METRIC = "cmr.tracked";
 
     private final CuratorDb curator;
     private final NodeRepository nodeRepository;
     private final ChangeRequestClient changeRequestClient;
     private final SystemName system;
+    private final Metric metric;
 
-    public VcmrMaintainer(Controller controller, Duration interval) {
+    public VcmrMaintainer(Controller controller, Duration interval, Metric metric) {
         super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
         this.curator = controller.curator();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
         this.changeRequestClient = controller.serviceRegistry().changeRequestClient();
         this.system = controller.system();
+        this.metric = metric;
     }
 
     @Override
     protected double maintain() {
         var changeRequests = curator.readChangeRequests()
                 .stream()
-                .filter(shouldUpdate())
-                .collect(Collectors.toList());
+                .filter(shouldUpdate()).toList();
 
         var nodesByZone = nodesByZone();
 
@@ -86,6 +89,7 @@ public class VcmrMaintainer extends ControllerMaintainer {
                         });
             }
         });
+        updateMetrics();
         return 1.0;
     }
 
@@ -357,4 +361,15 @@ public class VcmrMaintainer extends ControllerMaintainer {
         return time;
     }
 
+    private void updateMetrics() {
+        var cmrsByStatus = curator.readChangeRequests()
+                .stream()
+                .collect(Collectors.groupingBy(VespaChangeRequest::getStatus));
+
+        for (var status : Status.values()) {
+            var count = cmrsByStatus.getOrDefault(status, List.of()).size();
+            metric.set(TRACKED_CMRS_METRIC, count, metric.createContext(Map.of("status", status.name())));
+        }
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java
index bfbd3836ce7..321ec3ad8ea 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State;
 import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport;
 import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest;
 import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status;
+import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
 import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -25,6 +26,7 @@ import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAdjusters;
 import java.util.List;
 
+import static com.yahoo.vespa.hosted.controller.maintenance.VcmrMaintainer.TRACKED_CMRS_METRIC;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -37,6 +39,7 @@ public class VcmrMaintainerTest {
     private ControllerTester tester;
     private VcmrMaintainer maintainer;
     private NodeRepositoryMock nodeRepo;
+    private MetricsMock metrics;
     private final ZoneId zoneId = ZoneId.from("prod.us-east-3");
     private final ZoneId zone2 = ZoneId.from("prod.us-west-1");
     private final HostName host1 = HostName.of("host1");
@@ -47,7 +50,8 @@ public class VcmrMaintainerTest {
     @Before
     public void setup() {
         tester = new ControllerTester();
-        maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1));
+        metrics = new MetricsMock();
+        maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1), metrics);
         nodeRepo = tester.serviceRegistry().configServer().nodeRepository().allowPatching(true);
     }
 
@@ -244,6 +248,8 @@ public class VcmrMaintainerTest {
 
         assertEquals(State.OUT_OF_SYNC, action.getState());
         assertEquals(Status.OUT_OF_SYNC, writtenChangeRequest.getStatus());
+        assertEquals(1, metrics.getMetric(context -> "OUT_OF_SYNC".equals(context.get("status")), TRACKED_CMRS_METRIC).get());
+        assertEquals(0, metrics.getMetric(context -> "REQUIRES_OPERATOR_ACTION".equals(context.get("status")), TRACKED_CMRS_METRIC).get());
     }
 
     @Test