Add metric for number of infrastructure instance events (#13970)

3 files changed, 17 insertions, 5 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
index c0d79861fae..402a82a9ca1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent;
@@ -14,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl
 
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
@@ -35,17 +37,22 @@ public class CloudEventReporter extends ControllerMaintainer {
     private final AwsEventFetcher eventFetcher;
     private final Map<String, List<ZoneApi>> zonesByCloudNativeRegion;
     private final NodeRepository nodeRepository;
+    private final Metric metric;
 
-    CloudEventReporter(Controller controller, Duration interval) {
+    private static final String INFRASTRUCTURE_INSTANCE_EVENTS = "infrastructure_instance_events";
+
+    CloudEventReporter(Controller controller, Duration interval, Metric metric) {
         super(controller, interval);
         this.issueHandler = controller.serviceRegistry().issueHandler();
         this.eventFetcher = controller.serviceRegistry().eventFetcherService();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
         this.zonesByCloudNativeRegion = getZonesByCloudNativeRegion();
+        this.metric = metric;
     }
 
     @Override
     protected boolean maintain() {
+        int numberOfInfrastructureEvents = 0;
         for (var awsRegion : zonesByCloudNativeRegion.keySet()) {
             List<CloudEvent> events = eventFetcher.getEvents(awsRegion);
             for (var event : events) {
@@ -53,10 +60,13 @@ public class CloudEventReporter extends ControllerMaintainer {
                         event.instanceEventId,
                         event.affectedInstances));
                 List<Node> needsManualIntervention = handleInstances(awsRegion, event);
-                if (!needsManualIntervention.isEmpty())
+                if (!needsManualIntervention.isEmpty()) {
+                    numberOfInfrastructureEvents += needsManualIntervention.size();
                     submitIssue(event);
+                }
             }
         }
+        metric.set(INFRASTRUCTURE_INSTANCE_EVENTS, numberOfInfrastructureEvents, metric.createContext(Collections.emptyMap()));
         return true;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 10b21ece233..336dc5ddd04 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -69,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent {
         nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10));
         costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), controller.serviceRegistry().costReportConsumer());
         resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(1), metric, controller.serviceRegistry().meteringService());
-        cloudEventReporter = new CloudEventReporter(controller, Duration.ofMinutes(30));
+        cloudEventReporter = new CloudEventReporter(controller, Duration.ofMinutes(30), metric);
         rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval);
         resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger());
         systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
index b141457dc18..0749a077a77 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler;
+import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import org.junit.Test;
 
@@ -28,6 +29,7 @@ import static org.junit.Assert.*;
 public class CloudEventReporterTest {
 
     private final ControllerTester tester = new ControllerTester();
+    private final MetricsMock metrics = new MetricsMock();
     private final ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other");
     private final ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws");
     private final ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws");
@@ -44,7 +46,7 @@ public class CloudEventReporterTest {
     @Test
     public void maintain() {
         setUpZones();
-        CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15));
+        CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15), metrics);
 
         assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(nonAwsZone.getId()));
         assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(awsZone1.getId()));
@@ -61,7 +63,7 @@ public class CloudEventReporterTest {
         assertEquals(1, createdIssues.size());
         String description = createdIssues.get(IssueId.from("1")).issue().description();
         assertTrue(description.contains("confighost"));
-
+        assertEquals(1, metrics.getMetric("infrastructure_instance_events"));
     }
 
     private void mockEvents() {