diff options
3 files changed, 17 insertions, 5 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index c0d79861fae..402a82a9ca1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; @@ -14,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -35,17 +37,22 @@ public class CloudEventReporter extends ControllerMaintainer { private final AwsEventFetcher eventFetcher; private final Map<String, List<ZoneApi>> zonesByCloudNativeRegion; private final NodeRepository nodeRepository; + private final Metric metric; - CloudEventReporter(Controller controller, Duration interval) { + private static final String INFRASTRUCTURE_INSTANCE_EVENTS = "infrastructure_instance_events"; + + CloudEventReporter(Controller controller, Duration interval, Metric metric) { super(controller, interval); this.issueHandler = controller.serviceRegistry().issueHandler(); this.eventFetcher = controller.serviceRegistry().eventFetcherService(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.zonesByCloudNativeRegion = getZonesByCloudNativeRegion(); + this.metric = metric; } @Override protected boolean maintain() { + int numberOfInfrastructureEvents = 0; for (var awsRegion : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { @@ -53,10 +60,13 @@ public class CloudEventReporter extends ControllerMaintainer { event.instanceEventId, event.affectedInstances)); List<Node> needsManualIntervention = handleInstances(awsRegion, event); - if (!needsManualIntervention.isEmpty()) + if (!needsManualIntervention.isEmpty()) { + numberOfInfrastructureEvents += needsManualIntervention.size(); submitIssue(event); + } } } + metric.set(INFRASTRUCTURE_INSTANCE_EVENTS, numberOfInfrastructureEvents, metric.createContext(Collections.emptyMap())); return true; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 10b21ece233..336dc5ddd04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -69,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10)); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), controller.serviceRegistry().costReportConsumer()); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(1), metric, controller.serviceRegistry().meteringService()); - cloudEventReporter = new CloudEventReporter(controller, Duration.ofMinutes(30)); + cloudEventReporter = new CloudEventReporter(controller, Duration.ofMinutes(30), metric); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval); resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index b141457dc18..0749a077a77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; @@ -28,6 +29,7 @@ import static org.junit.Assert.*; public class CloudEventReporterTest { private final ControllerTester tester = new ControllerTester(); + private final MetricsMock metrics = new MetricsMock(); private final ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other"); private final ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws"); private final ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws"); @@ -44,7 +46,7 @@ public class CloudEventReporterTest { @Test public void maintain() { setUpZones(); - CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15)); + CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15), metrics); assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(nonAwsZone.getId())); assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(awsZone1.getId())); @@ -61,7 +63,7 @@ public class CloudEventReporterTest { assertEquals(1, createdIssues.size()); String description = createdIssues.get(IssueId.from("1")).issue().description(); assertTrue(description.contains("confighost")); - + assertEquals(1, metrics.getMetric("infrastructure_instance_events")); } private void mockEvents() { |