diff options
author | Ola Aunrønning <olaa@verizonmedia.com> | 2022-07-07 15:49:45 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-07-07 15:49:45 +0200 |
commit | 6d01ee5b7573a455ce1c09c2fa140f724b71383c (patch) | |
tree | 9210977b3aa7ee96a2016e321f6aedf8dd37fac2 /controller-server | |
parent | 45dbbbe2b41cfe3955f431d43e2f6f66d42525be (diff) | |
parent | 4a154bd066a16e87f67dac4cfc8e17fe0381385c (diff) |
Merge pull request #23396 from vespa-engine/olaa/cmr-metrics
Add CMR tracking metric
Diffstat (limited to 'controller-server')
3 files changed, 26 insertions, 5 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 4aeecdcd4ff..9793cded918 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -70,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); - maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer)); + maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer, metric)); maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 551f803f368..daba7e74f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -46,26 +47,28 @@ public class VcmrMaintainer extends ControllerMaintainer { private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName()); private static final int DAYS_TO_RETIRE = 2; private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); + protected static final String TRACKED_CMRS_METRIC = "cmr.tracked"; private final CuratorDb curator; private final NodeRepository nodeRepository; private final ChangeRequestClient changeRequestClient; private final SystemName system; + private final Metric metric; - public VcmrMaintainer(Controller controller, Duration interval) { + public VcmrMaintainer(Controller controller, Duration interval, Metric metric) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.curator = controller.curator(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); this.system = controller.system(); + this.metric = metric; } @Override protected double maintain() { var changeRequests = curator.readChangeRequests() .stream() - .filter(shouldUpdate()) - .collect(Collectors.toList()); + .filter(shouldUpdate()).toList(); var nodesByZone = nodesByZone(); @@ -86,6 +89,7 @@ public class VcmrMaintainer extends ControllerMaintainer { }); } }); + updateMetrics(); return 1.0; } @@ -357,4 +361,15 @@ public class VcmrMaintainer extends ControllerMaintainer { return time; } + private void updateMetrics() { + var cmrsByStatus = curator.readChangeRequests() + .stream() + .collect(Collectors.groupingBy(VespaChangeRequest::getStatus)); + + for (var status : Status.values()) { + var count = cmrsByStatus.getOrDefault(status, List.of()).size(); + metric.set(TRACKED_CMRS_METRIC, count, metric.createContext(Map.of("status", status.name()))); + } + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index bfbd3836ce7..321ec3ad8ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Before; import org.junit.Test; @@ -25,6 +26,7 @@ import java.time.ZonedDateTime; import java.time.temporal.TemporalAdjusters; import java.util.List; +import static com.yahoo.vespa.hosted.controller.maintenance.VcmrMaintainer.TRACKED_CMRS_METRIC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -37,6 +39,7 @@ public class VcmrMaintainerTest { private ControllerTester tester; private VcmrMaintainer maintainer; private NodeRepositoryMock nodeRepo; + private MetricsMock metrics; private final ZoneId zoneId = ZoneId.from("prod.us-east-3"); private final ZoneId zone2 = ZoneId.from("prod.us-west-1"); private final HostName host1 = HostName.of("host1"); @@ -47,7 +50,8 @@ public class VcmrMaintainerTest { @Before public void setup() { tester = new ControllerTester(); - maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1)); + metrics = new MetricsMock(); + maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1), metrics); nodeRepo = tester.serviceRegistry().configServer().nodeRepository().allowPatching(true); } @@ -244,6 +248,8 @@ public class VcmrMaintainerTest { assertEquals(State.OUT_OF_SYNC, action.getState()); assertEquals(Status.OUT_OF_SYNC, writtenChangeRequest.getStatus()); + assertEquals(1, metrics.getMetric(context -> "OUT_OF_SYNC".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); + assertEquals(0, metrics.getMetric(context -> "REQUIRES_OPERATOR_ACTION".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); } @Test |