diff options
author | Bjørn Meland <bjormel@users.noreply.github.com> | 2020-05-19 17:01:59 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-19 17:01:59 +0200 |
commit | ae0ac11a0ef7d790964f51dadb2653857a7ee591 (patch) | |
tree | 4f1d1aa222bd2cfd69bde5a6838dbe2d7effaea5 /controller-server | |
parent | 7c4ec805157ce9f662377cfff2367e3711e47ac0 (diff) |
Reporting metrics from audit log with user as dimension (#13293)
* Reporting metrics from audit log with user as dimension
* Clean up after code review
* Iterate over audit log first, then set metrics
* Simplify
Diffstat (limited to 'controller-server')
2 files changed, 100 insertions, 0 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 15f01c558ad..cc4a8c628eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -52,6 +53,7 @@ public class MetricsReporter extends ControllerMaintainer { public static final String PLATFORM_NODE_COUNT = "deployment.nodeCountByPlatformVersion"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; + public static final String OPERATION_PREFIX = "operation."; private final Metric metric; private final Clock clock; @@ -71,6 +73,39 @@ public class MetricsReporter extends ControllerMaintainer { reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); + reportAuditLog(); + } + + private void reportAuditLog() { + AuditLog log = controller().auditLogger().readLog(); + HashMap<String, HashMap<String, Integer>> metricCounts = new HashMap<>(); + + for (AuditLog.Entry entry : log.entries()) { + String[] resource = entry.resource().split("/"); + if((resource.length > 1) && (resource[1] != null)) { + String api = resource[1]; + String operationMetric = OPERATION_PREFIX + api; + HashMap<String, Integer> dimension = metricCounts.get(operationMetric); + if (dimension != null) { + Integer count = dimension.get(entry.principal()); + if (count != null) { + dimension.replace(entry.principal(), ++count); + } else { + dimension.put(entry.principal(), 1); + } + + } else { + dimension = new HashMap<>(); + dimension.put(entry.principal(),1); + metricCounts.put(operationMetric, dimension); + } + } + } + for (String operationMetric : metricCounts.keySet()) { + for (String userDimension : metricCounts.get(operationMetric).keySet()) { + metric.set(operationMetric, (metricCounts.get(operationMetric)).get(userDimension), metric.createContext(Map.of("operator", userDimension))); + } + } } private void reportInfrastructureUpgradeMetrics() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 168f9da776f..6eeabc98932 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -44,6 +45,64 @@ public class MetricsReporterTest { private final MetricsMock metrics = new MetricsMock(); @Test + public void audit_log_metric() { + var tester = new ControllerTester(); + + MetricsReporter metricsReporter = createReporter(tester.controller()); + // Log some operator actions + HttpRequest req1 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/some_region/nodes/v2/state/dirty/hostname", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req1.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req1)); + HttpRequest req2 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_tenant/application/some_app/instance/default/environment/prod/region/some-region", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req2.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req2)); + + // Report metrics + metricsReporter.maintain(); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); + + // Log some more operator actions + HttpRequest req3 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req3.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req3)); + HttpRequest req4 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req4.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req4)); + HttpRequest req5 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req5.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req5)); + HttpRequest req6 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req6.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req6)); + + // Report metrics + metricsReporter.maintain(); + assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); + assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.johndoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.janedoe")); + } + + @Test public void deployment_fail_ratio() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -483,6 +542,12 @@ public class MetricsReporterTest { .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + id)); } + private Number getMetric(String name, String operator) { + return metrics.getMetric((dimensions) -> operator.equals(dimensions.get("operator")), + name) + .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + operator)); + } + private MetricsReporter createReporter(Controller controller) { return new MetricsReporter(controller, metrics); } |