Reporting metrics from audit log with user as dimension (#13293)

* Reporting metrics from audit log with user as dimension

* Clean up after code review

* Iterate over audit log first, then set metrics

* Simplify

2 files changed, 100 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 15f01c558ad..cc4a8c628eb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
 import com.yahoo.vespa.hosted.controller.deployment.JobList;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
@@ -52,6 +53,7 @@ public class MetricsReporter extends ControllerMaintainer {
     public static final String PLATFORM_NODE_COUNT = "deployment.nodeCountByPlatformVersion";
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
+    public static final String OPERATION_PREFIX = "operation.";
 
     private final Metric metric;
     private final Clock clock;
@@ -71,6 +73,39 @@ public class MetricsReporter extends ControllerMaintainer {
         reportRemainingRotations();
         reportQueuedNameServiceRequests();
         reportInfrastructureUpgradeMetrics();
+        reportAuditLog();
+    }
+
+    private void reportAuditLog() {
+        AuditLog log = controller().auditLogger().readLog();
+        HashMap<String, HashMap<String, Integer>> metricCounts = new HashMap<>();
+
+        for (AuditLog.Entry entry : log.entries()) {
+            String[] resource = entry.resource().split("/");
+            if((resource.length > 1) && (resource[1] != null)) {
+                String api = resource[1];
+                String operationMetric = OPERATION_PREFIX + api;
+                HashMap<String, Integer> dimension = metricCounts.get(operationMetric);
+                if (dimension != null) {
+                    Integer count = dimension.get(entry.principal());
+                    if (count != null) {
+                        dimension.replace(entry.principal(), ++count);
+                    } else {
+                        dimension.put(entry.principal(), 1);
+                    }
+
+                } else {
+                    dimension = new HashMap<>();
+                    dimension.put(entry.principal(),1);
+                    metricCounts.put(operationMetric, dimension);
+                }
+            }
+        }
+        for (String operationMetric : metricCounts.keySet()) {
+            for (String userDimension : metricCounts.get(operationMetric).keySet()) {
+                metric.set(operationMetric, (metricCounts.get(operationMetric)).get(userDimension), metric.createContext(Map.of("operator", userDimension)));
+            }
+        }
     }
 
     private void reportInfrastructureUpgradeMetrics() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index 168f9da776f..6eeabc98932 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -44,6 +45,64 @@ public class MetricsReporterTest {
     private final MetricsMock metrics = new MetricsMock();
 
     @Test
+    public void audit_log_metric() {
+        var tester = new ControllerTester();
+
+        MetricsReporter metricsReporter = createReporter(tester.controller());
+        // Log some operator actions
+        HttpRequest req1 = HttpRequest.createTestRequest(
+                "http://localhost:8080/zone/v2/prod/some_region/nodes/v2/state/dirty/hostname",
+                com.yahoo.jdisc.http.HttpRequest.Method.PUT
+        );
+        req1.getJDiscRequest().setUserPrincipal(() -> "user.janedoe");
+        tester.controller().auditLogger().log((req1));
+        HttpRequest req2 = HttpRequest.createTestRequest(
+                "http://localhost:8080/routing/v1/inactive/tenant/some_tenant/application/some_app/instance/default/environment/prod/region/some-region",
+                com.yahoo.jdisc.http.HttpRequest.Method.POST
+        );
+        req2.getJDiscRequest().setUserPrincipal(() -> "user.johndoe");
+        tester.controller().auditLogger().log((req2));
+
+        // Report metrics
+        metricsReporter.maintain();
+        assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe"));
+        assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe"));
+
+        // Log some more operator actions
+        HttpRequest req3 = HttpRequest.createTestRequest(
+                "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com",
+                com.yahoo.jdisc.http.HttpRequest.Method.PUT
+        );
+        req3.getJDiscRequest().setUserPrincipal(() -> "user.janedoe");
+        tester.controller().auditLogger().log((req3));
+        HttpRequest req4 = HttpRequest.createTestRequest(
+                "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1",
+                com.yahoo.jdisc.http.HttpRequest.Method.POST
+        );
+        req4.getJDiscRequest().setUserPrincipal(() -> "user.johndoe");
+        tester.controller().auditLogger().log((req4));
+        HttpRequest req5 = HttpRequest.createTestRequest(
+                "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com",
+                com.yahoo.jdisc.http.HttpRequest.Method.PUT
+        );
+        req5.getJDiscRequest().setUserPrincipal(() -> "user.johndoe");
+        tester.controller().auditLogger().log((req5));
+        HttpRequest req6 = HttpRequest.createTestRequest(
+                "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1",
+                com.yahoo.jdisc.http.HttpRequest.Method.POST
+        );
+        req6.getJDiscRequest().setUserPrincipal(() -> "user.janedoe");
+        tester.controller().auditLogger().log((req6));
+
+        // Report metrics
+        metricsReporter.maintain();
+        assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe"));
+        assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe"));
+        assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.johndoe"));
+        assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.janedoe"));
+    }
+
+    @Test
     public void deployment_fail_ratio() {
         var tester = new DeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
@@ -483,6 +542,12 @@ public class MetricsReporterTest {
                       .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + id));
     }
 
+    private Number getMetric(String name, String operator) {
+        return metrics.getMetric((dimensions) -> operator.equals(dimensions.get("operator")),
+                name)
+                .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + operator));
+    }
+
     private MetricsReporter createReporter(Controller controller) {
         return new MetricsReporter(controller, metrics);
     }