Merge pull request #22194 from vespa-engine/olaa/zms-quota-monitoring

Monitor ZMS quota

8 files changed, 167 insertions, 4 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
index 5f567e8b84a..0415b33b29d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
@@ -11,6 +11,7 @@ import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.AthenzRoleInformation;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.api.OAuthCredentials;
+import com.yahoo.vespa.athenz.client.zms.QuotaUsage;
 import com.yahoo.vespa.athenz.client.zms.RoleAction;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
 import com.yahoo.vespa.athenz.client.zms.ZmsClientException;
@@ -262,6 +263,11 @@ public class ZmsClientMock implements ZmsClient {
     }
 
     @Override
+    public QuotaUsage getQuotaUsage() {
+        return new QuotaUsage(0.1, 0.2, 0.3, 0.4, 0.5);
+    }
+
+    @Override
     public void close() {}
 
     private static AthenzDomain getTenantDomain(AthenzResourceName resource) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index deafcd35e9b..041d0694ca9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
 import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement;
 
 import java.time.Duration;
@@ -36,7 +37,7 @@ public class ControllerMaintenance extends AbstractComponent {
 
     @Inject
     @SuppressWarnings("unused") // instantiated by Dependency Injection
-    public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) {
+    public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) {
         Intervals intervals = new Intervals(controller.system());
         upgrader = new Upgrader(controller, intervals.defaultInterval);
         maintainers.add(upgrader);
@@ -44,7 +45,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval));
         maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval));
         maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval));
-        maintainers.add(new MetricsReporter(controller, metric));
+        maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient()));
         maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer));
         maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater));
         maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 74bbf906653..294d5bad42d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.athenz.client.zms.ZmsClient;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -62,17 +63,20 @@ public class MetricsReporter extends ControllerMaintainer {
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
     public static final String OPERATION_PREFIX = "operation.";
+    public static final String ZMS_QUOTA_USAGE = "zms.quota.usage";
 
     private final Metric metric;
     private final Clock clock;
+    private final ZmsClient zmsClient;
 
     // Keep track of reported node counts for each version
     private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>();
 
-    public MetricsReporter(Controller controller, Metric metric) {
+    public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) {
         super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics
         this.metric = metric;
         this.clock = controller.clock();
+        this.zmsClient = zmsClient;
     }
 
     @Override
@@ -85,6 +89,7 @@ public class MetricsReporter extends ControllerMaintainer {
         reportAuditLog();
         reportBrokenSystemVersion(versionStatus);
         reportTenantMetrics();
+        reportZmsQuotaMetrics();
         return 1.0;
     }
 
@@ -252,6 +257,20 @@ public class MetricsReporter extends ControllerMaintainer {
         });
     }
 
+    private void reportZmsQuotaMetrics() {
+        var quota = zmsClient.getQuotaUsage();
+        reportZmsQuota("subdomains", quota.getSubdomainUsage());
+        reportZmsQuota("services", quota.getServiceUsage());
+        reportZmsQuota("policies", quota.getPolicyUsage());
+        reportZmsQuota("roles", quota.getRoleUsage());
+        reportZmsQuota("groups", quota.getGroupUsage());
+    }
+
+    private void reportZmsQuota(String resourceType, double usage) {
+        var context = metric.createContext(Map.of("resourceType", resourceType));
+        metric.set(ZMS_QUOTA_USAGE, usage, context);
+    }
+
     private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) {
         return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index bfc6345083c..64e3a95605a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -10,8 +10,11 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClientMock;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter;
@@ -50,6 +53,7 @@ import static org.junit.Assert.assertTrue;
 public class MetricsReporterTest {
 
     private final MetricsMock metrics = new MetricsMock();
+    private final ZmsClientMock zmsClient = new ZmsClientMock(new AthenzDbMock(), AthenzIdentities.from("mock.identity"));
 
     @Test
     public void audit_log_metric() {
@@ -553,6 +557,19 @@ public class MetricsReporterTest {
         assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get());
     }
 
+    @Test
+    public void zms_quota_metrics() {
+        var tester = new ControllerTester();
+        var reporter = createReporter(tester.controller());
+        reporter.maintain();
+
+        assertEquals(0.1, metrics.getMetric(d -> "subdomains".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get());
+        assertEquals(0.2, metrics.getMetric(d -> "roles".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get());
+        assertEquals(0.3, metrics.getMetric(d -> "policies".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get());
+        assertEquals(0.4, metrics.getMetric(d -> "services".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get());
+        assertEquals(0.5, metrics.getMetric(d -> "groups".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get());
+    }
+
     private void assertNodeCount(String metric, int n, Version version) {
         long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric)
                                 .stream()
@@ -656,7 +673,7 @@ public class MetricsReporterTest {
     }
 
     private MetricsReporter createReporter(Controller controller) {
-        return new MetricsReporter(controller, metrics);
+        return new MetricsReporter(controller, metrics, zmsClient);
     }
 
     private static String appDimension(ApplicationId id) {
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
index 136ae1df8ae..8ffb9331ddb 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.athenz.client.zms.bindings.ResponseListEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.ServiceEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.ServiceListResponseEntity;
+import com.yahoo.vespa.athenz.client.zms.bindings.StatisticsEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.athenz.utils.AthenzIdentities;
@@ -408,6 +409,17 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient {
         execute(request, response -> readEntity(response, Void.class));
     }
 
+    @Override
+    public QuotaUsage getQuotaUsage() {
+        var uri = zmsUrl.resolve(String.format("domain/%s/quota", identity.getDomainName()));
+        var quotaEntity = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, StatisticsEntity.class));
+
+        uri = zmsUrl.resolve(String.format("domain/%s/stats", identity.getDomainName()));
+        var usageEntity = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, StatisticsEntity.class));
+
+        return QuotaUsage.calculateUsage(usageEntity, quotaEntity);
+    }
+
     public AthenzRoleInformation getFullRoleInformation(AthenzRole role) {
         var uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true&auditLog=true", role.domain().getName(), role.roleName()));
         var request = RequestBuilder.get(uri).build();
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java
new file mode 100644
index 00000000000..8e9c7de9272
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java
@@ -0,0 +1,55 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zms;
+
+import com.yahoo.vespa.athenz.client.zms.bindings.StatisticsEntity;
+
+/**
+ * @author olaa
+ */
+public class QuotaUsage {
+
+    private double subdomainUsage;
+    private double roleUsage;
+    private double policyUsage;
+    private double serviceUsage;
+    private double groupUsage;
+
+
+    public QuotaUsage(double subdomainUsage, double roleUsage, double policyUsage, double serviceUsage, double groupUsage) {
+        this.subdomainUsage = subdomainUsage;
+        this.roleUsage = roleUsage;
+        this.policyUsage = policyUsage;
+        this.serviceUsage = serviceUsage;
+        this.groupUsage = groupUsage;
+    }
+
+    public static QuotaUsage calculateUsage(StatisticsEntity used, StatisticsEntity quota) {
+        return new QuotaUsage(
+                (double) used.getSubdomains() / quota.getSubdomains(),
+                (double) used.getRoles() / quota.getRoles(),
+                (double) used.getPolicies() / quota.getPolicies(),
+                (double) used.getServices() / quota.getServices(),
+                (double) used.getGroups() / quota.getGroups()
+        );
+    }
+
+    public double getSubdomainUsage() {
+        return subdomainUsage;
+    }
+
+    public double getRoleUsage() {
+        return roleUsage;
+    }
+
+    public double getPolicyUsage() {
+        return policyUsage;
+    }
+
+    public double getServiceUsage() {
+        return serviceUsage;
+    }
+
+    public double getGroupUsage() {
+        return groupUsage;
+    }
+}
\ No newline at end of file
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
index 3ff2ff843a0..b07f6da1a01 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
@@ -83,5 +83,7 @@ public interface ZmsClient extends AutoCloseable {
 
     AthenzRoleInformation getFullRoleInformation(AthenzRole role);
 
+    QuotaUsage getQuotaUsage();
+
     void close();
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java
new file mode 100644
index 00000000000..bba6195363c
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zms.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author olaa
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StatisticsEntity {
+
+    private int subdomains;
+    private int roles;
+    private int policies;
+    private int services;
+    private int groups;
+
+    public StatisticsEntity(@JsonProperty("subdomain") int subdomains,
+                            @JsonProperty("role") int roles,
+                            @JsonProperty("policy") int policies,
+                            @JsonProperty("service") int services,
+                            @JsonProperty("group") int groups) {
+        this.subdomains = subdomains;
+        this.roles = roles;
+        this.policies = policies;
+        this.services = services;
+        this.groups = groups;
+    }
+
+    public int getSubdomains() {
+        return subdomains;
+    }
+
+    public int getRoles() {
+        return roles;
+    }
+
+    public int getPolicies() {
+        return policies;
+    }
+
+    public int getServices() {
+        return services;
+    }
+
+    public int getGroups() {
+        return groups;
+    }
+
+}
\ No newline at end of file