Monitor metering freshness

4 files changed, 140 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index f7368aab143..1bb594e99ed 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -75,6 +75,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer));
         maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer()));
         maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer));
+        maintainers.add(new MeteringMonitorMaintainer(controller, intervals.meteringMonitorMaintainer, controller.serviceRegistry().resourceDatabase(), metric));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -131,6 +132,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration retriggerMaintainer;
         private final Duration userManagementMaintainer;
         private final Duration billingDatabaseMaintainer;
+        private final Duration meteringMonitorMaintainer;
 
         public Intervals(SystemName system) {
             this.system = Objects.requireNonNull(system);
@@ -164,6 +166,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.retriggerMaintainer = duration(1, MINUTES);
             this.userManagementMaintainer = duration(12, HOURS);
             this.billingDatabaseMaintainer = duration(5, MINUTES);
+            this.meteringMonitorMaintainer = duration(30, MINUTES);
         }
 
         private Duration duration(long amount, TemporalUnit unit) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java
new file mode 100644
index 00000000000..c4d205cde9f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java
@@ -0,0 +1,77 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Reports discrepancies between currently deployed applications and
+ * recently stored metering data in ResourceDatabaseClient.
+ *
+ * @author olaa
+ */
+public class MeteringMonitorMaintainer extends ControllerMaintainer {
+
+    private final ResourceDatabaseClient resourceDatabaseClient;
+    private final Metric metric;
+
+    protected static final String STALE_METERING_METRIC_NAME = "metering.is_stale";
+    private static final Logger logger = Logger.getLogger(MeteringMonitorMaintainer.class.getName());
+
+    public MeteringMonitorMaintainer(Controller controller, Duration interval, ResourceDatabaseClient resourceDatabaseClient, Metric metric) {
+        super(controller, interval, null, SystemName.allOf(SystemName::isPublic));
+        this.resourceDatabaseClient = resourceDatabaseClient;
+        this.metric = metric;
+    }
+
+    @Override
+    protected double maintain() {
+        var lastSnapshots = resourceDatabaseClient.getLastSnapshots();
+        var activeDeployments = activeDeployments();
+        var isStale = activeDeployments.entrySet()
+                .stream()
+                .anyMatch(entry -> {
+                    var applicationId = entry.getKey();
+                    var expectedZones = entry.getValue();
+                    var actualZones = lastSnapshots.getOrDefault(applicationId, Set.of());
+                    if (expectedZones.equals(actualZones))
+                        return false;
+                    logger.warning(
+                            String.format("Metering discrepancy detected for application %s\n" +
+                                    "Active deployments: %s\n" +
+                                    "Last snapshots: %s\n" +
+                                    "This message can be ignored if last snapshots contains a recently deleted deployment",
+                                    applicationId.toFullString(), expectedZones, actualZones)
+                    );
+                    return true;
+                });
+
+        metric.set(STALE_METERING_METRIC_NAME, isStale ? 1 : 0, metric.createContext(Collections.emptyMap()));
+        return 1;
+    }
+
+
+    private Map<ApplicationId, Set<ZoneId>> activeDeployments() {
+        return controller().applications().asList()
+                .stream()
+                .flatMap(app -> app.instances().values().stream())
+                .filter(instance -> instance.deployments().size() > 0)
+                .collect(Collectors.toMap(
+                        Instance::id,
+                        instance -> instance.deployments().keySet()
+                ));
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java
new file mode 100644
index 00000000000..900bccb02c2
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java
@@ -0,0 +1,57 @@
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author olaa
+ */
+public class MeteringMonitorMaintainerTest {
+
+    private ControllerTester tester;
+    private DeploymentTester deploymentTester;
+    private MetricsMock metrics;
+    private ResourceDatabaseClientMock database;
+    private MeteringMonitorMaintainer maintainer;
+    private final ApplicationId applicationId = ApplicationId.from("foo", "bar", "default");
+    private final ZoneId zone = ZoneId.from("prod.aws-us-east-1c");
+
+    @Before
+    public void setup() {
+        tester = new ControllerTester(SystemName.Public);
+        deploymentTester = new DeploymentTester(tester);
+        metrics = new MetricsMock();
+        database = new ResourceDatabaseClientMock(new PlanRegistryMock());
+        maintainer = new MeteringMonitorMaintainer(tester.controller(), Duration.ofMinutes(5), database, metrics);
+    }
+    @Test
+    public void finds_stale_data() {
+        deploymentTester.newDeploymentContext(applicationId).submit().deploy();
+        maintainer.maintain();
+        assertEquals(1, metrics.getMetric(MeteringMonitorMaintainer.STALE_METERING_METRIC_NAME));
+    }
+
+    @Test
+    public void fresh_metering_data() {
+        deploymentTester.newDeploymentContext(applicationId).submit().deploy();
+        database.setLastSnapshots(Map.of(
+                applicationId, Set.of(zone)
+        ));
+        maintainer.maintain();
+        assertEquals(0, metrics.getMetric(MeteringMonitorMaintainer.STALE_METERING_METRIC_NAME));
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index be28d88abaa..6226e94f7b4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -55,6 +55,9 @@
       "name": "JobRunner"
     },
     {
+      "name": "MeteringMonitorMaintainer"
+    },
+    {
       "name": "MetricsReporter"
     },
     {