Merge pull request #22684 from vespa-engine/ogronnesby/persist-scaling-events

Persist scaling events

8 files changed, 66 insertions, 27 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index f7368aab143..8c52ae0560a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer));
         maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher));
         maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer()));
-        maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService()));
+        maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase()));
         maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger()));
         maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector));
         maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index d4905f7e20a..205fb7e0e79 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -10,13 +10,18 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -37,6 +42,7 @@ import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.stream.Collector;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Creates a {@link ResourceSnapshot} per application, which is then passed on to a MeteringClient
@@ -57,7 +63,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
 
     private final ApplicationController applications;
     private final NodeRepository nodeRepository;
-    private final MeteringClient meteringClient;
+    private final ResourceDatabaseClient resourceClient;
     private final CuratorDb curator;
     private final SystemName systemName;
     private final Metric metric;
@@ -71,11 +77,11 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
     public ResourceMeterMaintainer(Controller controller,
                                    Duration interval,
                                    Metric metric,
-                                   MeteringClient meteringClient) {
+                                   ResourceDatabaseClient resourceClient) {
         super(controller, interval);
         this.applications = controller.applications();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-        this.meteringClient = meteringClient;
+        this.resourceClient = resourceClient;
         this.curator = controller.curator();
         this.systemName = controller.serviceRegistry().zoneRegistry().system();
         this.metric = metric;
@@ -94,6 +100,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
         }
 
         if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots);
+        if (systemName.isPublic()) reportAllScalingEvents();
         updateDeploymentCost(resourceSnapshots);
         return 1.0;
     }
@@ -124,13 +131,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
     }
 
     private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) {
-        meteringClient.consume(resourceSnapshots);
+        resourceClient.writeResourceSnapshots(resourceSnapshots);
 
         updateMeteringMetrics(resourceSnapshots);
 
         try (var lock = curator.lockMeteringRefreshTime()) {
             if (needsRefresh(curator.readMeteringRefreshTime())) {
-                meteringClient.refresh();
+                resourceClient.refreshMaterializedView();
                 curator.writeMeteringRefreshTime(clock.millis());
             }
         } catch (TimeoutException ignored) {
@@ -147,6 +154,37 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
                 .collect(Collectors.toList());
     }
 
+    private Stream<Instance> mapApplicationToInstances(Application application) {
+        return application.instances().values().stream();
+    }
+
+    private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) {
+        return instance.deployments().keySet().stream().map(zoneId -> {
+            return new DeploymentId(instance.id(), zoneId);
+        });
+    }
+
+    private Stream<Map.Entry<ClusterId, List<Cluster.ScalingEvent>>> mapDeploymentToClusterScalingEvent(DeploymentId deploymentId) {
+        return nodeRepository.getApplication(deploymentId.zoneId(), deploymentId.applicationId())
+                .clusters().entrySet().stream()
+                .map(cluster -> Map.entry(new ClusterId(deploymentId, cluster.getKey()), cluster.getValue().scalingEvents()));
+    }
+
+    private void reportAllScalingEvents() {
+        var clusters = controller().applications().asList().stream()
+                .flatMap(this::mapApplicationToInstances)
+                .flatMap(this::mapInstanceToDeployments)
+                .flatMap(this::mapDeploymentToClusterScalingEvent)
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        Map.Entry::getValue
+                ));
+
+        for (var cluster : clusters.entrySet()) {
+            resourceClient.writeScalingEvents(cluster.getKey(), cluster.getValue());
+        }
+    }
+
     private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) {
         return nodes.stream()
                 .filter(this::unlessNodeOwnerIsSystemApplication)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index 3ca6335dcb7..25ac90ac0ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -84,7 +84,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler {
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(controller.jobControl());
         if (path.matches("/controller/v1/stats")) return new StatsResponse(controller);
         if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader());
-        if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().meteringService(), path.get("tenant"), path.get("month"));
+        if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().resourceDatabase(), path.get("tenant"), path.get("month"));
         return notFound(path);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
index 33cd4948a7e..17461aafd02 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
@@ -5,7 +5,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 
 import java.time.YearMonth;
@@ -16,14 +16,14 @@ import java.util.List;
  */
 public class MeteringResponse extends SlimeJsonResponse {
 
-    public MeteringResponse(MeteringClient meteringClient, String tenantName, String month) {
-        super(toSlime(meteringClient, tenantName, month));
+    public MeteringResponse(ResourceDatabaseClient resourceClient, String tenantName, String month) {
+        super(toSlime(resourceClient, tenantName, month));
     }
 
-    private static Slime toSlime(MeteringClient meteringClient, String tenantName, String month) {
+    private static Slime toSlime(ResourceDatabaseClient resourceClient, String tenantName, String month) {
         Slime slime = new Slime();
         Cursor root = slime.setArray();
-        List<ResourceSnapshot> snapshots = meteringClient.getSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month));
+        List<ResourceSnapshot> snapshots = resourceClient.getRawSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month));
         snapshots.forEach(snapshot -> {
             Cursor object = root.addObject();
             object.setString("applicationId", snapshot.getApplicationId().toShortString());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 8e0d70bdb80..af542521b31 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIss
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
-import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer;
@@ -69,7 +68,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockMailer mockMailer = new MockMailer();
     private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock);
     private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock();
-    private final MockMeteringClient mockMeteringClient = new MockMeteringClient();
     private final MockContactRetriever mockContactRetriever = new MockContactRetriever();
     private final MockIssueHandler mockIssueHandler = new MockIssueHandler();
     private final DummyOwnershipIssues dummyOwnershipIssues = new DummyOwnershipIssues();
@@ -148,11 +146,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public MockMeteringClient meteringService() {
-        return mockMeteringClient;
-    }
-
-    @Override
     public MockContactRetriever contactRetriever() {
         return mockContactRetriever;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
index df30b6b57ee..187b8f932cf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
@@ -83,6 +83,13 @@ public class CloudTrialExpirerTest {
         assertTrue(tester.controller().tenants().get("with-apps").isEmpty());
     }
 
+    @Test
+    public void keep_tenants_without_applications_that_are_idle() {
+        registerTenant("active", "none", Duration.ofDays(364));
+        expirer.maintain();
+        assertPlan("active", "none");
+    }
+
     private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) {
         var name = TenantName.from(tenantName);
         tester.createTenant(tenantName, Tenant.Type.cloud);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
index f9441f76a38..320938f00e4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -9,9 +9,10 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
-import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -38,10 +39,10 @@ import static org.junit.Assert.assertTrue;
 public class ResourceMeterMaintainerTest {
 
     private final ControllerTester tester = new ControllerTester(SystemName.Public);
-    private final MockMeteringClient snapshotConsumer = new MockMeteringClient();
+    private final ResourceDatabaseClientMock resourceClient = new ResourceDatabaseClientMock(new PlanRegistryMock());
     private final MetricsMock metrics = new MetricsMock();
     private final ResourceMeterMaintainer maintainer =
-            new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer);
+            new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, resourceClient);
 
     @Test
     public void updates_deployment_costs() {
@@ -89,7 +90,7 @@ public class ResourceMeterMaintainerTest {
         long lastRefreshTime = tester.clock().millis();
         tester.curator().writeMeteringRefreshTime(lastRefreshTime);
         maintainer.maintain();
-        Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources();
+        Collection<ResourceSnapshot> consumedResources = resourceClient.resourceSnapshots();
 
         // The mocked repository contains two applications, so we should also consume two ResourceSnapshots
         assertEquals(4, consumedResources.size());
@@ -110,13 +111,13 @@ public class ResourceMeterMaintainerTest {
         assertEquals(40d, (Double) metrics.getMetric(context -> "tenant2".equals(context.get("tenant")), "metering.vcpu").get(), Double.MIN_VALUE);
 
         // Metering is not refreshed
-        assertFalse(snapshotConsumer.isRefreshed());
+        assertFalse(resourceClient.hasRefreshedMaterializedView());
         assertEquals(lastRefreshTime, tester.curator().readMeteringRefreshTime());
 
         var millisAdvanced = 3600 * 1000;
         tester.clock().advance(Duration.ofMillis(millisAdvanced));
         maintainer.maintain();
-        assertTrue(snapshotConsumer.isRefreshed());
+        assertTrue(resourceClient.hasRefreshedMaterializedView());
         assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 94ca4268000..5936c135af9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -161,7 +161,7 @@ public class ControllerApiTest extends ControllerContainerTest {
                 new ResourceSnapshot(applicationId, 12,48,1200, NodeResources.Architecture.arm64, timestamp, zoneId),
                 new ResourceSnapshot(applicationId, 24, 96,2400,  NodeResources.Architecture.x86_64, timestamp, zoneId)
         );
-        tester.controller().serviceRegistry().meteringService().consume(snapshots);
+        tester.controller().serviceRegistry().resourceDatabase().writeResourceSnapshots(snapshots);
         tester.assertResponse(
                 operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET),
                 new File("metering.json")