Periodically update deployment cost

6 files changed, 127 insertions, 20 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
index 515b156af48..72d3cf7723b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
@@ -102,6 +102,15 @@ public class Instance {
         return with(deployment.withMetrics(deploymentMetrics));
     }
 
+    public Instance withDeploymentCosts(Map<ZoneId, Double> costByZone) {
+        Map<ZoneId, Deployment> deployments = this.deployments.entrySet().stream()
+                .map(entry -> Optional.ofNullable(costByZone.get(entry.getKey()))
+                        .map(entry.getValue()::withCost)
+                        .orElseGet(entry.getValue()::withoutCost))
+                .collect(Collectors.toUnmodifiableMap(Deployment::zone, deployment -> deployment));
+        return with(deployments);
+    }
+
     public Instance withoutDeploymentIn(ZoneId zone) {
         Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments);
         deployments.remove(zone);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index 53305c54bc4..43ce466e8e6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -78,10 +78,12 @@ public class Deployment {
     }
 
     public Deployment withCost(double cost) {
+        if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this;
         return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost));
     }
 
     public Deployment withoutCost() {
+        if (cost.isEmpty()) return this;
         return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty());
     }
 
@@ -96,12 +98,13 @@ public class Deployment {
                deployTime.equals(that.deployTime) &&
                metrics.equals(that.metrics) &&
                activity.equals(that.activity) &&
-               quota.equals(that.quota);
+               quota.equals(that.quota) &&
+               cost.equals(that.cost);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota);
+        return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index d9cd7941a87..aed2e637e4b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -1,17 +1,23 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.CloudName;
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.yolean.Exceptions;
 
@@ -22,6 +28,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
@@ -44,9 +51,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
             Node.State.inactive    // an application is not using it, but it is reserved for being re-introduced or decommissioned
     );
 
+    private final ApplicationController applications;
     private final NodeRepository nodeRepository;
     private final MeteringClient meteringClient;
     private final CuratorDb curator;
+    private final SystemName systemName;
+    private final Metric metric;
+    private final Clock clock;
 
     private static final String METERING_LAST_REPORTED = "metering_last_reported";
     private static final String METERING_TOTAL_REPORTED = "metering_total_reported";
@@ -57,28 +68,57 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
                                    Duration interval,
                                    Metric metric,
                                    MeteringClient meteringClient) {
-        super(controller, interval, null, SystemName.allOf(SystemName::isPublic));
-        this.clock = controller.clock();
+        super(controller, interval);
+        this.applications = controller.applications();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-        this.metric = metric;
         this.meteringClient = meteringClient;
         this.curator = controller.curator();
+        this.systemName = controller.serviceRegistry().zoneRegistry().system();
+        this.metric = metric;
+        this.clock = controller.clock();
     }
 
     @Override
     protected boolean maintain() {
+        Collection<ResourceSnapshot> resourceSnapshots;
         try {
-            collectResourceSnapshots();
-            return true;
+            resourceSnapshots = getAllResourceSnapshots();
         } catch (Exception e) {
             log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " +
                                    Exceptions.toMessageString(e));
+            return false;
+        }
+
+        if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots);
+        updateDeploymentCost(resourceSnapshots);
+        return true;
+    }
+
+    void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) {
+        resourceSnapshots.stream()
+                .collect(Collectors.groupingBy(snapshot -> TenantAndApplicationId.from(snapshot.getApplicationId()),
+                         Collectors.groupingBy(snapshot -> snapshot.getApplicationId().instance())))
+                .forEach(this::updateDeploymentCost);
+    }
+
+    private void updateDeploymentCost(TenantAndApplicationId tenantAndApplication, Map<InstanceName, List<ResourceSnapshot>> snapshotsByInstance) {
+        try {
+            applications.lockApplicationIfPresent(tenantAndApplication, locked -> {
+                for (InstanceName instanceName : locked.get().instances().keySet()) {
+                    Map<ZoneId, Double> deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream()
+                            .collect(Collectors.toUnmodifiableMap(
+                                    ResourceSnapshot::getZoneId,
+                                    snapshot -> cost(snapshot.allocation(), systemName)));
+                    locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts));
+                }
+                applications.store(locked);
+            });
+        } catch (UncheckedTimeoutException ignored) {
+            // Will be retried on next maintenance, avoid throwing so we can update the other apps instead
         }
-        return false;
     }
 
-    private void collectResourceSnapshots() {
-        Collection<ResourceSnapshot> resourceSnapshots = getAllResourceSnapshots();
+    private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) {
         meteringClient.consume(resourceSnapshots);
 
         metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap()));
@@ -111,7 +151,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
         return nodes.stream()
                 .filter(this::unlessNodeOwnerIsSystemApplication)
                 .filter(this::isNodeStateMeterable)
-                .filter(this::isNodeTypeMeterable)
+                .filter(this::isClusterTypeMeterable)
                 .collect(Collectors.groupingBy(node ->
                                 node.owner().get(),
                                 Collectors.collectingAndThen(Collectors.toList(),
@@ -142,4 +182,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
                 .isAfter(Instant.ofEpochMilli(lastRefreshTimestamp));
     }
 
+    public static double cost(ClusterResources clusterResources, SystemName systemName) {
+        NodeResources nr = clusterResources.nodeResources();
+        return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb()).multiply(clusterResources.nodes()), systemName);
+    }
+
+    private static double cost(ResourceAllocation allocation, SystemName systemName) {
+        // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost
+        double costDivisor = systemName.isPublic() ? 1.0 : 3.0;
+        double cost = new NodeResources(allocation.getCpuCores(), allocation.getMemoryGb(), allocation.getDiskGb(), 0).cost();
+        return Math.round(cost * 100.0 / costDivisor) / 100.0;
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index d44bccc1034..b2ad372ce71 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -88,6 +88,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToC
 import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer;
+import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer;
 import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer;
@@ -2115,9 +2116,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         object.setLong("groups", resources.groups());
         toSlime(resources.nodeResources(), object.setObject("nodeResources"));
 
-        // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost
-        double costDivisor = controller.zoneRegistry().system().isPublic() ? 1.0 : 3.0;
-        object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / costDivisor) / 100.0);
+        double cost = ResourceMeterMaintainer.cost(resources, controller.serviceRegistry().zoneRegistry().system());
+        object.setDouble("cost", cost);
     }
 
     private void utilizationToSlime(Cluster.Utilization utilization, Cursor utilizationObject) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index bc81924225c..ca3909af0ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -53,7 +53,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
         this.zones = system.isPublic() ?
                 List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
                         ZoneApiMock.fromId("staging.aws-us-east-1c"),
-                        ZoneApiMock.fromId("prod.aws-us-east-1c")) :
+                        ZoneApiMock.fromId("prod.aws-us-east-1c"),
+                        ZoneApiMock.fromId("prod.aws-eu-west-1a")) :
                 List.of(ZoneApiMock.fromId("test.us-east-1"),
                         ZoneApiMock.fromId("staging.us-east-3"),
                         ZoneApiMock.fromId("dev.us-east-1"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
index f7631d019cb..e61516cbb1a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -6,17 +6,25 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -27,17 +35,52 @@ import static org.junit.Assert.*;
  */
 public class ResourceMeterMaintainerTest {
 
-    private final ControllerTester tester = new ControllerTester();
+    private final ControllerTester tester = new ControllerTester(SystemName.Public);
     private final MockMeteringClient snapshotConsumer = new MockMeteringClient();
     private final MetricsMock metrics = new MetricsMock();
+    private final ResourceMeterMaintainer maintainer =
+            new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer);
+
+    @Test
+    public void updates_deployment_costs() {
+        ApplicationId app1 = ApplicationId.from("t1", "a1", "default");
+        ApplicationId app2 = ApplicationId.from("t2", "a1", "default");
+        ZoneId z1 = ZoneId.from("prod.aws-us-east-1c");
+        ZoneId z2 = ZoneId.from("prod.aws-eu-west-1a");
+
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(z1.region()).region(z2.region()).trustDefaultCertificate().build();
+        List.of(app1, app2).forEach(app -> deploymentTester.newDeploymentContext(app).submit(applicationPackage).deploy());
+
+        BiConsumer<ApplicationId, Map<ZoneId, Double>> assertCost = (appId, costs) ->
+                assertEquals(costs, tester.controller().applications().getInstance(appId).get().deployments().entrySet().stream()
+                        .filter(entry -> entry.getValue().cost().isPresent())
+                        .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble())));
+
+        List<ResourceSnapshot> resourceSnapshots = List.of(
+                new ResourceSnapshot(app1, 12, 34, 56, Instant.EPOCH, z1),
+                new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2),
+                new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1));
+        maintainer.updateDeploymentCost(resourceSnapshots);
+        assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50));
+        assertCost.accept(app2, Map.of(z1, 3.59));
+
+        // Remove a region from app1 and add region to app2
+        resourceSnapshots = List.of(
+                new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2),
+                new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1),
+                new ResourceSnapshot(app2, 45, 67, 89, Instant.EPOCH, z2));
+        maintainer.updateDeploymentCost(resourceSnapshots);
+        assertCost.accept(app1, Map.of(z2, 2.50));
+        assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68));
+    }
 
     @Test
     public void testMaintainer() {
         setUpZones();
-        ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer);
         long lastRefreshTime = tester.clock().millis();
         tester.curator().writeMeteringRefreshTime(lastRefreshTime);
-        resourceMeterMaintainer.maintain();
+        maintainer.maintain();
         Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources();
 
         // The mocked repository contains two applications, so we should also consume two ResourceSnapshots
@@ -62,7 +105,7 @@ public class ResourceMeterMaintainerTest {
 
         var millisAdvanced = 3600 * 1000;
         tester.clock().advance(Duration.ofMillis(millisAdvanced));
-        resourceMeterMaintainer.maintain();
+        maintainer.maintain();
         assertTrue(snapshotConsumer.isRefreshed());
         assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime());
     }