From ff2090df7420e2b6f412d3ec593c9fddbbece3ee Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 31 May 2021 15:52:26 +0200 Subject: Periodically update deployment cost --- .../yahoo/vespa/hosted/controller/Instance.java | 9 +++ .../hosted/controller/application/Deployment.java | 7 ++- .../maintenance/ResourceMeterMaintainer.java | 71 +++++++++++++++++++--- .../restapi/application/ApplicationApiHandler.java | 6 +- .../controller/integration/ZoneRegistryMock.java | 3 +- .../maintenance/ResourceMeterMaintainerTest.java | 51 ++++++++++++++-- 6 files changed, 127 insertions(+), 20 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 515b156af48..72d3cf7723b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -102,6 +102,15 @@ public class Instance { return with(deployment.withMetrics(deploymentMetrics)); } + public Instance withDeploymentCosts(Map costByZone) { + Map deployments = this.deployments.entrySet().stream() + .map(entry -> Optional.ofNullable(costByZone.get(entry.getKey())) + .map(entry.getValue()::withCost) + .orElseGet(entry.getValue()::withoutCost)) + .collect(Collectors.toUnmodifiableMap(Deployment::zone, deployment -> deployment)); + return with(deployments); + } + public Instance withoutDeploymentIn(ZoneId zone) { Map deployments = new LinkedHashMap<>(this.deployments); deployments.remove(zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 53305c54bc4..43ce466e8e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -78,10 +78,12 @@ public class Deployment { } public Deployment withCost(double cost) { + if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); } public Deployment withoutCost() { + if (cost.isEmpty()) return this; return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @@ -96,12 +98,13 @@ public class Deployment { deployTime.equals(that.deployTime) && metrics.equals(that.metrics) && activity.equals(that.activity) && - quota.equals(that.quota); + quota.equals(that.quota) && + cost.equals(that.cost); } @Override public int hashCode() { - return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index d9cd7941a87..aed2e637e4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,17 +1,23 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.CloudName; +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.Exceptions; @@ -22,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.logging.Level; @@ -44,9 +51,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned ); + private final ApplicationController applications; private final NodeRepository nodeRepository; private final MeteringClient meteringClient; private final CuratorDb curator; + private final SystemName systemName; + private final Metric metric; + private final Clock clock; private static final String METERING_LAST_REPORTED = "metering_last_reported"; private static final String METERING_TOTAL_REPORTED = "metering_total_reported"; @@ -57,28 +68,57 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { Duration interval, Metric metric, MeteringClient meteringClient) { - super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); - this.clock = controller.clock(); + super(controller, interval); + this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.metric = metric; this.meteringClient = meteringClient; this.curator = controller.curator(); + this.systemName = controller.serviceRegistry().zoneRegistry().system(); + this.metric = metric; + this.clock = controller.clock(); } @Override protected boolean maintain() { + Collection resourceSnapshots; try { - collectResourceSnapshots(); - return true; + resourceSnapshots = getAllResourceSnapshots(); } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); + return false; + } + + if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); + updateDeploymentCost(resourceSnapshots); + return true; + } + + void updateDeploymentCost(Collection resourceSnapshots) { + resourceSnapshots.stream() + .collect(Collectors.groupingBy(snapshot -> TenantAndApplicationId.from(snapshot.getApplicationId()), + Collectors.groupingBy(snapshot -> snapshot.getApplicationId().instance()))) + .forEach(this::updateDeploymentCost); + } + + private void updateDeploymentCost(TenantAndApplicationId tenantAndApplication, Map> snapshotsByInstance) { + try { + applications.lockApplicationIfPresent(tenantAndApplication, locked -> { + for (InstanceName instanceName : locked.get().instances().keySet()) { + Map deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream() + .collect(Collectors.toUnmodifiableMap( + ResourceSnapshot::getZoneId, + snapshot -> cost(snapshot.allocation(), systemName))); + locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts)); + } + applications.store(locked); + }); + } catch (UncheckedTimeoutException ignored) { + // Will be retried on next maintenance, avoid throwing so we can update the other apps instead } - return false; } - private void collectResourceSnapshots() { - Collection resourceSnapshots = getAllResourceSnapshots(); + private void reportResourceSnapshots(Collection resourceSnapshots) { meteringClient.consume(resourceSnapshots); metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); @@ -111,7 +151,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { return nodes.stream() .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) - .filter(this::isNodeTypeMeterable) + .filter(this::isClusterTypeMeterable) .collect(Collectors.groupingBy(node -> node.owner().get(), Collectors.collectingAndThen(Collectors.toList(), @@ -142,4 +182,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .isAfter(Instant.ofEpochMilli(lastRefreshTimestamp)); } + public static double cost(ClusterResources clusterResources, SystemName systemName) { + NodeResources nr = clusterResources.nodeResources(); + return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb()).multiply(clusterResources.nodes()), systemName); + } + + private static double cost(ResourceAllocation allocation, SystemName systemName) { + // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost + double costDivisor = systemName.isPublic() ? 1.0 : 3.0; + double cost = new NodeResources(allocation.getCpuCores(), allocation.getMemoryGb(), allocation.getDiskGb(), 0).cost(); + return Math.round(cost * 100.0 / costDivisor) / 100.0; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d44bccc1034..b2ad372ce71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -88,6 +88,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToC import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; +import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; @@ -2115,9 +2116,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setLong("groups", resources.groups()); toSlime(resources.nodeResources(), object.setObject("nodeResources")); - // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost - double costDivisor = controller.zoneRegistry().system().isPublic() ? 1.0 : 3.0; - object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / costDivisor) / 100.0); + double cost = ResourceMeterMaintainer.cost(resources, controller.serviceRegistry().zoneRegistry().system()); + object.setDouble("cost", cost); } private void utilizationToSlime(Cluster.Utilization utilization, Cursor utilizationObject) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index bc81924225c..ca3909af0ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -53,7 +53,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry this.zones = system.isPublic() ? List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")) : + ZoneApiMock.fromId("prod.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-eu-west-1a")) : List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), ZoneApiMock.fromId("dev.us-east-1"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index f7631d019cb..e61516cbb1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -6,17 +6,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; +import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -27,17 +35,52 @@ import static org.junit.Assert.*; */ public class ResourceMeterMaintainerTest { - private final ControllerTester tester = new ControllerTester(); + private final ControllerTester tester = new ControllerTester(SystemName.Public); private final MockMeteringClient snapshotConsumer = new MockMeteringClient(); private final MetricsMock metrics = new MetricsMock(); + private final ResourceMeterMaintainer maintainer = + new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); + + @Test + public void updates_deployment_costs() { + ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + ApplicationId app2 = ApplicationId.from("t2", "a1", "default"); + ZoneId z1 = ZoneId.from("prod.aws-us-east-1c"); + ZoneId z2 = ZoneId.from("prod.aws-eu-west-1a"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(z1.region()).region(z2.region()).trustDefaultCertificate().build(); + List.of(app1, app2).forEach(app -> deploymentTester.newDeploymentContext(app).submit(applicationPackage).deploy()); + + BiConsumer> assertCost = (appId, costs) -> + assertEquals(costs, tester.controller().applications().getInstance(appId).get().deployments().entrySet().stream() + .filter(entry -> entry.getValue().cost().isPresent()) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble()))); + + List resourceSnapshots = List.of( + new ResourceSnapshot(app1, 12, 34, 56, Instant.EPOCH, z1), + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59)); + + // Remove a region from app1 and add region to app2 + resourceSnapshots = List.of( + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1), + new ResourceSnapshot(app2, 45, 67, 89, Instant.EPOCH, z2)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68)); + } @Test public void testMaintainer() { setUpZones(); - ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); long lastRefreshTime = tester.clock().millis(); tester.curator().writeMeteringRefreshTime(lastRefreshTime); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); Collection consumedResources = snapshotConsumer.consumedResources(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots @@ -62,7 +105,7 @@ public class ResourceMeterMaintainerTest { var millisAdvanced = 3600 * 1000; tester.clock().advance(Duration.ofMillis(millisAdvanced)); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); assertTrue(snapshotConsumer.isRefreshed()); assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime()); } -- cgit v1.2.3