diff options
20 files changed, 212 insertions, 104 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index 5c6effced93..a8d9c4b1f8a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -53,6 +53,10 @@ public class ResourceSnapshot { return applicationId; } + public ResourceAllocation allocation() { + return resourceAllocation; + } + public double getCpuCores() { return resourceAllocation.getCpuCores(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 0fcb3cd9be4..72d3cf7723b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.Set; import java.util.function.Function; @@ -69,11 +70,13 @@ public class Instance { version, instant, DeploymentMetrics.none, DeploymentActivity.none, - QuotaUsage.none)); + QuotaUsage.none, + OptionalDouble.empty())); Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), - quotaUsage); + quotaUsage, + previousDeployment.cost()); return with(newDeployment); } @@ -99,6 +102,15 @@ public class Instance { return with(deployment.withMetrics(deploymentMetrics)); } + public Instance withDeploymentCosts(Map<ZoneId, Double> costByZone) { + Map<ZoneId, Deployment> deployments = this.deployments.entrySet().stream() + .map(entry -> Optional.ofNullable(costByZone.get(entry.getKey())) + .map(entry.getValue()::withCost) + .orElseGet(entry.getValue()::withoutCost)) + .collect(Collectors.toUnmodifiableMap(Deployment::zone, deployment -> deployment)); + return with(deployments); + } + public Instance withoutDeploymentIn(ZoneId zone) { Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments); deployments.remove(zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 3d17a7f8681..43ce466e8e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import java.time.Instant; import java.util.Objects; +import java.util.OptionalDouble; /** * A deployment of an application in a particular zone. @@ -23,9 +24,10 @@ public class Deployment { private final DeploymentMetrics metrics; private final DeploymentActivity activity; private final QuotaUsage quota; + private final OptionalDouble cost; public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, - DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota) { + DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota, OptionalDouble cost) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); @@ -33,6 +35,7 @@ public class Deployment { this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); this.quota = Objects.requireNonNull(quota, "usage cannot be null"); + this.cost = Objects.requireNonNull(cost, "cost cannot be null"); } /** Returns the zone this was deployed to */ @@ -58,17 +61,30 @@ public class Deployment { /** Returns quota usage for this */ public QuotaUsage quota() { return quota; } + /** Returns cost, in dollars per hour, for this */ + public OptionalDouble cost() { return cost; } + public Deployment recordActivityAt(Instant instant) { return new Deployment(zone, applicationVersion, version, deployTime, metrics, - activity.recordAt(instant, metrics), quota); + activity.recordAt(instant, metrics), quota, cost); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); } public Deployment withQuota(QuotaUsage quota) { - return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); + } + + public Deployment withCost(double cost) { + if (this.cost.isPresent() && Double.compare(this.cost.getAsDouble(), cost) == 0) return this; + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.of(cost)); + } + + public Deployment withoutCost() { + if (cost.isEmpty()) return this; + return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity, quota, OptionalDouble.empty()); } @Override @@ -82,12 +98,13 @@ public class Deployment { deployTime.equals(that.deployTime) && metrics.equals(that.metrics) && activity.equals(that.activity) && - quota.equals(that.quota); + quota.equals(that.quota) && + cost.equals(that.cost); } @Override public int hashCode() { - return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota); + return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota, cost); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 5b873f11618..8981ad4e6db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -246,7 +246,8 @@ public class DeploymentStatus { existing.at(), existing.metrics(), existing.activity(), - existing.quota()) + existing.quota(), + existing.cost()) : existing); if ( job.application().instance().equals(instance) && job.type().isProduction() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index b40f2232504..aed2e637e4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,17 +1,23 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.CloudName; +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.Exceptions; @@ -20,7 +26,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.logging.Level; @@ -33,11 +41,23 @@ import java.util.stream.Collectors; */ public class ResourceMeterMaintainer extends ControllerMaintainer { - private final Clock clock; - private final Metric metric; + /** + * Checks if the node is in some state where it is in active use by the tenant, + * and not transitioning out of use, in a failed state, etc. + */ + private static final Set<Node.State> METERABLE_NODE_STATES = EnumSet.of( + Node.State.reserved, // an application will soon use this node + Node.State.active, // an application is currently using this node + Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned + ); + + private final ApplicationController applications; private final NodeRepository nodeRepository; private final MeteringClient meteringClient; private final CuratorDb curator; + private final SystemName systemName; + private final Metric metric; + private final Clock clock; private static final String METERING_LAST_REPORTED = "metering_last_reported"; private static final String METERING_TOTAL_REPORTED = "metering_total_reported"; @@ -48,28 +68,57 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { Duration interval, Metric metric, MeteringClient meteringClient) { - super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); - this.clock = controller.clock(); + super(controller, interval); + this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.metric = metric; this.meteringClient = meteringClient; this.curator = controller.curator(); + this.systemName = controller.serviceRegistry().zoneRegistry().system(); + this.metric = metric; + this.clock = controller.clock(); } @Override protected boolean maintain() { + Collection<ResourceSnapshot> resourceSnapshots; try { - collectResourceSnapshots(); - return true; + resourceSnapshots = getAllResourceSnapshots(); } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); + return false; + } + + if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); + updateDeploymentCost(resourceSnapshots); + return true; + } + + void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { + resourceSnapshots.stream() + .collect(Collectors.groupingBy(snapshot -> TenantAndApplicationId.from(snapshot.getApplicationId()), + Collectors.groupingBy(snapshot -> snapshot.getApplicationId().instance()))) + .forEach(this::updateDeploymentCost); + } + + private void updateDeploymentCost(TenantAndApplicationId tenantAndApplication, Map<InstanceName, List<ResourceSnapshot>> snapshotsByInstance) { + try { + applications.lockApplicationIfPresent(tenantAndApplication, locked -> { + for (InstanceName instanceName : locked.get().instances().keySet()) { + Map<ZoneId, Double> deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream() + .collect(Collectors.toUnmodifiableMap( + ResourceSnapshot::getZoneId, + snapshot -> cost(snapshot.allocation(), systemName))); + locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts)); + } + applications.store(locked); + }); + } catch (UncheckedTimeoutException ignored) { + // Will be retried on next maintenance, avoid throwing so we can update the other apps instead } - return false; } - private void collectResourceSnapshots() { - Collection<ResourceSnapshot> resourceSnapshots = getAllResourceSnapshots(); + private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) { meteringClient.consume(resourceSnapshots); metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); @@ -89,9 +138,8 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } } - private Collection<ResourceSnapshot> getAllResourceSnapshots() { + private List<ResourceSnapshot> getAllResourceSnapshots() { return controller().zoneRegistry().zones() - .ofCloud(CloudName.from("aws")) .reachable().zones().stream() .map(ZoneApi::getId) .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId, false))) @@ -103,7 +151,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { return nodes.stream() .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) - .filter(this::isNodeTypeMeterable) + .filter(this::isClusterTypeMeterable) .collect(Collectors.groupingBy(node -> node.owner().get(), Collectors.collectingAndThen(Collectors.toList(), @@ -120,21 +168,11 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .orElse(false); } - /** - * Checks if the node is in some state where it is in active use by the tenant, - * and not transitioning out of use, in a failed state, etc. - */ - private static final Set<Node.State> METERABLE_NODE_STATES = Set.of( - Node.State.reserved, // an application will soon use this node - Node.State.active, // an application is currently using this node - Node.State.inactive // an application is not using it, but it is reserved for being re-introduced or decommissioned - ); - private boolean isNodeStateMeterable(Node node) { return METERABLE_NODE_STATES.contains(node.state()); } - private boolean isNodeTypeMeterable(Node node) { + private boolean isClusterTypeMeterable(Node node) { return node.clusterType() != Node.ClusterType.admin; // log servers and shared cluster controllers } @@ -144,4 +182,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .isAfter(Instant.ofEpochMilli(lastRefreshTimestamp)); } + public static double cost(ClusterResources clusterResources, SystemName systemName) { + NodeResources nr = clusterResources.nodeResources(); + return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb()).multiply(clusterResources.nodes()), systemName); + } + + private static double cost(ResourceAllocation allocation, SystemName systemName) { + // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost + double costDivisor = systemName.isPublic() ? 1.0 : 3.0; + double cost = new NodeResources(allocation.getCpuCores(), allocation.getMemoryGb(), allocation.getDiskGb(), 0).cost(); + return Math.round(cost * 100.0 / costDivisor) / 100.0; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 24b553e5153..06442779b9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -142,6 +142,8 @@ public class ApplicationSerializer { // Quota usage fields private static final String quotaUsageRateField = "quotaUsageRate"; + private static final String deploymentCostField = "cost"; + // ------------------ Serialization public Slime toSlime(Application application) { @@ -196,6 +198,7 @@ public class ApplicationSerializer { deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value)); deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value)); object.setDouble(quotaUsageRateField, deployment.quota().rate()); + deployment.cost().ifPresent(cost -> object.setDouble(deploymentCostField, cost)); } private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) { @@ -357,7 +360,8 @@ public class ApplicationSerializer { Serializers.optionalInstant(deploymentObject.field(lastWrittenField)), Serializers.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), Serializers.optionalDouble(deploymentObject.field(lastWritesPerSecondField))), - QuotaUsage.create(Serializers.optionalDouble(deploymentObject.field(quotaUsageRateField)))); + QuotaUsage.create(Serializers.optionalDouble(deploymentObject.field(quotaUsageRateField))), + Serializers.optionalDouble(deploymentObject.field(deploymentCostField))); } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 91d85e62fcb..60d8afe0f5e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -95,7 +95,6 @@ class RunSerializer { private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; private static final String noNodesDownSinceField = "noNodesDownSince"; - private static final String oldConvergenceSummaryField = "convergenceSummary"; // TODO (freva): Remove after 7.410 private static final String convergenceSummaryField = "convergenceSummaryV2"; private static final String testerCertificateField = "testerCertificate"; @@ -137,8 +136,7 @@ class RunSerializer { runObject.field(lastTestRecordField).asLong(), Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), Serializers.optionalInstant(runObject.field(noNodesDownSinceField)), - convergenceSummaryFrom(runObject.field(convergenceSummaryField)) - .or(() ->convergenceSummaryFrom(runObject.field(oldConvergenceSummaryField))), + convergenceSummaryFrom(runObject.field(convergenceSummaryField)), Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) .map(certificate -> X509CertificateUtils.fromPem(certificate.asString()))); @@ -223,10 +221,7 @@ class RunSerializer { runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); - run.convergenceSummary().ifPresent(convergenceSummary -> { - toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField), false); - toSlime(convergenceSummary, runObject.setArray(oldConvergenceSummaryField), true); - }); + run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); Cursor stepsObject = runObject.setObject(stepsField); @@ -263,7 +258,7 @@ class RunSerializer { } // Don't change this - introduce a separate array with new values if needed. - private void toSlime(ConvergenceSummary summary, Cursor summaryArray, boolean oldFormat) { + private void toSlime(ConvergenceSummary summary, Cursor summaryArray) { summaryArray.addLong(summary.nodes()); summaryArray.addLong(summary.down()); summaryArray.addLong(summary.upgradingOs()); @@ -276,8 +271,7 @@ class RunSerializer { summaryArray.addLong(summary.restarting()); summaryArray.addLong(summary.services()); summaryArray.addLong(summary.needNewConfig()); - if (!oldFormat) - summaryArray.addLong(summary.retiring()); + summaryArray.addLong(summary.retiring()); } static String valueOf(Step step) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d44bccc1034..b72a6d2f820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -88,6 +88,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToC import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; +import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; @@ -1403,6 +1404,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { }); } + response.setDouble("quota", deployment.quota().rate()); + deployment.cost().ifPresent(cost -> response.setDouble("cost", cost)); + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant()) .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); @@ -2115,9 +2119,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setLong("groups", resources.groups()); toSlime(resources.nodeResources(), object.setObject("nodeResources")); - // Divide cost by 3 in non-public zones to show approx. AWS equivalent cost - double costDivisor = controller.zoneRegistry().system().isPublic() ? 1.0 : 3.0; - object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / costDivisor) / 100.0); + double cost = ResourceMeterMaintainer.cost(resources, controller.serviceRegistry().zoneRegistry().system()); + object.setDouble("cost", cost); } private void utilizationToSlime(Cluster.Utilization utilization, Cursor utilizationObject) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index bc81924225c..ca3909af0ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -53,7 +53,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry this.zones = system.isPublic() ? List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")) : + ZoneApiMock.fromId("prod.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-eu-west-1a")) : List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), ZoneApiMock.fromId("dev.us-east-1"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 0deaa21d13b..e61516cbb1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -6,18 +6,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; +import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -28,17 +35,52 @@ import static org.junit.Assert.*; */ public class ResourceMeterMaintainerTest { - private final ControllerTester tester = new ControllerTester(); + private final ControllerTester tester = new ControllerTester(SystemName.Public); private final MockMeteringClient snapshotConsumer = new MockMeteringClient(); private final MetricsMock metrics = new MetricsMock(); + private final ResourceMeterMaintainer maintainer = + new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); + + @Test + public void updates_deployment_costs() { + ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + ApplicationId app2 = ApplicationId.from("t2", "a1", "default"); + ZoneId z1 = ZoneId.from("prod.aws-us-east-1c"); + ZoneId z2 = ZoneId.from("prod.aws-eu-west-1a"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(z1.region()).region(z2.region()).trustDefaultCertificate().build(); + List.of(app1, app2).forEach(app -> deploymentTester.newDeploymentContext(app).submit(applicationPackage).deploy()); + + BiConsumer<ApplicationId, Map<ZoneId, Double>> assertCost = (appId, costs) -> + assertEquals(costs, tester.controller().applications().getInstance(appId).get().deployments().entrySet().stream() + .filter(entry -> entry.getValue().cost().isPresent()) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble()))); + + List<ResourceSnapshot> resourceSnapshots = List.of( + new ResourceSnapshot(app1, 12, 34, 56, Instant.EPOCH, z1), + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59)); + + // Remove a region from app1 and add region to app2 + resourceSnapshots = List.of( + new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1), + new ResourceSnapshot(app2, 45, 67, 89, Instant.EPOCH, z2)); + maintainer.updateDeploymentCost(resourceSnapshots); + assertCost.accept(app1, Map.of(z2, 2.50)); + assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68)); + } @Test public void testMaintainer() { setUpZones(); - ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); long lastRefreshTime = tester.clock().millis(); tester.curator().writeMeteringRefreshTime(lastRefreshTime); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots @@ -63,26 +105,18 @@ public class ResourceMeterMaintainerTest { var millisAdvanced = 3600 * 1000; tester.clock().advance(Duration.ofMillis(millisAdvanced)); - resourceMeterMaintainer.maintain(); + maintainer.maintain(); assertTrue(snapshotConsumer.isRefreshed()); assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime()); } private void setUpZones() { - ZoneApiMock nonAwsZone = ZoneApiMock.newBuilder().withId("test.region-1").build(); - ZoneApiMock awsZone1 = ZoneApiMock.newBuilder().withId("prod.region-2").withCloud("aws").build(); - ZoneApiMock awsZone2 = ZoneApiMock.newBuilder().withId("test.region-3").withCloud("aws").build(); - tester.zoneRegistry().setZones( - nonAwsZone, - awsZone1, - awsZone2); - tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); - tester.configServer().nodeRepository().putNodes( - awsZone1.getId(), - createNodes() - ); + ZoneApiMock zone1 = ZoneApiMock.newBuilder().withId("prod.region-2").build(); + ZoneApiMock zone2 = ZoneApiMock.newBuilder().withId("test.region-3").build(); + tester.zoneRegistry().setZones(zone1, zone2); + tester.configServer().nodeRepository().setFixedNodes(zone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone2.getId()); + tester.configServer().nodeRepository().putNodes(zone1.getId(), createNodes()); } private List<Node> createNodes() { @@ -92,23 +126,21 @@ public class ResourceMeterMaintainerTest { Node.State.failed, Node.State.parked, Node.State.active) - .map(state -> { - return new Node.Builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) - .state(state) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) - .build(); - }) + .map(state -> new Node.Builder() + .hostname(HostName.from("host" + state)) + .parentHostname(HostName.from("parenthost" + state)) + .state(state) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) + .build()) .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 2dcf012ac6d..37a173ffc37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -96,14 +96,15 @@ public class ApplicationSerializerTest { Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3), - DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none)); + DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt), OptionalDouble.of(200), OptionalDouble.of(10)), - QuotaUsage.create(OptionalDouble.of(23.5)))); + QuotaUsage.create(OptionalDouble.of(23.5)), + OptionalDouble.of(12.3))); var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"), new RotationStatus.Targets( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 8df2fd87398..6e12373640d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -19,14 +19,12 @@ import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import org.junit.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Collections; import java.util.Optional; -import java.util.function.BiConsumer; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -155,18 +153,4 @@ public class RunSerializerTest { assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } - @Test - public void convergenceSummaryMigrationTest() throws IOException { - String data = Files.readString(runFile); - BiConsumer<String, ConvergenceSummary> replaceAndAssert = (replace, convergenceSummaryOrNull) -> { - byte[] newData = data.replace("\"convergenceSummaryV2\": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233],", replace).getBytes(StandardCharsets.UTF_8); - assertEquals(convergenceSummaryOrNull, serializer.runsFromSlime(SlimeUtils.jsonToSlime(newData)).get(id).convergenceSummary().orElse(null)); - }; - - replaceAndAssert.accept("", null); - replaceAndAssert.accept("\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0)); - replaceAndAssert.accept("\"convergenceSummaryV2\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],\n" + - "\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 69677215f5e..47aa3e6b9d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -75,7 +75,6 @@ import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import javax.security.auth.x500.X500Principal; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index 3353d80204e..3a3fdfbf6c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -36,6 +36,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "archiveUri": "s3://bucketName/scoober/", "activity": {}, "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json index 6efcc822264..eb508b2459e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json @@ -60,6 +60,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 8767c369bc3..97ac87fb5a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -44,6 +44,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json index b59c1d6cf73..39b8c779184 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json @@ -36,6 +36,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": {}, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 6c00d654008..3ce83528b2c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -53,6 +53,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 1084afc9388..d61bebc81d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -21,6 +21,7 @@ "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", "screwdriverId": "123", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 9059ea338b1..4edbc58121b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -56,6 +56,7 @@ "commit": "commit1" }, "status": "complete", + "quota": "(ignore)", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, |