Merge remote-tracking branch 'origin/master' into olaa/metering-monitoring-metrics

# Conflicts:
#	controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java
#	controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java

32 files changed, 318 insertions, 300 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
index 8546eb5a971..281ac50e63a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Instant;
@@ -88,15 +87,14 @@ public class ApplicationPackageValidator {
     /** Verify that each of the production zones listed in the deployment spec exist in this system */
     private void validateSteps(DeploymentSpec deploymentSpec) {
         for (var spec : deploymentSpec.instances()) {
-            new DeploymentSteps(spec, controller.zoneRegistry()).jobs();
-            spec.zones().stream()
-                .filter(zone -> zone.environment() == Environment.prod)
-                .forEach(zone -> {
-                    if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(),
-                                                                         zone.region().orElseThrow()))) {
-                        throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!");
-                    }
-                });
+            for (var zone : spec.zones()) {
+                if (zone.environment().isManuallyDeployed())
+                    throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment());
+
+                if (     zone.environment() == Environment.prod
+                    && ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), zone.region().orElseThrow())))
+                    throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!");
+            }
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index 52283a3e27d..7578c133fc6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableMap;
+import com.yahoo.collections.Iterables;
 import com.yahoo.component.Version;
 import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
@@ -13,6 +14,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.stream.CustomCollectors;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
@@ -22,6 +24,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -41,6 +45,7 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static com.yahoo.collections.Iterables.reversed;
 import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.next;
 import static com.yahoo.config.provision.Environment.prod;
 import static com.yahoo.config.provision.Environment.staging;
@@ -69,17 +74,21 @@ public class DeploymentStatus {
     private final JobList allJobs;
     private final JobType systemTest;
     private final JobType stagingTest;
+    private final VersionStatus versionStatus;
     private final Version systemVersion;
     private final Function<InstanceName, VersionCompatibility> versionCompatibility;
+    private final ZoneRegistry zones;
     private final Instant now;
     private final Map<JobId, StepStatus> jobSteps;
     private final List<StepStatus> allSteps;
 
-    public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones,
+    public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, VersionStatus versionStatus,
                             Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) {
         this.application = requireNonNull(application);
+        this.zones = zones;
         this.systemTest = JobType.systemTest(zones);
         this.stagingTest = JobType.stagingTest(zones);
+        this.versionStatus = requireNonNull(versionStatus);
         this.systemVersion = requireNonNull(systemVersion);
         this.versionCompatibility = versionCompatibility;
         this.now = requireNonNull(now);
@@ -142,8 +151,7 @@ public class DeploymentStatus {
     public Map<JobType, JobStatus> instanceJobs(InstanceName instance) {
         return allJobs.asList().stream()
                       .filter(job -> job.id().application().equals(application.id().instance(instance)))
-                      .collect(Collectors.toUnmodifiableMap(job -> job.id().type(),
-                                                            Function.identity()));
+                      .collect(CustomCollectors.toLinkedMap(job -> job.id().type(), Function.identity()));
     }
 
     /** Filterable job status lists for each instance of this application. */
@@ -202,13 +210,30 @@ public class DeploymentStatus {
                                                                        .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment())
                                                                        .filter(jobId -> deploymentFor(jobId).isPresent())
                                                                        .findFirst();
-            Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion);
+            Versions versions = Versions.from(change,
+                                              application,
+                                              firstProductionJobWithDeployment.flatMap(this::deploymentFor),
+                                              fallbackPlatform(change, job));
             if (step.completedAt(change, Optional.empty()).isEmpty())
                 jobs.merge(job, List.of(new Job(job.type(), versions, step.readyAt(change), change)), DeploymentStatus::union);
         });
         return Collections.unmodifiableMap(jobs);
     }
 
+    /** Fall back to the newest, deployable platform, which is compatible with what we want to deploy. */
+    public Version fallbackPlatform(Change change, JobId job) {
+        Optional<Version> compileVersion = change.revision().map(application.revisions()::get).flatMap(ApplicationVersion::compileVersion);
+        if (compileVersion.isEmpty())
+            return systemVersion;
+
+        for (VespaVersion version : reversed(versionStatus.deployableVersions()))
+            if (versionCompatibility.apply(job.application().instance()).accept(version.versionNumber(), compileVersion.get()))
+                return version.versionNumber();
+
+        throw new IllegalArgumentException("no legal platform version exists in this system for compile version " + compileVersion.get());
+    }
+
+
     /** The set of jobs that need to run for the given changes to be considered complete. */
     public boolean hasCompleted(InstanceName instance, Change change) {
         if ( ! application.deploymentSpec().requireInstance(instance).concerns(prod)) {
@@ -365,7 +390,7 @@ public class DeploymentStatus {
                     return;
 
                 Change currentChange = application.require(instance).change();
-                Versions target = Versions.from(currentChange, application, deployment, systemVersion);
+                Versions target = Versions.from(currentChange, application, deployment, fallbackPlatform(currentChange, job));
                 existingPlatform = Optional.of(target.targetPlatform());
                 existingRevision = Optional.of(target.targetRevision());
             }
@@ -373,7 +398,7 @@ public class DeploymentStatus {
             List<Change> changes = deployingCompatibilityChange ? List.of(change) : changes(job, step, change);
             for (Change partial : changes) {
                 Job jobToRun = new Job(job.type(),
-                                       Versions.from(partial, application, existingPlatform, existingRevision, systemVersion),
+                                       Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)),
                                        step.readyAt(partial, Optional.of(job)),
                                        partial);
                 toRun.add(jobToRun);
@@ -847,7 +872,7 @@ public class DeploymentStatus {
                 @Override
                 public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) {
                     Optional<Instant> readyAt = super.readyAt(change, dependent);
-                    Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.systemVersion));
+                    Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id())));
                     if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty();
                     return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt;
                 }
@@ -918,7 +943,7 @@ public class DeploymentStatus {
                                                             .map(deployment -> run.versions().targetsMatch(Versions.from(change,
                                                                                                                          status.application,
                                                                                                                          Optional.of(deployment),
-                                                                                                                         status.systemVersion)))
+                                                                                                                         status.fallbackPlatform(change, dependent.get()))))
                                                             .orElseGet(() ->    (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform()))
                                                                              && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision()))))
                                   .matching(Run::hasSucceeded)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java
deleted file mode 100644
index 44079a90097..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.deployment;
-
-import com.yahoo.config.application.api.DeploymentInstanceSpec;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.util.Comparator.comparingInt;
-import static java.util.stream.Collectors.collectingAndThen;
-
-/**
- * This class provides helper methods for reading a deployment spec.
- *
- * @author mpolden
- */
-public class DeploymentSteps {
-
-    private final DeploymentInstanceSpec spec;
-    private final ZoneRegistry zones;
-
-    public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) {
-        this.spec = Objects.requireNonNull(spec, "spec cannot be null");
-        this.zones = Objects.requireNonNull(zones, "system cannot be null");
-    }
-
-    /** Returns jobs for this, in the order they should run */
-    public List<JobType> jobs() {
-        return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)),
-                             spec.steps().stream().flatMap(step -> toJobs(step).stream()))
-                     .distinct()
-                     .collect(Collectors.toUnmodifiableList());
-    }
-
-    /** Returns job status sorted according to deployment spec */
-    public List<JobStatus> sortedJobs(Collection<JobStatus> jobStatus) {
-        List<JobType> sortedJobs = jobs();
-        return jobStatus.stream()
-                        .sorted(comparingInt(job -> sortedJobs.indexOf(job.id().type())))
-                        .collect(Collectors.toUnmodifiableList());
-    }
-
-    /** Returns deployments sorted according to declared zones */
-    public List<Deployment> sortedDeployments(Collection<Deployment> deployments) {
-        List<ZoneId> productionZones = spec.zones().stream()
-                                           .filter(z -> z.region().isPresent())
-                                           .map(z -> ZoneId.from(z.environment(), z.region().get()))
-                                           .collect(Collectors.toUnmodifiableList());
-        return deployments.stream()
-                          .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone())))
-                          .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
-    }
-
-    /** Resolve jobs from step */
-    public List<JobType> toJobs(DeploymentSpec.Step step) {
-        return step.zones().stream()
-                   .map(this::toJob)
-                   .collect(Collectors.toUnmodifiableList());
-    }
-
-    /** Returns test jobs to run for this spec */
-    public List<JobType> testJobs() {
-        return jobs().stream().filter(type -> type.environment().isTest()).collect(Collectors.toUnmodifiableList());
-    }
-
-    /** Returns declared production jobs in this */
-    public List<JobType> productionJobs() {
-        return toJobs(production());
-    }
-
-    /** Returns declared production steps in this */
-    public List<DeploymentSpec.Step> production() {
-        return spec.steps().stream()
-                   .filter(step -> ! isTest(step))
-                   .collect(Collectors.toUnmodifiableList());
-    }
-
-    private boolean isTest(DeploymentSpec.Step step) {
-        return step.concerns(Environment.test) || step.concerns(Environment.staging);
-    }
-
-    /** Resolve job from deployment zone */
-    private JobType toJob(DeploymentSpec.DeclaredZone zone) {
-        switch (zone.environment()) {
-            case prod: return JobType.prod(zone.region().get());
-            case test: return JobType.systemTest(zones);
-            case staging: return JobType.stagingTest(zones);
-            default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment());
-        }
-    }
-
-    /** Resolve jobs from steps */
-    private List<JobType> toJobs(List<DeploymentSpec.Step> steps) {
-        return steps.stream()
-                    .flatMap(step -> toJobs(step).stream())
-                    .collect(Collectors.toUnmodifiableList());
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 6e2ae0da46d..df8ee910dc4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -71,10 +71,6 @@ public class DeploymentTrigger {
         this.jobs = controller.jobController();
     }
 
-    public DeploymentSteps steps(DeploymentInstanceSpec spec) {
-        return new DeploymentSteps(spec, controller.zoneRegistry());
-    }
-
     /**
      * Propagates the latest revision to ready instances.
      * Ready instances are those whose dependencies are complete, and which aren't blocked, and, additionally,
@@ -158,7 +154,7 @@ public class DeploymentTrigger {
      *
      * Only one job per type is triggered each run for test jobs, since their environments have limited capacity.
      */
-    public long triggerReadyJobs() {
+    public TriggerResult triggerReadyJobs() {
         List<Job> readyJobs = computeReadyJobs();
 
         var prodJobs = new ArrayList<Job>();
@@ -182,23 +178,34 @@ public class DeploymentTrigger {
                 .collect(groupingBy(Job::jobType));
 
         // Trigger all prod jobs
-        sortedProdJobs.forEach(this::trigger);
-        long triggeredJobs = sortedProdJobs.size();
+        long triggeredJobs = 0;
+        long failedJobs = 0;
+        for (Job job : sortedProdJobs) {
+            if (trigger(job)) ++triggeredJobs;
+            else ++failedJobs;
+        }
 
         // Trigger max one test job per type
-        for (var jobs : sortedTestJobsByType.values()) {
-            if (jobs.size() > 0) {
-                trigger(jobs.get(0));
-                triggeredJobs++;
-            }
-        }
-        return triggeredJobs;
+        for (Collection<Job> jobs: sortedTestJobsByType.values())
+            for (Job job : jobs)
+                if (trigger(job)) { ++triggeredJobs; break; }
+                else ++failedJobs;
+
+        return new TriggerResult(triggeredJobs, failedJobs);
     }
 
+    public record TriggerResult(long triggered, long failed) { }
 
     /** Attempts to trigger the given job. */
-    private void trigger(Job job) {
-        trigger(job, null);
+    private boolean trigger(Job job) {
+        try {
+            trigger(job, null);
+            return true;
+        }
+        catch (Exception e) {
+            log.log(Level.WARNING, "Failed triggering " + job.jobType() + " for " + job.instanceId, e);
+            return false;
+        }
     }
 
     /** Attempts to trigger the given job. */
@@ -237,7 +244,7 @@ public class DeploymentTrigger {
         Change change = instance.change();
         if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication();
         if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform();
-        Versions versions = Versions.from(change, application, status.deploymentFor(job), controller.readSystemVersion());
+        Versions versions = Versions.from(change, application, status.deploymentFor(job), status.fallbackPlatform(change, job));
         DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change());
         Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger)));
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 3d4a2f40303..5113d386b23 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
 import com.yahoo.component.VersionCompatibility;
+import com.yahoo.concurrent.UncheckedTimeoutException;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.transaction.Mutex;
@@ -34,12 +35,14 @@ import com.yahoo.vespa.hosted.controller.notification.Notification.Type;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Deque;
@@ -343,7 +346,7 @@ public class JobController {
     public List<Run> active() {
         return controller.applications().idList().stream()
                          .flatMap(id -> active(id).stream())
-                         .collect(toUnmodifiableList());
+                         .toList();
     }
 
     /** Returns a list of all active runs for the given application. */
@@ -353,7 +356,7 @@ public class JobController {
                                                  .map(type -> last(id.instance(name), type))
                                                  .flatMap(Optional::stream)
                                                  .filter(run -> ! run.hasEnded()))
-                         .collect(toUnmodifiableList());
+                         .toList();
     }
 
     /** Returns a list of all active runs for the given instance. */
@@ -362,7 +365,7 @@ public class JobController {
                       .map(type -> last(id, type))
                       .flatMap(Optional::stream)
                       .filter(run -> !run.hasEnded())
-                      .collect(toUnmodifiableList());
+                      .toList();
     }
 
     /** Returns the job status of the given job, possibly empty. */
@@ -372,28 +375,32 @@ public class JobController {
 
     /** Returns the deployment status of the given application. */
     public DeploymentStatus deploymentStatus(Application application) {
-        return deploymentStatus(application, controller.readSystemVersion());
+        VersionStatus versionStatus = controller.readVersionStatus();
+        return deploymentStatus(application, versionStatus, controller.systemVersion(versionStatus));
     }
 
-    private DeploymentStatus deploymentStatus(Application application, Version systemVersion) {
+    private DeploymentStatus deploymentStatus(Application application, VersionStatus versionStatus, Version systemVersion) {
         return new DeploymentStatus(application,
                                     this::jobStatus,
                                     controller.zoneRegistry(),
+                                    versionStatus,
                                     systemVersion,
                                     instance -> controller.applications().versionCompatibility(application.id().instance(instance)),
                                     controller.clock().instant());
     }
 
     /** Adds deployment status to each of the given applications. */
-    public DeploymentStatusList deploymentStatuses(ApplicationList applications, Version systemVersion) {
+    public DeploymentStatusList deploymentStatuses(ApplicationList applications, VersionStatus versionStatus) {
+        Version systemVersion = controller.systemVersion(versionStatus);
         return DeploymentStatusList.from(applications.asList().stream()
-                                                     .map(application -> deploymentStatus(application, systemVersion))
-                                                     .collect(toUnmodifiableList()));
+                                                     .map(application -> deploymentStatus(application, versionStatus, systemVersion))
+                                                     .toList());
     }
 
     /** Adds deployment status to each of the given applications. Calling this will do an implicit read of the controller's version status */
     public DeploymentStatusList deploymentStatuses(ApplicationList applications) {
-        return deploymentStatuses(applications, controller.readSystemVersion());
+        VersionStatus versionStatus = controller.readVersionStatus();
+        return deploymentStatuses(applications, versionStatus);
     }
 
     /** Changes the status of the given step, for the given run, provided it is still active. */
@@ -508,7 +515,7 @@ public class JobController {
 
             application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId));
             application = application.withRevisions(revisions -> revisions.with(version.get()));
-            application = withPrunedPackages(application);
+            application = withPrunedPackages(application, version.get().id());
 
             TestSummary testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage());
             if (testSummary.problems().isEmpty())
@@ -531,21 +538,25 @@ public class JobController {
             });
 
             applications.storeWithUpdatedConfig(application, submission.applicationPackage());
-            applications.deploymentTrigger().triggerNewRevision(id);
+            if (application.get().projectId().isPresent())
+                applications.deploymentTrigger().triggerNewRevision(id);
         });
         return version.get();
     }
 
-    private LockedApplication withPrunedPackages(LockedApplication application){
+    private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest){
         TenantAndApplicationId id = application.get().id();
-        Optional<RevisionId> oldestDeployed = application.get().oldestDeployedRevision();
-        if (oldestDeployed.isPresent()) {
-            controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed.get());
-
-            for (ApplicationVersion version : application.get().revisions().withPackage())
-                if (version.id().compareTo(oldestDeployed.get()) < 0)
-                    application = application.withRevisions(revisions -> revisions.with(version.withoutPackage()));
-        }
+        Application wrapped = application.get();
+        RevisionId oldestDeployed = application.get().oldestDeployedRevision()
+                                               .or(() -> wrapped.instances().values().stream()
+                                                                .flatMap(instance -> instance.change().revision().stream())
+                                                                .min(naturalOrder()))
+                                               .orElse(latest);
+        controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed);
+
+        for (ApplicationVersion version : application.get().revisions().withPackage())
+            if (version.id().compareTo(oldestDeployed) < 0)
+                application = application.withRevisions(revisions -> revisions.with(version.withoutPackage()));
         return application;
     }
 
@@ -584,8 +595,8 @@ public class JobController {
         if (revision.compileVersion()
                     .map(version -> controller.applications().versionCompatibility(id).refuse(versions.targetPlatform(), version))
                     .orElse(false))
-            throw new IllegalArgumentException("Will not start a job with incompatible platform version (" + versions.targetPlatform() + ") " +
-                                               "and compile versions (" + revision.compileVersion().get() + ")");
+            throw new IllegalArgumentException("Will not start " + type + " for " + id + " with incompatible platform version (" +
+                                               versions.targetPlatform() + ") " + "and compile versions (" + revision.compileVersion().get() + ")");
 
         locked(id, type, __ -> {
             Optional<Run> last = last(id, type);
@@ -618,7 +629,7 @@ public class JobController {
 
         DeploymentId deploymentId = new DeploymentId(id, type.zone());
         Optional<Run> lastRun = last(id, type);
-        lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
+        lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id(), Duration.ofMinutes(2)));
 
         long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L);
         RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type));
@@ -659,14 +670,6 @@ public class JobController {
     }
 
     private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) {
-        Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion();
-        if (major.isPresent())
-            return controller.applications().lastCompatibleVersion(major.get())
-                             .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " +
-                                                                             "but no version on this major was found"));
-
-        VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId());
-
         // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed.
         List<Version> versions = controller.readVersionStatus().deployableVersions().stream()
                                            .map(VespaVersion::versionNumber)
@@ -676,24 +679,47 @@ public class JobController {
                 .map(Deployment::version)
                 .ifPresent(versions::add);
 
+        if (versions.isEmpty())
+            throw new IllegalStateException("no deployable platform version found in the system");
+
+        VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId());
+        List<Version> compatibleVersions = new ArrayList<>();
         for (Version target : reversed(versions))
             if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get()))
+                compatibleVersions.add(target);
+
+        if (compatibleVersions.isEmpty())
+            throw new IllegalArgumentException("no platforms are compatible with compile version " + applicationPackage.compileVersion().get());
+
+        Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion();
+        List<Version> versionOnRightMajor = new ArrayList<>();
+        for (Version target : reversed(versions))
+            if (major.isEmpty() || major.get() == target.getMajor())
+                versionOnRightMajor.add(target);
+
+        if (versionOnRightMajor.isEmpty())
+            throw new IllegalArgumentException("no platforms were found for major version " + major.get() + " specified in deployment.xml");
+
+        for (Version target : compatibleVersions)
+            if (versionOnRightMajor.contains(target))
                 return target;
 
-        throw new IllegalArgumentException("no suitable platform version found" +
-                                           applicationPackage.compileVersion()
-                                                             .map(version -> " for package compiled against " + version)
-                                                             .orElse(""));
+        throw new IllegalArgumentException("no platforms on major version " + major.get() + " specified in deployment.xml " +
+                                           "are compatible with compile version " + applicationPackage.compileVersion().get());
     }
 
     /** Aborts a run and waits for it complete. */
-    private void abortAndWait(RunId id) {
+    private void abortAndWait(RunId id, Duration timeout) {
         abort(id, "replaced by new deployment");
         runner.get().accept(last(id.application(), id.type()).get());
 
+        Instant doom = controller.clock().instant().plus(timeout);
+        Duration sleep = Duration.ofMillis(100);
         while ( ! last(id.application(), id.type()).get().hasEnded()) {
+            if (controller.clock().instant().plus(sleep).isAfter(doom))
+                throw new UncheckedTimeoutException("timeout waiting for " + id + " to abort and finish");
             try {
-                Thread.sleep(100);
+                Thread.sleep(sleep.toMillis());
             }
             catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 1bb594e99ed..4aeecdcd4ff 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer));
         maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher));
         maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer()));
-        maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService()));
+        maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase()));
         maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger()));
         maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector));
         maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
index 59ecd3e5aef..51b40a9a4c7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -59,8 +59,9 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
     private boolean canTriggerAt(Instant instant) {
         int hourOfDay = instant.atZone(ZoneOffset.UTC).getHour();
         int dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek().getValue();
-        // Upgrade can only be scheduled between 07:00 and 12:59 UTC, Monday-Thursday
-        return hourOfDay >= 7 && hourOfDay <= 12 && dayOfWeek < 5;
+        // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday
+        int startHour = controller().system().isCd() ? 2 : 7;
+        return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5;
     }
 
     private Release releaseIn(CloudName cloud) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
index db9dc51ffa9..8a14dd3a146 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
@@ -22,6 +22,7 @@ public class OutstandingChangeDeployer extends ControllerMaintainer {
     protected double maintain() {
         for (Application application : ApplicationList.from(controller().applications().readable())
                                                       .withProductionDeployment()
+                                                      .withProjectId()
                                                       .withDeploymentSpec()
                                                       .asList())
             controller().applications().deploymentTrigger().triggerNewRevision(application.id());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java
index 26df8669fb1..974345330aa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.TriggerResult;
 
 import java.time.Duration;
 
@@ -18,8 +19,9 @@ public class ReadyJobsTrigger extends ControllerMaintainer {
 
     @Override
     public double maintain() {
-        controller().applications().deploymentTrigger().triggerReadyJobs();
-        return 1.0;
+        TriggerResult result = controller().applications().deploymentTrigger().triggerReadyJobs();
+        long total = result.triggered() + result.failed();
+        return total == 0 ? 1 : (double) result.triggered() / (result.triggered() + result.failed());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index d4905f7e20a..892ad669e4b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -10,13 +10,18 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -37,6 +42,7 @@ import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.stream.Collector;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Creates a {@link ResourceSnapshot} per application, which is then passed on to a MeteringClient
@@ -57,7 +63,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
 
     private final ApplicationController applications;
     private final NodeRepository nodeRepository;
-    private final MeteringClient meteringClient;
+    private final ResourceDatabaseClient resourceClient;
     private final CuratorDb curator;
     private final SystemName systemName;
     private final Metric metric;
@@ -71,11 +77,11 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
     public ResourceMeterMaintainer(Controller controller,
                                    Duration interval,
                                    Metric metric,
-                                   MeteringClient meteringClient) {
+                                   ResourceDatabaseClient resourceClient) {
         super(controller, interval);
         this.applications = controller.applications();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-        this.meteringClient = meteringClient;
+        this.resourceClient = resourceClient;
         this.curator = controller.curator();
         this.systemName = controller.serviceRegistry().zoneRegistry().system();
         this.metric = metric;
@@ -94,6 +100,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
         }
 
         if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots);
+        if (systemName.isPublic() && systemName.isCd()) reportAllScalingEvents();
         updateDeploymentCost(resourceSnapshots);
         return 1.0;
     }
@@ -124,13 +131,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
     }
 
     private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) {
-        meteringClient.consume(resourceSnapshots);
+        resourceClient.writeResourceSnapshots(resourceSnapshots);
 
         updateMeteringMetrics(resourceSnapshots);
 
         try (var lock = curator.lockMeteringRefreshTime()) {
             if (needsRefresh(curator.readMeteringRefreshTime())) {
-                meteringClient.refresh();
+                resourceClient.refreshMaterializedView();
                 curator.writeMeteringRefreshTime(clock.millis());
             }
         } catch (TimeoutException ignored) {
@@ -147,6 +154,37 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
                 .collect(Collectors.toList());
     }
 
+    private Stream<Instance> mapApplicationToInstances(Application application) {
+        return application.instances().values().stream();
+    }
+
+    private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) {
+        return instance.deployments().keySet().stream().map(zoneId -> {
+            return new DeploymentId(instance.id(), zoneId);
+        });
+    }
+
+    private Stream<Map.Entry<ClusterId, List<Cluster.ScalingEvent>>> mapDeploymentToClusterScalingEvent(DeploymentId deploymentId) {
+        return nodeRepository.getApplication(deploymentId.zoneId(), deploymentId.applicationId())
+                .clusters().entrySet().stream()
+                .map(cluster -> Map.entry(new ClusterId(deploymentId, cluster.getKey()), cluster.getValue().scalingEvents()));
+    }
+
+    private void reportAllScalingEvents() {
+        var clusters = controller().applications().asList().stream()
+                .flatMap(this::mapApplicationToInstances)
+                .flatMap(this::mapInstanceToDeployments)
+                .flatMap(this::mapDeploymentToClusterScalingEvent)
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        Map.Entry::getValue
+                ));
+
+        for (var cluster : clusters.entrySet()) {
+            resourceClient.writeScalingEvents(cluster.getKey(), cluster.getValue());
+        }
+    }
+
     private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) {
         return nodes.stream()
                 .filter(this::unlessNodeOwnerIsSystemApplication)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index 9cddbd0b903..d654f63fff2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -58,7 +58,7 @@ public class Upgrader extends ControllerMaintainer {
         cancelBrokenUpgrades(versionStatus);
 
         OptionalInt targetMajorVersion = targetMajorVersion();
-        InstanceList instances = instances(controller().systemVersion(versionStatus));
+        InstanceList instances = instances(versionStatus);
         for (UpgradePolicy policy : UpgradePolicy.values())
             updateTargets(versionStatus, instances, policy, targetMajorVersion);
 
@@ -66,10 +66,10 @@ public class Upgrader extends ControllerMaintainer {
     }
 
     /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */
-    private InstanceList instances(Version systemVersion) {
+    private InstanceList instances(VersionStatus versionStatus) {
         return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable())
                                                                                                 .withProjectId(),
-                                                                                 systemVersion))
+                                                                                 versionStatus))
                            .withDeclaredJobs()
                            .shuffle(random)
                            .byIncreasingDeployedVersion()
@@ -78,7 +78,7 @@ public class Upgrader extends ControllerMaintainer {
 
     private void cancelBrokenUpgrades(VersionStatus versionStatus) {
         // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation)
-        InstanceList instances = instances(controller().systemVersion(versionStatus));
+        InstanceList instances = instances(controller().readVersionStatus());
         for (VespaVersion version : versionStatus.versions()) {
             if (version.confidence() == Confidence.broken)
                 cancelUpgradesOf(instances.upgradingTo(version.versionNumber()).not().with(UpgradePolicy.canary),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 63f33540721..cf21b8ef0af 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -12,6 +12,7 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.annotation.Inject;
 import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
@@ -90,7 +91,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
 import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
@@ -141,6 +141,7 @@ import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
@@ -158,7 +159,9 @@ import java.util.stream.Stream;
 import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
 import static com.yahoo.jdisc.Response.Status.CONFLICT;
 import static com.yahoo.yolean.Exceptions.uncheck;
+import static java.util.Comparator.comparingInt;
 import static java.util.Map.Entry.comparingByKey;
+import static java.util.stream.Collectors.collectingAndThen;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toUnmodifiableList;
@@ -1528,10 +1531,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         object.setString("instance", instance.name().value());
 
         if (deploymentSpec.instance(instance.name()).isPresent()) {
-            // Jobs sorted according to deployment spec
-            List<JobStatus> jobStatus = controller.applications().deploymentTrigger()
-                                                  .steps(deploymentSpec.requireInstance(instance.name()))
-                                                  .sortedJobs(status.instanceJobs(instance.name()).values());
+            // Jobs ordered according to deployment spec
+            Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values();
 
             if ( ! instance.change().isEmpty())
                 toSlime(object.setObject("deploying"), instance.change(), status.application());
@@ -1559,8 +1560,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
 
         // Deployments sorted according to deployment spec
         List<Deployment> deployments = deploymentSpec.instance(instance.name())
-                                                     .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry()))
-                                                     .map(steps -> steps.sortedDeployments(instance.deployments().values()))
+                                                     .map(spec -> sortedDeployments(instance.deployments().values(), spec))
                                                      .orElse(List.copyOf(instance.deployments().values()));
 
         Cursor deploymentsArray = object.setArray("deployments");
@@ -1613,10 +1613,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         application.projectId().ifPresent(id -> object.setLong("projectId", id));
 
         if (application.deploymentSpec().instance(instance.name()).isPresent()) {
-            // Jobs sorted according to deployment spec
-            List<JobStatus> jobStatus = controller.applications().deploymentTrigger()
-                                                  .steps(application.deploymentSpec().requireInstance(instance.name()))
-                                                  .sortedJobs(status.instanceJobs(instance.name()).values());
+            // Jobs ordered according to deployment spec
+            Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values();
 
             if ( ! instance.change().isEmpty())
                 toSlime(object.setObject("deploying"), instance.change(), application);
@@ -1645,11 +1643,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         addRotationId(object, instance);
 
         // Deployments sorted according to deployment spec
-        List<Deployment> deployments =
-                application.deploymentSpec().instance(instance.name())
-                           .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry()))
-                           .map(steps -> steps.sortedDeployments(instance.deployments().values()))
-                           .orElse(List.copyOf(instance.deployments().values()));
+        List<Deployment> deployments = application.deploymentSpec().instance(instance.name())
+                                                  .map(spec -> sortedDeployments(instance.deployments().values(), spec))
+                                                  .orElse(List.copyOf(instance.deployments().values()));
         Cursor instancesArray = object.setArray("instances");
         for (Deployment deployment : deployments) {
             Cursor deploymentObject = instancesArray.addObject();
@@ -3017,5 +3013,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         }
     }
 
+    private List<Deployment> sortedDeployments(Collection<Deployment> deployments, DeploymentInstanceSpec spec) {
+        List<ZoneId> productionZones = spec.zones().stream()
+                                           .filter(z -> z.region().isPresent())
+                                           .map(z -> ZoneId.from(z.environment(), z.region().get()))
+                                           .toList();
+        return deployments.stream()
+                          .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone())))
+                          .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
 }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java
index 419d1f4cb33..44a8b636ae0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java
@@ -20,7 +20,9 @@ import com.yahoo.vespa.hosted.controller.TenantController;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
@@ -44,6 +46,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
     private final ApplicationController applications;
     private final TenantController tenants;
     private final BillingController billing;
+    private final PlanRegistry planRegistry;
     private final Clock clock;
 
     public BillingApiHandlerV2(ThreadedHttpRequestHandler.Context context, Controller controller) {
@@ -51,6 +54,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         this.applications = controller.applications();
         this.tenants = controller.tenants();
         this.billing = controller.serviceRegistry().billingController();
+        this.planRegistry = controller.serviceRegistry().planRegistry();
         this.clock = controller.serviceRegistry().clock();
     }
 
@@ -76,6 +80,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
                 .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}")
                         .get(self::previewBill)
                         .post(Slime.class, self::createBill))
+                .addRoute(RestApi.route("/billing/v2/accountant/plans")
+                        .get(self::plans))
                 .addExceptionMapper(RuntimeException.class, (__, e) -> ErrorResponse.internalServerError(e.getMessage()))
                 .build();
     }
@@ -86,13 +92,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant"));
         var tenant = tenants.require(tenantName, CloudTenant.class);
 
-        var plan = billing.getPlan(tenant.name());
+        var plan = planFor(tenant.name());
         var collectionMethod = billing.getCollectionMethod(tenant.name());
 
         var response = new Slime();
         var cursor = response.setObject();
         cursor.setString("tenant", tenant.name().value());
-        cursor.setString("plan", plan.value());
+
+        toSlime(cursor.setObject("plan"), plan);
         cursor.setString("collection", collectionMethod.name());
         return response;
     }
@@ -129,7 +136,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         var response = new Slime();
         var cursor = response.setObject();
         cursor.setString("tenant", tenant.name().value());
-        cursor.setString("plan", billing.getPlan(tenant.name()).value());
+        toSlime(cursor.setObject("plan"), planFor(tenant.name()));
         cursor.setString("collection", billing.getCollectionMethod(tenant.name()).name());
         return response;
     }
@@ -197,7 +204,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
             var usage = Optional.ofNullable(usagePerTenant.get(tenant.name()));
             var tenantResponse = tenantsResponse.addObject();
             tenantResponse.setString("tenant", tenant.name().value());
-            tenantResponse.setString("plan", billing.getPlan(tenant.name()).value());
+            toSlime(tenantResponse.setObject("plan"), planFor(tenant.name()));
             tenantResponse.setString("collection", billing.getCollectionMethod(tenant.name()).name());
             tenantResponse.setString("lastBill", usage.map(Bill::getStartDate).map(DateTimeFormatter.ISO_DATE::format).orElse(null));
             tenantResponse.setString("unbilled", usage.map(Bill::sum).map(BigDecimal::toPlainString).orElse("0.00"));
@@ -236,6 +243,18 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         return new MessageResponse("Created bill " + invoiceId.value());
     }
 
+    private HttpResponse plans(RestApi.RequestContext ctx) {
+        var slime = new Slime();
+        var root = slime.setObject();
+        var plans = root.setArray("plans");
+        for (var plan : planRegistry.all()) {
+            var p = plans.addObject();
+            p.setString("id", plan.id().value());
+            p.setString("name", plan.displayName());
+        }
+        return new SlimeJsonResponse(slime);
+    }
+
 
     // --------- INVOICE RENDERING ----------
 
@@ -284,8 +303,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         slime.setString("id", item.id());
         slime.setString("description", item.description());
         slime.setString("amount",item.amount().toString());
-        slime.setString("plan", item.plan());
-        slime.setString("planName", billing.getPlanDisplayName(PlanId.from(item.plan())));
+        toSlime(slime.setObject("plan"), planRegistry.plan(item.plan()).orElseThrow(() -> new RuntimeException("No such plan: '" + item.plan() + "'")));
 
         item.applicationId().ifPresent(appId -> {
             slime.setString("application", appId.application().value());
@@ -334,4 +352,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler
         return inspector.field(field).asString();
     }
 
+    private void toSlime(Cursor cursor, Plan plan) {
+        cursor.setString("id", plan.id().value());
+        cursor.setString("name", plan.displayName());
+    }
+
+    private Plan planFor(TenantName tenant) {
+        var planId = billing.getPlan(tenant);
+        return planRegistry.plan(planId)
+                .orElseThrow(() -> new RuntimeException("No such plan: '" + planId + "'"));
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index 3ca6335dcb7..25ac90ac0ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -84,7 +84,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler {
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(controller.jobControl());
         if (path.matches("/controller/v1/stats")) return new StatsResponse(controller);
         if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader());
-        if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().meteringService(), path.get("tenant"), path.get("month"));
+        if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().resourceDatabase(), path.get("tenant"), path.get("month"));
         return notFound(path);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
index 33cd4948a7e..17461aafd02 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
@@ -5,7 +5,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 
 import java.time.YearMonth;
@@ -16,14 +16,14 @@ import java.util.List;
  */
 public class MeteringResponse extends SlimeJsonResponse {
 
-    public MeteringResponse(MeteringClient meteringClient, String tenantName, String month) {
-        super(toSlime(meteringClient, tenantName, month));
+    public MeteringResponse(ResourceDatabaseClient resourceClient, String tenantName, String month) {
+        super(toSlime(resourceClient, tenantName, month));
     }
 
-    private static Slime toSlime(MeteringClient meteringClient, String tenantName, String month) {
+    private static Slime toSlime(ResourceDatabaseClient resourceClient, String tenantName, String month) {
         Slime slime = new Slime();
         Cursor root = slime.setArray();
-        List<ResourceSnapshot> snapshots = meteringClient.getSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month));
+        List<ResourceSnapshot> snapshots = resourceClient.getRawSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month));
         snapshots.forEach(snapshot -> {
             Cursor object = root.addObject();
             object.setString("applicationId", snapshot.getApplicationId().toShortString());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index effa0906b94..7e43de9f274 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -94,9 +94,8 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler {
         Cursor root = slime.setObject();
         Cursor platformArray = root.setArray("versions");
         var versionStatus = controller.readVersionStatus();
-        var systemVersion = controller.systemVersion(versionStatus);
         ApplicationList applications = ApplicationList.from(controller.applications().asList()).withJobs();
-        var deploymentStatuses = controller.jobController().deploymentStatuses(applications, systemVersion);
+        var deploymentStatuses = controller.jobController().deploymentStatuses(applications, versionStatus);
         var deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toList()),
                                                                 deploymentStatuses)
                                                        .stream().collect(toMap(DeploymentStatistics::version, identity()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 53557bafcb0..28536f36e20 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -748,15 +748,15 @@ public class ControllerTest {
             fail("Should fail when specifying a major that does not yet exist");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("major 8 specified in deployment.xml, but no version on this major was found", e.getMessage());
+            assertEquals("no platforms were found for major version 8 specified in deployment.xml", e.getMessage());
         }
 
         try {
             context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build());
-            fail("Should fail when compiled against a version which does not yet exist");
+            fail("Should fail when compiled against a version which is only compatible with not-yet-existent versions");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("no suitable platform version found for package compiled against 8", e.getMessage());
+            assertEquals("no platforms are compatible with compile version 8", e.getMessage());
         }
 
         tester.controllerTester().upgradeSystem(version3);
@@ -765,7 +765,7 @@ public class ControllerTest {
             fail("Should fail when specifying a major which is incompatible with compile version");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("Will not start a job with incompatible platform version (8) and compile versions (7)", e.getMessage());
+            assertEquals("no platforms on major version 8 specified in deployment.xml are compatible with compile version 7", e.getMessage());
         }
 
         context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index b3db4a8b845..347f1d4ab15 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -220,12 +220,11 @@ public class DeploymentContext {
                                 .allMatch(deployments -> deployments.stream()
                                                                     .allMatch(deployment -> deployment.version().equals(version))));
 
-        for (var spec : application().deploymentSpec().instances())
-            for (JobType type : new DeploymentSteps(spec, tester.controller().zoneRegistry()).productionJobs())
-                assertTrue(tester.configServer().nodeRepository()
-                                 .list(type.zone(),
-                                       NodeFilter.all().applications(applicationId.defaultInstance())).stream()
-                                 .allMatch(node -> node.currentVersion().equals(version)));
+        for (JobId job : deploymentStatus().jobs().matching(job -> job.id().type().isProduction()).mapToList(JobStatus::id))
+            assertTrue(tester.configServer().nodeRepository()
+                             .list(job.type().zone(),
+                                   NodeFilter.all().applications(job.application())).stream()
+                             .allMatch(node -> node.currentVersion().equals(version)));
 
         assertFalse(instance().change().hasTargets());
         return this;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 78e7606d7c6..2b4a2baa17e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -146,7 +146,7 @@ public class DeploymentTester {
         int triggered;
         int triggeredTotal = 0;
         do {
-            triggered = (int) deploymentTrigger().triggerReadyJobs();
+            triggered = (int) deploymentTrigger().triggerReadyJobs().triggered();
             triggeredTotal += triggered;
         } while (triggered > 0);
         return triggeredTotal;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index d602ee8cde3..67ddc767b39 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -2146,6 +2146,14 @@ public class DeploymentTriggerTest {
         app.deploy();
         assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform());
         assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get());
+
+        DeploymentContext newApp = tester.newDeploymentContext("new", "app", "default")
+                                         .submit(new ApplicationPackageBuilder().region("us-east-3")
+                                                                                .compileVersion(version1)
+                                                                                .build())
+                                         .deploy();
+        assertEquals(version1, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform());
+        assertEquals(version1, newApp.application().revisions().get(tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get());
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 8e0d70bdb80..af542521b31 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIss
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
-import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer;
@@ -69,7 +68,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockMailer mockMailer = new MockMailer();
     private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock);
     private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock();
-    private final MockMeteringClient mockMeteringClient = new MockMeteringClient();
     private final MockContactRetriever mockContactRetriever = new MockContactRetriever();
     private final MockIssueHandler mockIssueHandler = new MockIssueHandler();
     private final DummyOwnershipIssues dummyOwnershipIssues = new DummyOwnershipIssues();
@@ -148,11 +146,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public MockMeteringClient meteringService() {
-        return mockMeteringClient;
-    }
-
-    @Override
     public MockContactRetriever contactRetriever() {
         return mockContactRetriever;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
index df30b6b57ee..187b8f932cf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java
@@ -83,6 +83,13 @@ public class CloudTrialExpirerTest {
         assertTrue(tester.controller().tenants().get("with-apps").isEmpty());
     }
 
+    @Test
+    public void keep_tenants_without_applications_that_are_idle() {
+        registerTenant("active", "none", Duration.ofDays(364));
+        expirer.maintain();
+        assertPlan("active", "none");
+    }
+
     private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) {
         var name = TenantName.from(tenantName);
         tester.createTenant(tenantName, Tenant.Type.cloud);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
index f9441f76a38..320938f00e4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -9,9 +9,10 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
-import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -38,10 +39,10 @@ import static org.junit.Assert.assertTrue;
 public class ResourceMeterMaintainerTest {
 
     private final ControllerTester tester = new ControllerTester(SystemName.Public);
-    private final MockMeteringClient snapshotConsumer = new MockMeteringClient();
+    private final ResourceDatabaseClientMock resourceClient = new ResourceDatabaseClientMock(new PlanRegistryMock());
     private final MetricsMock metrics = new MetricsMock();
     private final ResourceMeterMaintainer maintainer =
-            new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer);
+            new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, resourceClient);
 
     @Test
     public void updates_deployment_costs() {
@@ -89,7 +90,7 @@ public class ResourceMeterMaintainerTest {
         long lastRefreshTime = tester.clock().millis();
         tester.curator().writeMeteringRefreshTime(lastRefreshTime);
         maintainer.maintain();
-        Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources();
+        Collection<ResourceSnapshot> consumedResources = resourceClient.resourceSnapshots();
 
         // The mocked repository contains two applications, so we should also consume two ResourceSnapshots
         assertEquals(4, consumedResources.size());
@@ -110,13 +111,13 @@ public class ResourceMeterMaintainerTest {
         assertEquals(40d, (Double) metrics.getMetric(context -> "tenant2".equals(context.get("tenant")), "metering.vcpu").get(), Double.MIN_VALUE);
 
         // Metering is not refreshed
-        assertFalse(snapshotConsumer.isRefreshed());
+        assertFalse(resourceClient.hasRefreshedMaterializedView());
         assertEquals(lastRefreshTime, tester.curator().readMeteringRefreshTime());
 
         var millisAdvanced = 3600 * 1000;
         tester.clock().advance(Duration.ofMillis(millisAdvanced));
         maintainer.maintain();
-        assertTrue(snapshotConsumer.isRefreshed());
+        assertTrue(resourceClient.hasRefreshedMaterializedView());
         assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index d9f0f010104..2c0ab97c00e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -797,13 +797,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               },
                               200);
 
-        // GET application package for previous build
+        // GET application package for specific build
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET)
-                                      .properties(Map.of("build", "1"))
+                                      .properties(Map.of("build", "2"))
                                       .userIdentity(HOSTED_VESPA_OPERATOR),
                               (response) -> {
-                                  assertEquals("attachment; filename=\"tenant1.application1-build1.zip\"", response.getHeaders().getFirst("Content-Disposition"));
-                                  assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody());
+                                  assertEquals("attachment; filename=\"tenant1.application1-build2.zip\"", response.getHeaders().getFirst("Content-Disposition"));
+                                  assertArrayEquals(packageWithService.zippedContent(), response.getBody());
                               },
                               200);
 
@@ -842,7 +842,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               "{\"message\":\"Marked build '2' as non-deployable\"}");
 
         // GET deployment job overview, after triggering system and staging test jobs.
-        assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs());
+        assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs().triggered());
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job", GET)
                                       .userIdentity(USER_ID),
                               new File("jobs.json"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
index 81d363aa3e8..f39aab26d75 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
@@ -47,14 +47,6 @@
                 "sourceUrl": "repository1/tree/commit1",
                 "commit": "commit1"
               }
-            },
-            {
-              "application": {
-                "build": 1,
-                "compileVersion": "6.1.0",
-                "sourceUrl": "repository1/tree/commit1",
-                "commit": "commit1"
-              }
             }
           ],
           "blockers": [ ]
@@ -587,14 +579,6 @@
                 "sourceUrl": "repository1/tree/commit1",
                 "commit": "commit1"
               }
-            },
-            {
-              "application": {
-                "build": 1,
-                "compileVersion": "6.1.0",
-                "sourceUrl": "repository1/tree/commit1",
-                "commit": "commit1"
-              }
             }
           ],
           "blockers": [ ]
@@ -707,15 +691,6 @@
       "description": "my best commit yet",
       "risk": 9001,
       "deployable": false
-    },
-    {
-      "build": 1,
-      "compileVersion": "6.1.0",
-      "sourceUrl": "repository1/tree/commit1",
-      "commit": "commit1",
-      "description": "my best commit yet",
-      "risk": 9001,
-      "deployable": true
     }
   ]
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java
index c9e86849ed8..38ebe030e8e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java
@@ -254,7 +254,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest {
                 "some-id",
                 "description",
                 new BigDecimal("123.00"),
-                "some-plan",
+                "paid",
                 "Smith",
                 addedAt
         );
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java
index 69245f0cded..58701732515 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java
@@ -70,18 +70,19 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest {
     @Test
     public void require_tenant_info() {
         var request = request("/billing/v2/tenant/" + tenant.value()).roles(tenantReader);
-        tester.assertResponse(request, "{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"AUTO\"}");
+        tester.assertResponse(request, "{\"tenant\":\"tenant1\",\"plan\":{\"id\":\"trial\",\"name\":\"Free Trial - for testing purposes\"},\"collection\":\"AUTO\"}");
     }
 
     @Test
     public void require_admin_for_update_plan() {
         var request = request("/billing/v2/tenant/" + tenant.value(), Request.Method.PATCH)
-                .data("{\"plan\": \"pay-as-you-go\"}");
+                .data("{\"plan\": \"paid\"}");
 
         var forbidden = request.roles(tenantReader);
         tester.assertResponse(forbidden, ACCESS_DENIED, 403);
         var success = request.roles(tenantAdmin);
-        tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"pay-as-you-go\",\"collection\":\"AUTO\"}");
+        tester.assertResponse(success, """
+                {"tenant":"tenant1","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"collection":"AUTO"}""");
     }
 
     @Test
@@ -93,7 +94,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest {
         tester.assertResponse(forbidden, "{\"error-code\":\"FORBIDDEN\",\"message\":\"Only accountant can change billing method\"}", 403);
 
         var success = request.roles(financeAdmin);
-        tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"INVOICE\"}");
+        tester.assertResponse(success, """
+                {"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"INVOICE"}""");
     }
 
     @Test
@@ -108,7 +110,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest {
         tester.assertResponse(listRequest, "{\"invoices\":[{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\"}]}");
 
         var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader);
-        tester.assertResponse(singleRequest, "{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\",\"statusHistory\":[{\"at\":\"2020-05-23T00:00:00Z\",\"status\":\"OPEN\"}],\"items\":[{\"id\":\"some-id\",\"description\":\"description\",\"amount\":\"123.00\",\"plan\":\"some-plan\",\"planName\":\"Plan with id: some-plan\",\"cpu\":{},\"memory\":{},\"disk\":{}}]}");
+        tester.assertResponse(singleRequest, """
+                {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"cpu":{},"memory":{},"disk":{}}]}""");
     }
 
     @Test
@@ -120,7 +123,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest {
                 "}", 403);
 
         var accountantRequest = request("/billing/v2/accountant").roles(Role.hostedAccountant());
-        tester.assertResponse(accountantRequest, "{\"tenants\":[{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"AUTO\",\"lastBill\":null,\"unbilled\":\"0.00\"}]}");
+        tester.assertResponse(accountantRequest, """
+                {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"AUTO","lastBill":null,"unbilled":"0.00"}]}""");
     }
 
     @Test
@@ -136,4 +140,11 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest {
                 .data("{\"from\": \"2020-05-01\",\"to\": \"2020-06-01\"}");
         tester.assertResponse(accountantRequest, "{\"message\":\"Created bill id-123\"}");
     }
+
+    @Test
+    public void require_list_of_all_plans() {
+        var accountantRequest = request("/billing/v2/accountant/plans")
+                .roles(Role.hostedAccountant());
+        tester.assertResponse(accountantRequest, "{\"plans\":[{\"id\":\"trial\",\"name\":\"Free Trial - for testing purposes\"},{\"id\":\"paid\",\"name\":\"Paid Plan - for testing purposes\"},{\"id\":\"none\",\"name\":\"None Plan - for testing purposes\"}]}");
+    }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json
index fe89ef246bb..e9b18a879b9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json
@@ -15,8 +15,8 @@
             "id": "some-id",
             "description": "description",
             "amount": "123.00",
-            "plan": "some-plan",
-            "planName": "Plan with id: some-plan"
+            "plan": "paid",
+            "planName": "Plan with id: paid"
           }
         ]
       },
@@ -46,8 +46,8 @@
             "id": "some-id",
             "description": "description",
             "amount": "123.00",
-            "plan": "some-plan",
-            "planName": "Plan with id: some-plan"
+            "plan": "paid",
+            "planName": "Plan with id: paid"
           }
         ]
       },
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json
index 953b946c329..adb319a3642 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json
@@ -11,8 +11,8 @@
         "id": "some-id",
         "description": "description",
         "amount": "123.00",
-        "plan": "some-plan",
-        "planName": "Plan with id: some-plan"
+        "plan": "paid",
+        "planName": "Plan with id: paid"
       }
     ]
   },
@@ -37,8 +37,8 @@
           "id": "some-id",
           "description": "description",
           "amount": "123.00",
-          "plan": "some-plan",
-          "planName": "Plan with id: some-plan"
+          "plan": "paid",
+          "planName": "Plan with id: paid"
         }
       ]
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 94ca4268000..5936c135af9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -161,7 +161,7 @@ public class ControllerApiTest extends ControllerContainerTest {
                 new ResourceSnapshot(applicationId, 12,48,1200, NodeResources.Architecture.arm64, timestamp, zoneId),
                 new ResourceSnapshot(applicationId, 24, 96,2400,  NodeResources.Architecture.x86_64, timestamp, zoneId)
         );
-        tester.controller().serviceRegistry().meteringService().consume(snapshots);
+        tester.controller().serviceRegistry().resourceDatabase().writeResourceSnapshots(snapshots);
         tester.assertResponse(
                 operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET),
                 new File("metering.json")
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
index 279bd289c00..d99239e8f23 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
+import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
@@ -25,7 +26,9 @@ public class BadgeApiTest extends ControllerContainerTest {
     @Test
     public void testBadgeApi() throws IOException {
         ContainerTester tester = new ContainerTester(container, responseFiles);
-        var application = new DeploymentTester(new ControllerTester(tester)).newDeploymentContext("tenant", "application", "default");
+        DeploymentTester deploymentTester = new DeploymentTester(new ControllerTester(tester));
+        deploymentTester.controllerTester().upgradeSystem(Version.fromString("6.1"));
+        var application = deploymentTester.newDeploymentContext("tenant", "application", "default");
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder().parallel("us-west-1", "aws-us-east-1a")
                                                                                .test("us-west-1")
                                                                                .region("ap-southeast-1")
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
index 0b128ebf7a5..ea7ec7c1589 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.restapi.routing;
 
 import com.yahoo.application.container.handler.Request;
+import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
@@ -35,6 +36,7 @@ public class RoutingApiTest extends ControllerContainerTest {
     public void before() {
         tester = new ContainerTester(container, responseFiles);
         deploymentTester = new DeploymentTester(new ControllerTester(tester));
+        deploymentTester.controllerTester().upgradeSystem(Version.fromString("6.1"));
     }
 
     @Test