Merge branch 'master' into smorgrav/tenantinfo_get_api

42 files changed, 576 insertions, 278 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 513a18f8745..a97bf55e17d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
@@ -239,7 +240,7 @@ public class ApplicationController {
                                                                   .map(Node::currentVersion)
                                                                   .filter(version -> ! version.isEmpty()))
                                      .min(naturalOrder())
-                                     .orElse(controller.systemVersion());
+                                     .orElseGet(controller::readSystemVersion);
     }
 
     /**
@@ -368,12 +369,8 @@ public class ApplicationController {
     }
 
     private QuotaUsage deploymentQuotaUsage(ZoneId zoneId, ApplicationId applicationId) {
-        var quotaUsage = configServer.nodeRepository().getApplication(zoneId, applicationId)
-                .clusters().values().stream()
-                .map(Cluster::max)
-                .mapToDouble(max -> max.nodes() * max.nodeResources().cost())
-                .sum();
-        return QuotaUsage.create(quotaUsage);
+        var application = configServer.nodeRepository().getApplication(zoneId, applicationId);
+        return DeploymentQuotaCalculator.calculateQuotaUsage(application);
     }
 
     private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) {
@@ -414,7 +411,7 @@ public class ApplicationController {
                     platformVersion = options.vespaVersion.map(Version::new)
                                                           .orElse(applicationPackage.deploymentSpec().majorVersion()
                                                                                     .flatMap(this::lastCompatibleVersion)
-                                                                                    .orElseGet(controller::systemVersion));
+                                                                                    .orElseGet(controller::readSystemVersion));
                 }
                 else {
                     JobType jobType = JobType.from(controller.system(), zone)
@@ -893,7 +890,7 @@ public class ApplicationController {
 
     /** Returns the latest known version within the given major. */
     public Optional<Version> lastCompatibleVersion(int targetMajorVersion) {
-        return controller.versionStatus().versions().stream()
+        return controller.readVersionStatus().versions().stream()
                          .map(VespaVersion::versionNumber)
                          .filter(version -> version.getMajor() == targetMajorVersion)
                          .max(naturalOrder());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index b0309957604..81cba77b3c3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -163,7 +163,7 @@ public class Controller extends AbstractComponent {
 
     /** Replace the current version status by a new one */
     public void updateVersionStatus(VersionStatus newStatus) {
-        VersionStatus currentStatus = versionStatus();
+        VersionStatus currentStatus = readVersionStatus();
         if (newStatus.systemVersion().isPresent() &&
             ! newStatus.systemVersion().equals(currentStatus.systemVersion())) {
             log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) +
@@ -177,7 +177,7 @@ public class Controller extends AbstractComponent {
     }
     
     /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */
-    public VersionStatus versionStatus() { return curator.readVersionStatus(); }
+    public VersionStatus readVersionStatus() { return curator.readVersionStatus(); }
 
     /** Remove confidence override for versions matching given filter */
     public void removeConfidenceOverride(Predicate<Version> filter) {
@@ -189,10 +189,15 @@ public class Controller extends AbstractComponent {
     }
     
     /** Returns the current system version: The controller should drive towards running all applications on this version */
-    public Version systemVersion() {
-        return versionStatus().systemVersion()
-                              .map(VespaVersion::versionNumber)
-                              .orElse(Vtag.currentVersion);
+    public Version readSystemVersion() {
+        return systemVersion(readVersionStatus());
+    }
+
+    /** Returns the current system version from given status: The controller should drive towards running all applications on this version */
+    public Version systemVersion(VersionStatus versionStatus) {
+        return versionStatus.systemVersion()
+                            .map(VespaVersion::versionNumber)
+                            .orElse(Vtag.currentVersion);
     }
 
     /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java
index 82d83ea3585..4bef2ef7648 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java
@@ -1,7 +1,9 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
@@ -10,7 +12,12 @@ import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.List;
 
-/** Calculates the quota to allocate to a deployment. */
+/**
+ * Calculates the quota to allocate to a deployment.
+ *
+ * @author ogronnesby
+ * @author andreer
+ */
 public class DeploymentQuotaCalculator {
 
     public static Quota calculate(Quota tenantQuota,
@@ -25,6 +32,23 @@ public class DeploymentQuotaCalculator {
         return getMaximumAllowedQuota(tenantQuota, tenantApps, deployingApp, deployingZone);
     }
 
+    public static QuotaUsage calculateQuotaUsage(com.yahoo.vespa.hosted.controller.api.integration.configserver.Application application) {
+        // the .max() resources are only specified when the user has specified a max.  to make sure we enforce quotas
+        // correctly we retrieve the maximum of .current() and .max() - otherwise we would keep adding 0s for those
+        // that are not using autoscaling.
+        var quotaUsageRate = application.clusters().values().stream()
+                .map(cluster -> largestQuotaUsage(cluster.current(), cluster.max()))
+                .mapToDouble(resources -> resources.nodes() * resources.nodeResources().cost())
+                .sum();
+        return QuotaUsage.create(quotaUsageRate);
+    }
+
+    private static ClusterResources largestQuotaUsage(ClusterResources a, ClusterResources b) {
+        var usageA = a.nodes() * a.nodeResources().cost();
+        var usageB = b.nodes() * b.nodeResources().cost();
+        return usageA < usageB ? b : a;
+    }
+
     /** Just get the maximum quota we are allowed to use. */
     private static Quota getMaximumAllowedQuota(Quota tenantQuota, List<Application> applications,
                                                 ApplicationId application, ZoneId zone) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 709064c8715..25a6b119671 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.InstanceName;
-import java.util.logging.Level;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -31,6 +30,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalLong;
 import java.util.function.Supplier;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static java.util.Comparator.comparing;
@@ -189,8 +189,7 @@ public class DeploymentTrigger {
         Instance instance = application.require(applicationId.instance());
         DeploymentStatus status = jobs.deploymentStatus(application);
         JobId job = new JobId(instance.id(), jobType);
-        Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.systemVersion());
-        String reason = "Job triggered manually by " + user;
+        Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion());
         Map<JobId, List<Versions>> jobs = status.testJobs(Map.of(job, versions));
         if (jobs.isEmpty() || ! requireTests)
             jobs = Map.of(job, List.of(versions));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 6458c8e93b7..cb6e5d123ef 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -400,7 +400,7 @@ public class InternalStepRunner implements StepRunner {
     private Version testerPlatformVersion(RunId id) {
         return application(id.application()).change().isPinned()
                ? controller.jobController().run(id).get().versions().targetPlatform()
-               : controller.systemVersion();
+               : controller.readSystemVersion();
     }
 
     private Optional<RunStatus> installTester(RunId id, DualLogger logger) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 3bc9bc01a76..4e715908ec4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -41,8 +41,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Optional;
+import java.util.Queue;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -55,6 +57,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableList;
@@ -315,7 +318,7 @@ public class JobController {
 
     /** Returns the deployment status of the given application. */
     public DeploymentStatus deploymentStatus(Application application) {
-        return deploymentStatus(application, controller.systemVersion());
+        return deploymentStatus(application, controller.readSystemVersion());
     }
 
     private DeploymentStatus deploymentStatus(Application application, Version systemVersion) {
@@ -339,7 +342,7 @@ public class JobController {
 
     /** Adds deployment status to each of the given applications. Calling this will do an implicit read of the controller's version status */
     public DeploymentStatusList deploymentStatuses(ApplicationList applications) {
-        return deploymentStatuses(applications, controller.systemVersion());
+        return deploymentStatuses(applications, controller.readSystemVersion());
     }
 
     /** Changes the status of the given step, for the given run, provided it is still active. */
@@ -352,36 +355,50 @@ public class JobController {
         locked(id, run -> run.with(timestamp, step));
     }
 
-    /** Changes the status of the given run to inactive, and stores it as a historic run. */
-    public void finish(RunId id) {
-        locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails.
-            Run finishedRun = run.finished(controller.clock().instant());
-            locked(id.application(), id.type(), runs -> {
-                runs.put(run.id(), finishedRun);
-                long last = id.number();
-                long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count();
-                var oldEntries = runs.entrySet().iterator();
-                for (var old = oldEntries.next();
-                        old.getKey().number() <= last - historyLength
-                     || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge));
-                     old = oldEntries.next()) {
-
-                    // Make sure we keep the last success and the first failing
-                    if (     successes == 1
-                        &&   old.getValue().status() == RunStatus.success
-                        && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) {
-                        oldEntries.next();
-                        continue;
+    /**
+     * Changes the status of the given run to inactive, and stores it as a historic run.
+     * Throws TimeoutException if some step in this job is still being run.
+     */
+    public void finish(RunId id) throws TimeoutException {
+        List<Lock> locks = new ArrayList<>();
+        try {
+            // Ensure no step is still running before we finish the run — report depends transitively on all the other steps.
+            for (Step step : report.allPrerequisites())
+                locks.add(curator.lock(id.application(), id.type(), step));
+
+            locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails.
+                Run finishedRun = run.finished(controller.clock().instant());
+                locked(id.application(), id.type(), runs -> {
+                    runs.put(run.id(), finishedRun);
+                    long last = id.number();
+                    long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count();
+                    var oldEntries = runs.entrySet().iterator();
+                    for (var old = oldEntries.next();
+                         old.getKey().number() <= last - historyLength
+                         || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge));
+                         old = oldEntries.next()) {
+
+                        // Make sure we keep the last success and the first failing
+                        if (successes == 1
+                            && old.getValue().status() == RunStatus.success
+                            && !old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) {
+                            oldEntries.next();
+                            continue;
+                        }
+
+                        logs.delete(old.getKey());
+                        oldEntries.remove();
                     }
-
-                    logs.delete(old.getKey());
-                    oldEntries.remove();
-                }
+                });
+                logs.flush(id);
+                metric.jobFinished(run.id().job(), finishedRun.status());
+                return finishedRun;
             });
-            logs.flush(id);
-            metric.jobFinished(run.id().job(), finishedRun.status());
-            return finishedRun;
-        });
+        }
+        finally {
+            for (Lock lock : locks)
+                lock.close();
+        }
     }
 
     /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */
@@ -389,9 +406,7 @@ public class JobController {
         locked(id, run -> run.aborted());
     }
 
-    /**
-     * Accepts and stores a new application package and test jar pair under a generated application version key.
-     */
+    /** Accepts and stores a new application package and test jar pair under a generated application version key. */
     public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail,
                                      Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage,
                                      byte[] testPackageBytes) {
@@ -462,7 +477,7 @@ public class JobController {
                   type,
                   new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion()
                                                                  .flatMap(controller.applications()::lastCompatibleVersion)
-                                                                 .orElseGet(controller::systemVersion)),
+                                                                 .orElseGet(controller::readSystemVersion)),
                                ApplicationVersion.unknown,
                                Optional.empty(),
                                Optional.empty()),
@@ -588,7 +603,7 @@ public class JobController {
     /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */
     public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException {
         try (Lock lock = curator.lock(id, type, step)) {
-            for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running.
+            for (Step prerequisite : step.allPrerequisites()) // Check that no prerequisite is still running.
                 try (Lock __ = curator.lock(id, type, prerequisite)) { ; }
 
             action.accept(new LockedStep(lock, step));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index 17cfd1bdf1d..baba4771370 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -1,9 +1,11 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
-import com.google.common.collect.ImmutableList;
-
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toUnmodifiableList;
 
 /**
  * Steps that make up a deployment job. See {@link JobProfile} for preset profiles.
@@ -70,16 +72,28 @@ public enum Step {
 
     private final boolean alwaysRun;
     private final List<Step> prerequisites;
+    private final List<Step> allPrerequisites;
 
     Step(boolean alwaysRun, Step... prerequisites) {
         this.alwaysRun = alwaysRun;
-        this.prerequisites = ImmutableList.copyOf(prerequisites);
+        this.prerequisites = List.of(prerequisites);
+        this.allPrerequisites = Stream.concat(Stream.of(prerequisites),
+                                              Stream.of(prerequisites).flatMap(pre -> pre.allPrerequisites().stream()))
+                                      .sorted()
+                                      .distinct()
+                                      .collect(toUnmodifiableList());
     }
 
     /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */
     public boolean alwaysRun() { return alwaysRun; }
 
-    /** Returns the prerequisite steps that must be successfully completed before this, assuming the job contains these steps. */
+    /** Returns all prerequisite steps for this, recursively. */
+    public List<Step> allPrerequisites() {
+        return allPrerequisites;
+    }
+
+
+    /** Returns the direct prerequisite steps that must be completed before this, assuming the job contains these steps. */
     public List<Step> prerequisites() { return prerequisites; }
 
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
index a94e7407898..a3070ef55a0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
@@ -84,10 +85,11 @@ public class DeploymentIssueReporter extends ControllerMaintainer {
         boolean success = true;
         if (controller().system() == SystemName.cd)
             return success;
-        
-        Version systemVersion = controller().systemVersion();
 
-        if ((controller().versionStatus().version(systemVersion).confidence() != broken))
+        VersionStatus versionStatus = controller().readVersionStatus();
+        Version systemVersion = controller().systemVersion(versionStatus);
+
+        if (versionStatus.version(systemVersion).confidence() != broken)
             return success;
 
         DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index e5ee06f81dd..f08a23ab8ed 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
+import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner;
@@ -35,7 +36,7 @@ public class JobRunner extends ControllerMaintainer {
     private final StepRunner runner;
 
     public JobRunner(Controller controller, Duration duration) {
-        this(controller, duration, Executors.newFixedThreadPool(32), new InternalStepRunner(controller));
+        this(controller, duration, Executors.newFixedThreadPool(128, new DaemonThreadFactory("job-runner-")), new InternalStepRunner(controller));
     }
 
     @TestOnly
@@ -90,6 +91,9 @@ public class JobRunner extends ControllerMaintainer {
             controller().jobController().run(id)
                         .ifPresent(run -> controller().applications().deploymentTrigger().notifyOfCompletion(id.application()));
         }
+        catch (TimeoutException e) {
+            // One of the steps are still being run — that's ok, we'll try to finish the run again later.
+        }
         catch (Exception e) {
             log.log(Level.WARNING, "Exception finishing " + id, e);
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 0c5ef123eef..780eec47e81 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -19,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.deployment.JobList;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Clock;
@@ -51,6 +52,7 @@ public class MetricsReporter extends ControllerMaintainer {
     public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration";
     public static final String OS_NODE_COUNT = "deployment.nodeCountByOsVersion";
     public static final String PLATFORM_NODE_COUNT = "deployment.nodeCountByPlatformVersion";
+    public static final String BROKEN_SYSTEM_VERSION = "deployment.brokenSystemVersion";
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
     public static final String OPERATION_PREFIX = "operation.";
@@ -72,11 +74,20 @@ public class MetricsReporter extends ControllerMaintainer {
         reportDeploymentMetrics();
         reportRemainingRotations();
         reportQueuedNameServiceRequests();
-        reportInfrastructureUpgradeMetrics();
+        VersionStatus versionStatus = controller().readVersionStatus();
+        reportInfrastructureUpgradeMetrics(versionStatus);
         reportAuditLog();
+        reportBrokenSystemVersion(versionStatus);
         return true;
     }
 
+    private void reportBrokenSystemVersion(VersionStatus versionStatus) {
+        Version systemVersion = controller().systemVersion(versionStatus);
+        VespaVersion.Confidence confidence = versionStatus.version(systemVersion).confidence();
+        int isBroken = confidence == VespaVersion.Confidence.broken ? 1 : 0;
+        metric.set(BROKEN_SYSTEM_VERSION, isBroken, metric.createContext(Map.of()));
+    }
+
     private void reportAuditLog() {
         AuditLog log = controller().auditLogger().readLog();
         HashMap<String, HashMap<String, Integer>> metricCounts = new HashMap<>();
@@ -109,9 +120,9 @@ public class MetricsReporter extends ControllerMaintainer {
         }
     }
 
-    private void reportInfrastructureUpgradeMetrics() {
+    private void reportInfrastructureUpgradeMetrics(VersionStatus versionStatus) {
         Map<NodeVersion, Duration> osChangeDurations = osChangeDurations();
-        Map<NodeVersion, Duration> platformChangeDurations = platformChangeDurations();
+        Map<NodeVersion, Duration> platformChangeDurations = platformChangeDurations(versionStatus);
         reportChangeDurations(osChangeDurations, OS_CHANGE_DURATION);
         reportChangeDurations(platformChangeDurations, PLATFORM_CHANGE_DURATION);
         reportNodeCount(osChangeDurations.keySet(), OS_NODE_COUNT);
@@ -182,8 +193,8 @@ public class MetricsReporter extends ControllerMaintainer {
         });
     }
 
-    private Map<NodeVersion, Duration> platformChangeDurations() {
-        return changeDurations(controller().versionStatus().versions(), VespaVersion::nodeVersions);
+    private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) {
+        return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions);
     }
 
     private Map<NodeVersion, Duration> osChangeDurations() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index d9c78c8a442..b84e77a1d85 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -52,7 +52,7 @@ public class SystemUpgrader extends InfrastructureUpgrader<Version> {
 
     @Override
     protected Optional<Version> targetVersion() {
-        return controller().versionStatus().controllerVersion()
+        return controller().readVersionStatus().controllerVersion()
                            .filter(vespaVersion -> !vespaVersion.isSystemVersion())
                            .filter(vespaVersion -> vespaVersion.confidence() != VespaVersion.Confidence.broken)
                            .map(VespaVersion::versionNumber);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index 9ab2b0e77e8..5ac89cc54be 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.InstanceList;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 
@@ -53,12 +54,13 @@ public class Upgrader extends ControllerMaintainer {
     @Override
     public boolean maintain() {
         // Determine target versions for each upgrade policy
-        Version canaryTarget = controller().systemVersion();
-        Collection<Version> defaultTargets = targetVersions(Confidence.normal);
-        Collection<Version> conservativeTargets = targetVersions(Confidence.high);
+        VersionStatus versionStatus = controller().readVersionStatus();
+        Version canaryTarget = controller().systemVersion(versionStatus);
+        Collection<Version> defaultTargets = targetVersions(Confidence.normal, versionStatus);
+        Collection<Version> conservativeTargets = targetVersions(Confidence.high, versionStatus);
 
         // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation)
-        for (VespaVersion version : controller().versionStatus().versions()) {
+        for (VespaVersion version : versionStatus.versions()) {
             if (version.confidence() == Confidence.broken)
                 cancelUpgradesOf(instances().upgradingTo(version.versionNumber())
                                             .not().with(UpgradePolicy.canary),
@@ -93,16 +95,16 @@ public class Upgrader extends ControllerMaintainer {
     }
 
     /** Returns the target versions for given confidence, one per major version in the system */
-    private Collection<Version> targetVersions(Confidence confidence) {
-        return controller().versionStatus().versions().stream()
-                           // Ensure we never pick a version newer than the system
-                           .filter(v -> !v.versionNumber().isAfter(controller().systemVersion()))
-                           .filter(v -> v.confidence().equalOrHigherThan(confidence))
-                           .map(VespaVersion::versionNumber)
-                           .collect(Collectors.toMap(Version::getMajor, // Key on major version
-                                                     Function.identity(),  // Use version as value
-                                                     BinaryOperator.<Version>maxBy(naturalOrder()))) // Pick highest version when merging versions within this major
-                           .values();
+    private Collection<Version> targetVersions(Confidence confidence, VersionStatus versionStatus) {
+        return versionStatus.versions().stream()
+                            // Ensure we never pick a version newer than the system
+                            .filter(v -> !v.versionNumber().isAfter(controller().systemVersion(versionStatus)))
+                            .filter(v -> v.confidence().equalOrHigherThan(confidence))
+                            .map(VespaVersion::versionNumber)
+                            .collect(Collectors.toMap(Version::getMajor, // Key on major version
+                                                      Function.identity(),  // Use version as value
+                                                      BinaryOperator.<Version>maxBy(naturalOrder()))) // Pick highest version when merging versions within this major
+                            .values();
     }
 
     /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index 879b307733a..8b7590e88a9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -152,9 +152,9 @@ public class TenantSerializer {
     }
 
     TenantInfo tenantInfoFromSlime(Inspector infoObject) {
-        if (!infoObject.valid()) return TenantInfo.EmptyInfo;
+        if (!infoObject.valid()) return TenantInfo.EMPTY;
 
-        return TenantInfo.EmptyInfo
+        return TenantInfo.EMPTY
                 .withName(infoObject.field("name").asString())
                 .withEmail(infoObject.field("email").asString())
                 .withWebsite(infoObject.field("website").asString())
@@ -166,7 +166,7 @@ public class TenantSerializer {
     }
 
     private TenantInfoAddress tenantInfoAddressFromSlime(Inspector addressObject) {
-        return TenantInfoAddress.EmptyAddress
+        return TenantInfoAddress.EMPTY
                 .withAddressLines(addressObject.field("addressLines").asString())
                 .withPostalCodeOrZip(addressObject.field("postalCodeOrZip").asString())
                 .withCity(addressObject.field("city").asString())
@@ -175,7 +175,7 @@ public class TenantSerializer {
     }
 
     private TenantInfoBillingContact tenantInfoBillingContactFromSlime(Inspector billingObject) {
-        return TenantInfoBillingContact.EmptyBillingContact
+        return TenantInfoBillingContact.EMPTY
                 .withName(billingObject.field("name").asString())
                 .withEmail(billingObject.field("email").asString())
                 .withPhone(billingObject.field("phone").asString())
@@ -279,5 +279,4 @@ public class TenantSerializer {
             default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'.");
         }
     }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 1472ea3d001..7d4a5545c92 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1083,7 +1083,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
      */
     private Version compileVersion(TenantAndApplicationId id) {
         Version oldestPlatform = controller.applications().oldestInstalledPlatform(id);
-        VersionStatus versionStatus = controller.versionStatus();
+        VersionStatus versionStatus = controller.readVersionStatus();
         return versionStatus.versions().stream()
                             .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low))
                             .filter(VespaVersion::isReleased)
@@ -1317,16 +1317,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         StringBuilder response = new StringBuilder();
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             Version version = Version.fromString(versionString);
+            VersionStatus versionStatus = controller.readVersionStatus();
             if (version.equals(Version.emptyVersion))
-                version = controller.systemVersion();
-            if ( ! systemHasVersion(version))
+                version = controller.systemVersion(versionStatus);
+            if ( versionStatus.version(version) == null)
                 throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " +
                                                    "Version is not active in this system. " +
-                                                   "Active versions: " + controller.versionStatus().versions()
-                                                                                   .stream()
-                                                                                   .map(VespaVersion::versionNumber)
-                                                                                   .map(Version::toString)
-                                                                                   .collect(joining(", ")));
+                                                   "Active versions: " + versionStatus.versions()
+                                                                                      .stream()
+                                                                                      .map(VespaVersion::versionNumber)
+                                                                                      .map(Version::toString)
+                                                                                      .collect(joining(", ")));
             Change change = Change.of(version);
             if (pin)
                 change = change.withPin();
@@ -1437,10 +1438,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             }
             // To avoid second guessing the orchestrated upgrades of system applications
             // we don't allow to deploy these during an system upgrade (i.e when new vespa is being rolled out)
-            if (controller.versionStatus().isUpgrading()) {
+            VersionStatus versionStatus = controller.readVersionStatus();
+            if (versionStatus.isUpgrading()) {
                 throw new IllegalArgumentException("Deployment of system applications during a system upgrade is not allowed");
             }
-            Optional<VespaVersion> systemVersion = controller.versionStatus().systemVersion();
+            Optional<VespaVersion> systemVersion = versionStatus.systemVersion();
             if (systemVersion.isEmpty()) {
                 throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined");
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index afe2f5684e5..622a7033519 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -31,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunLog;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.net.URI;
@@ -289,6 +290,7 @@ class JobControllerApiHandlerHelper {
 
         Map<JobId, List<Versions>> jobsToRun = status.jobsToRun();
         Cursor stepsArray = responseObject.setArray("steps");
+        VersionStatus versionStatus = controller.readVersionStatus();
         for (DeploymentStatus.StepStatus stepStatus : status.allSteps()) {
             Change change = status.application().require(stepStatus.instance()).change();
             Cursor stepObject = stepsArray.addObject();
@@ -305,7 +307,7 @@ class JobControllerApiHandlerHelper {
                       .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
             stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli()));
             stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()));
-            stepStatus.blockedUntil(Change.of(controller.systemVersion())) // Dummy version — just anything with a platform.
+            stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform.
                       .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli()));
             application.latestVersion().map(Change::of).flatMap(stepStatus::blockedUntil) // Dummy version — just anything with an application.
                       .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli()));
@@ -322,7 +324,7 @@ class JobControllerApiHandlerHelper {
 
                 Cursor latestVersionsObject = stepObject.setObject("latestVersions");
                 List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker();
-                latestVersionWithCompatibleConfidenceAndNotNewerThanSystem(controller.versionStatus().versions(),
+                latestVersionWithCompatibleConfidenceAndNotNewerThanSystem(versionStatus.versions(),
                                                                            application.deploymentSpec().requireInstance(stepStatus.instance()).upgradePolicy())
                           .ifPresent(latestPlatform -> {
                               Cursor latestPlatformObject = latestVersionsObject.setObject("platform");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 2cdfbcda571..da13a84a3d4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
-import com.yahoo.component.Vtag;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
@@ -20,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
@@ -96,8 +94,8 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
         Cursor platformArray = root.setArray("versions");
-        var versionStatus = controller.versionStatus();
-        var systemVersion = versionStatus.systemVersion().map(VespaVersion::versionNumber).orElse(Vtag.currentVersion);
+        var versionStatus = controller.readVersionStatus();
+        var systemVersion = controller.systemVersion(versionStatus);
         var deploymentStatuses = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()), systemVersion);
         var deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toList()),
                                                                 deploymentStatuses)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index 114a2967e9a..ace176bd91e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -265,6 +265,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
                     // Include status from routing policies
                     var routingPolicies = controller.routing().policies().get(deploymentId);
                     for (var policy : routingPolicies.values()) {
+                        if (policy.endpoints().isEmpty()) continue; // This policy does not apply to a global endpoint
                         if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue;
                         deploymentStatusToSlime(deploymentsArray.addObject(), new DeploymentId(policy.id().owner(),
                                                                                                policy.id().zone()),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
index f8edeee5939..5682d8b69fe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
@@ -61,19 +61,12 @@ public class AthenzTenant extends Tenant {
     /** Create a new Athenz tenant */
     public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property,
                                       Optional<PropertyId> propertyId) {
-        return new AthenzTenant(requireName(requireNoPrefix(name)), domain, property, propertyId, Optional.empty());
+        return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty());
     }
 
     public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property,
                                       Optional<PropertyId> propertyId, Optional<Contact> contact) {
-        return new AthenzTenant(requireName(requireNoPrefix(name)), domain, property, propertyId, contact);
-    }
-
-    private static TenantName requireNoPrefix(TenantName name) {
-        if (name.value().startsWith(Tenant.userPrefix)) {
-            throw new IllegalArgumentException("Athenz tenant name cannot have prefix '" + Tenant.userPrefix + "'");
-        }
-        return name;
+        return new AthenzTenant(requireName(name), domain, property, propertyId, contact);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index 4ee866c752d..67b285bb24f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -33,7 +33,7 @@ public class CloudTenant extends Tenant {
     public static CloudTenant create(TenantName tenantName, Principal creator) {
         return new CloudTenant(requireName(tenantName),
                                Optional.ofNullable(creator),
-                               ImmutableBiMap.of(), TenantInfo.EmptyInfo);
+                               ImmutableBiMap.of(), TenantInfo.EMPTY);
     }
 
     /** The user that created the tenant */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
index bac43517f1a..b1dc0d8a5d5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
@@ -14,8 +14,6 @@ import java.util.Optional;
  */
 public abstract class Tenant {
 
-    public static final String userPrefix = "by-";
-
     private final TenantName name;
     private final Optional<Contact> contact;
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java
index 181d09cf451..81c08e1083b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java
@@ -1,3 +1,4 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.tenant;
 
 import java.util.Objects;
@@ -8,6 +9,8 @@ import java.util.Objects;
  * This info is used to capture generic support information and invoiced billing information.
  *
  * All fields are non null but strings can be empty
+ *
+ * @author smorgrav
  */
 public class TenantInfo {
     private final String name;
@@ -31,8 +34,8 @@ public class TenantInfo {
         this.billingContact = Objects.requireNonNull(billingContact);
     }
 
-    public static TenantInfo EmptyInfo = new TenantInfo("","","", "", "", "",
-            TenantInfoAddress.EmptyAddress, TenantInfoBillingContact.EmptyBillingContact);
+    public static final TenantInfo EMPTY = new TenantInfo("","","", "", "", "",
+            TenantInfoAddress.EMPTY, TenantInfoBillingContact.EMPTY);
 
     public String name() {
         return name;
@@ -99,9 +102,7 @@ public class TenantInfo {
     }
 
     public boolean isEmpty() {
-        return (name + email + website + contactEmail + contactName + invoiceEmail).isEmpty()
-                && address.isEmpty()
-                && billingContact.isEmpty();
+        return this.equals(EMPTY);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java
index 707c6fc7ac1..a12f351abd6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoAddress.java
@@ -1,3 +1,4 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.tenant;
 
 import java.util.Objects;
@@ -10,6 +11,8 @@ import java.util.Objects;
  * The address lines can be street address, P.O box, c/o name, apartment, suite, unit, building floor etc etc.
  *
  * All fields are mandatory but can be an empty string (ie. not null)
+ *
+ * @author smorgrav
  */
 public class TenantInfoAddress {
 
@@ -27,7 +30,7 @@ public class TenantInfoAddress {
         this.stateRegionProvince = Objects.requireNonNull(stateRegionProvince);
     }
 
-    public static TenantInfoAddress EmptyAddress = new TenantInfoAddress("","","", "", "");
+    public static final TenantInfoAddress EMPTY = new TenantInfoAddress("","","", "", "");
 
     public String addressLines() {
         return addressLines;
@@ -70,7 +73,7 @@ public class TenantInfoAddress {
     }
 
     public boolean isEmpty() {
-        return (addressLines + postalCodeOrZip + city + country + stateRegionProvince).isEmpty();
+        return this.equals(EMPTY);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java
index e63d361b513..a00dd626f0a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfoBillingContact.java
@@ -1,7 +1,11 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.tenant;
 
 import java.util.Objects;
 
+/**
+ * @author smorgrav
+ */
 public class TenantInfoBillingContact {
     private final String name;
     private final String email;
@@ -15,8 +19,8 @@ public class TenantInfoBillingContact {
         this.address = Objects.requireNonNull(address);
     }
 
-    public static TenantInfoBillingContact EmptyBillingContact =
-                new TenantInfoBillingContact("","", "", TenantInfoAddress.EmptyAddress);
+    public static final TenantInfoBillingContact EMPTY =
+                new TenantInfoBillingContact("","", "", TenantInfoAddress.EMPTY);
 
     public String name() {
         return name;
@@ -49,7 +53,7 @@ public class TenantInfoBillingContact {
     }
 
     public boolean isEmpty() {
-        return (name + email + phone).isEmpty() && address.isEmpty();
+        return this.equals(EMPTY);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
index 4ce0a35e96f..e2791c0bfe5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
@@ -1,9 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 
@@ -31,15 +29,6 @@ public class NodeVersions {
         return nodeVersions;
     }
 
-    /** Returns host names in this, grouped by version */
-    public ListMultimap<Version, HostName> asVersionMap() {
-        var versions = ImmutableListMultimap.<Version, HostName>builder();
-        for (var kv : nodeVersions.entrySet()) {
-            versions.put(kv.getValue().currentVersion(), kv.getKey());
-        }
-        return versions.build();
-    }
-
     /** Returns host names in this */
     public Set<HostName> hostnames() {
         return nodeVersions.keySet();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index fffdd431be1..022ccbe266c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -1,9 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -14,8 +11,10 @@ import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
@@ -36,11 +35,11 @@ public class VersionStatus {
 
     private static final Logger log = Logger.getLogger(VersionStatus.class.getName());
 
-    private final ImmutableList<VespaVersion> versions;
+    private final List<VespaVersion> versions;
     
     /** Create a version status. DO NOT USE: Public for testing and serialization only */
     public VersionStatus(List<VespaVersion> versions) {
-        this.versions = ImmutableList.copyOf(versions);
+        this.versions = List.copyOf(versions);
     }
 
     /** Returns the current version of controllers in this system */
@@ -77,28 +76,33 @@ public class VersionStatus {
     }
 
     /** Create the empty version status */
-    public static VersionStatus empty() { return new VersionStatus(ImmutableList.of()); }
+    public static VersionStatus empty() { return new VersionStatus(List.of()); }
 
     /** Create a full, updated version status. This is expensive and should be done infrequently */
     public static VersionStatus compute(Controller controller) {
-        var systemApplicationVersions = findSystemApplicationVersions(controller);
-        var controllerVersions = findControllerVersions(controller);
-
-        var infrastructureVersions = ArrayListMultimap.<Version, HostName>create();
-        for (var kv : controllerVersions.asMap().entrySet()) {
-            infrastructureVersions.putAll(kv.getKey().version(), kv.getValue());
+        VersionStatus versionStatus = controller.readVersionStatus();
+        NodeVersions systemApplicationVersions = findSystemApplicationVersions(controller, versionStatus);
+        Map<ControllerVersion, List<HostName>> controllerVersions = findControllerVersions(controller);
+
+        Map<Version, List<HostName>> infrastructureVersions = new HashMap<>();
+        for (var kv : controllerVersions.entrySet()) {
+            infrastructureVersions.computeIfAbsent(kv.getKey().version(), (k) -> new ArrayList<>())
+                                  .addAll(kv.getValue());
+        }
+        for (var kv : systemApplicationVersions.asMap().entrySet()) {
+            infrastructureVersions.computeIfAbsent(kv.getValue().currentVersion(), (k) -> new ArrayList<>())
+                                  .add(kv.getKey());
         }
-        infrastructureVersions.putAll(systemApplicationVersions.asVersionMap());
 
         // The system version is the oldest infrastructure version, if that version is newer than the current system
         // version
         Version newSystemVersion = infrastructureVersions.keySet().stream().min(Comparator.naturalOrder()).get();
-        Version systemVersion = controller.versionStatus().systemVersion()
-                                          .map(VespaVersion::versionNumber)
-                                          .orElse(newSystemVersion);
+        Version systemVersion = versionStatus.systemVersion()
+                                             .map(VespaVersion::versionNumber)
+                                             .orElse(newSystemVersion);
         if (newSystemVersion.isBefore(systemVersion)) {
             log.warning("Refusing to lower system version from " +
-                        controller.systemVersion() +
+                        systemVersion +
                         " to " +
                         newSystemVersion +
                         ", nodes on " + newSystemVersion + ": " +
@@ -110,9 +114,9 @@ public class VersionStatus {
         }
 
 
-        var deploymentStatistics = DeploymentStatistics.compute(infrastructureVersions.keySet(),
-                                                                controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())
-                                                                                                                            .withProjectId()));
+        List<DeploymentStatistics> deploymentStatistics = DeploymentStatistics.compute(infrastructureVersions.keySet(),
+                                                                                       controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())
+                                                                                                                                                    .withProjectId()));
         List<VespaVersion> versions = new ArrayList<>();
         List<Version> releasedVersions = controller.mavenRepository().metadata().versions();
 
@@ -126,7 +130,8 @@ public class VersionStatus {
                                                           systemVersion,
                                                           isReleased,
                                                           systemApplicationVersions.matching(statistics.version()),
-                                                          controller);
+                                                          controller,
+                                                          versionStatus);
                 versions.add(vespaVersion);
             } catch (IllegalArgumentException e) {
                 log.log(Level.WARNING, "Unable to create VespaVersion for version " +
@@ -139,7 +144,7 @@ public class VersionStatus {
         return new VersionStatus(versions);
     }
 
-    private static NodeVersions findSystemApplicationVersions(Controller controller) {
+    private static NodeVersions findSystemApplicationVersions(Controller controller, VersionStatus versionStatus) {
         var nodeVersions = new LinkedHashMap<HostName, NodeVersion>();
         for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
             for (var application : SystemApplication.all()) {
@@ -148,14 +153,14 @@ public class VersionStatus {
                                       .filter(SystemUpgrader::eligibleForUpgrade)
                                       .collect(Collectors.toList());
                 if (nodes.isEmpty()) continue;
-                var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
+                boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
                 if (!configConverged) {
                     log.log(Level.WARNING, "Config for " + application.id() + " in " + zone.getId() +
                                               " has not converged");
                 }
                 for (var node : nodes) {
                     // Only use current node version if config has converged
-                    var version = configConverged ? node.currentVersion() : controller.systemVersion();
+                    var version = configConverged ? node.currentVersion() : controller.systemVersion(versionStatus);
                     var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(),
                                                       node.suspendedSince());
                     nodeVersions.put(nodeVersion.hostname(), nodeVersion);
@@ -165,14 +170,16 @@ public class VersionStatus {
         return NodeVersions.copyOf(nodeVersions);
     }
 
-    private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) {
-        ListMultimap<ControllerVersion, HostName> versions = ArrayListMultimap.create();
+    private static Map<ControllerVersion, List<HostName>> findControllerVersions(Controller controller) {
+        Map<ControllerVersion, List<HostName>> versions = new HashMap<>();
         if (controller.curator().cluster().isEmpty()) { // Use vtag if we do not have cluster
-            versions.put(ControllerVersion.CURRENT, controller.hostname());
+            versions.computeIfAbsent(ControllerVersion.CURRENT, (k) -> new ArrayList<>())
+                    .add(controller.hostname());
         } else {
             for (String host : controller.curator().cluster()) {
                 HostName hostname = HostName.from(host);
-                versions.put(controller.curator().readControllerVersion(hostname), hostname);
+                versions.computeIfAbsent(controller.curator().readControllerVersion(hostname), (k) -> new ArrayList<>())
+                        .add(hostname);
             }
         }
         return versions;
@@ -183,14 +190,15 @@ public class VersionStatus {
                                               Version systemVersion,
                                               boolean isReleased,
                                               NodeVersions nodeVersions,
-                                              Controller controller) {
-        var latestVersion = controllerVersions.stream().max(Comparator.naturalOrder()).get();
-        var controllerVersion = controllerVersions.stream().min(Comparator.naturalOrder()).get();
-        var isSystemVersion = statistics.version().equals(systemVersion);
-        var isControllerVersion = statistics.version().equals(controllerVersion.version());
-        var confidence = controller.curator().readConfidenceOverrides().get(statistics.version());
-        var confidenceIsOverridden = confidence != null;
-        var previousStatus = controller.versionStatus().version(statistics.version());
+                                              Controller controller,
+                                              VersionStatus versionStatus) {
+        ControllerVersion latestVersion = controllerVersions.stream().max(Comparator.naturalOrder()).get();
+        ControllerVersion controllerVersion = controllerVersions.stream().min(Comparator.naturalOrder()).get();
+        boolean isSystemVersion = statistics.version().equals(systemVersion);
+        boolean isControllerVersion = statistics.version().equals(controllerVersion.version());
+        VespaVersion.Confidence confidence = controller.curator().readConfidenceOverrides().get(statistics.version());
+        boolean confidenceIsOverridden = confidence != null;
+        VespaVersion existingVespaVersion = versionStatus.version(statistics.version());
 
         // Compute confidence
         if (!confidenceIsOverridden) {
@@ -199,21 +207,25 @@ public class VersionStatus {
                 confidence = VespaVersion.confidenceFrom(statistics, controller);
             } else {
                 // This is an older version so we preserve the existing confidence, if any
-                confidence = getOrUpdateConfidence(statistics, controller);
+                confidence = versionStatus.versions().stream()
+                                          .filter(v -> statistics.version().equals(v.versionNumber()))
+                                          .map(VespaVersion::confidence)
+                                          .findFirst()
+                                          .orElseGet(() -> VespaVersion.confidenceFrom(statistics, controller));
             }
         }
 
         // Preserve existing commit details if we've previously computed status for this version
         var commitSha = latestVersion.commitSha();
         var commitDate = latestVersion.commitDate();
-        if (previousStatus != null) {
-            commitSha = previousStatus.releaseCommit();
-            commitDate = previousStatus.committedAt();
+        if (existingVespaVersion != null) {
+            commitSha = existingVespaVersion.releaseCommit();
+            commitDate = existingVespaVersion.committedAt();
 
             // Keep existing confidence if we cannot raise it at this moment in time
             if (!confidenceIsOverridden &&
-                !previousStatus.confidence().canChangeTo(confidence, controller.clock().instant())) {
-                confidence = previousStatus.confidence();
+                !existingVespaVersion.confidence().canChangeTo(confidence, controller.clock().instant())) {
+                confidence = existingVespaVersion.confidence();
             }
         }
 
@@ -227,17 +239,4 @@ public class VersionStatus {
                                 confidence);
     }
 
-    /**
-     * Calculate confidence from given deployment statistics.
-     *
-     * @return previously calculated confidence for this version. If none exists, a new confidence will be calculated.
-     */
-    private static VespaVersion.Confidence getOrUpdateConfidence(DeploymentStatistics statistics, Controller controller) {
-        return controller.versionStatus().versions().stream()
-                         .filter(v -> statistics.version().equals(v.versionNumber()))
-                         .map(VespaVersion::confidence)
-                         .findFirst()
-                         .orElseGet(() -> VespaVersion.confidenceFrom(statistics, controller));
-    }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
index 675cb2f3f76..fd719ab4619 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
@@ -1,11 +1,20 @@
 package com.yahoo.vespa.hosted.controller.application;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.io.IOUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationData;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 
 import static org.junit.Assert.*;
@@ -48,4 +57,12 @@ public class DeploymentQuotaCalculatorTest {
         assertEquals(calculated.budget().get().doubleValue(), 0, 1e-5);
     }
 
+    @Test
+    public void using_highest_resource_use() throws IOException, URISyntaxException {
+        var content = new String(Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/application/response/application.json")));
+        var mapper = new ObjectMapper();
+        var application = mapper.readValue(content, ApplicationData.class).toApplication();
+        var usage = DeploymentQuotaCalculator.calculateQuotaUsage(application);
+        assertEquals(1.164, usage.rate(), 0.001);
+    }
 }
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/response/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/response/application.json
new file mode 100644
index 00000000000..ccfb6af1635
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/response/application.json
@@ -0,0 +1,156 @@
+{
+  "url": "...",
+  "id": "vespa.album-recommendation.default",
+  "clusters": {
+    "default": {
+      "min": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "max": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "current": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 2.0,
+          "memoryGb": 8.0,
+          "diskGb": 50.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "remote"
+        }
+      },
+      "suggested": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 2.0,
+          "memoryGb": 8.0,
+          "diskGb": 75.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "local"
+        }
+      }
+    },
+    "logserver": {
+      "min": {
+        "nodes": 1,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "max": {
+        "nodes": 1,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "current": {
+        "nodes": 1,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.5,
+          "memoryGb": 4.0,
+          "diskGb": 50.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "remote"
+        }
+      },
+      "suggested": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 2.0,
+          "memoryGb": 4.0,
+          "diskGb": 50.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "local"
+        }
+      }
+    },
+    "music": {
+      "min": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "max": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 0.0,
+          "memoryGb": 0.0,
+          "diskGb": 0.0,
+          "bandwidthGbps": 0.0,
+          "diskSpeed": "fast",
+          "storageType": "any"
+        }
+      },
+      "current": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 2.0,
+          "memoryGb": 8.0,
+          "diskGb": 50.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "remote"
+        }
+      },
+      "suggested": {
+        "nodes": 2,
+        "groups": 1,
+        "resources": {
+          "vcpu": 2.0,
+          "memoryGb": 4.0,
+          "diskGb": 50.0,
+          "bandwidthGbps": 0.3,
+          "diskSpeed": "fast",
+          "storageType": "local"
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index d90eb715499..a42dbe7fbde 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -337,7 +337,7 @@ public class DeploymentContext {
         return this;
     }
 
-    /** Abort the running job of the given type and. */
+    /** Abort the running job of the given type. */
     public DeploymentContext abortJob(JobType type) {
         var job = jobId(type);
         assertNotSame(RunStatus.aborted, currentRun(job).status());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 41d10015411..7fd02a8e780 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -101,6 +101,7 @@ public class DeploymentTester {
 
     public OutstandingChangeDeployer outstandingChangeDeployer() { return outstandingChangeDeployer; }
 
+    /** A tester with clock configured to a time when confidence can freely change */
     public DeploymentTester atMondayMorning() {
         return at(tester.clock().instant().atZone(ZoneOffset.UTC)
                         .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY))
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 6e5a9ddc7ab..a38f71214b5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -629,7 +629,7 @@ public class DeploymentTriggerTest {
                 .environment(Environment.prod)
                 .region("us-west-1")
                 .build();
-        Version version1 = tester.controller().versionStatus().systemVersion().get().versionNumber();
+        Version version1 = tester.controller().readSystemVersion();
         var app1 = tester.newDeploymentContext();
 
         // First deployment: An application change
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 02640cf8486..a58a12a1861 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -202,9 +202,9 @@ public class InternalStepRunnerTest {
 
         // Node is down too long in system test, and no nodes go down in staging.
         tester.runner().run();
-        tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.systemTest.zone(system()));
+        tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.systemTest.zone(system()));
         tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
-        tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.stagingTest.zone(system()));
+        tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.stagingTest.zone(system()));
         tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
index dcb7a6dd42b..4a739ce2722 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
@@ -147,11 +147,11 @@ public class DeploymentIssueReporterTest {
         Version version = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(version);
         tester.upgrader().maintain();
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readSystemVersion());
 
         app2.timeOutUpgrade(systemTest);
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         assertFalse("We have no platform issues initially.", issues.platformIssue());
         reporter.maintain();
@@ -165,7 +165,7 @@ public class DeploymentIssueReporterTest {
 
         app2.runJob(systemTest);
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(VespaVersion.Confidence.low, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.low, tester.controller().readVersionStatus().systemVersion().get().confidence());
     }
 
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 88eab642a60..f677cc079cc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -370,7 +370,7 @@ public class JobRunnerTest {
     }
 
     @Test
-    public void jobMetrics() {
+    public void jobMetrics() throws TimeoutException {
         DeploymentTester tester = new DeploymentTester();
         JobController jobs = tester.controller().jobController();
         Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index ab274dc5ef8..062bd97f901 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -13,11 +13,13 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -339,7 +341,7 @@ public class MetricsReporterTest {
             upgradeTo(version, hosts.subList(1, hosts.size()), zone, tester);
             runAll(tester::computeVersionStatus, reporter);
             assertPlatformChangeDuration(Duration.ZERO, hosts);
-            assertEquals(version, tester.controller().systemVersion());
+            assertEquals(version, tester.controller().readSystemVersion());
             assertPlatformNodeCount(hosts.size(), version);
         }
     }
@@ -445,6 +447,39 @@ public class MetricsReporterTest {
         }
     }
 
+    @Test
+    public void broken_system_version() {
+        var tester = new DeploymentTester().atMondayMorning();
+        var ctx = tester.newDeploymentContext();
+        var applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").region("us-west-1").build();
+
+        // Application deploys successfully on current system version
+        ctx.submit(applicationPackage).deploy();
+        tester.controllerTester().computeVersionStatus();
+        var reporter = createReporter(tester.controller());
+        reporter.maintain();
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
+        assertEquals(0, metrics.getMetric(MetricsReporter.BROKEN_SYSTEM_VERSION));
+
+        // System upgrades. Canary upgrade fails
+        Version version0 = Version.fromString("6.2");
+        tester.controllerTester().upgradeSystem(version0);
+        tester.upgrader().maintain();
+        assertEquals(Change.of(version0), ctx.instance().change());
+        ctx.failDeployment(stagingTest);
+        tester.controllerTester().computeVersionStatus();
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
+        reporter.maintain();
+        assertEquals(1, metrics.getMetric(MetricsReporter.BROKEN_SYSTEM_VERSION));
+
+        // Canary is healed and confidence is raised
+        ctx.deployPlatform(version0);
+        tester.controllerTester().computeVersionStatus();
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
+        reporter.maintain();
+        assertEquals(0, metrics.getMetric(MetricsReporter.BROKEN_SYSTEM_VERSION));
+    }
+
     private void assertNodeCount(String metric, int n, Version version) {
         long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric)
                                 .stream()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 98412b8147b..ba6da2a02b8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -295,7 +295,7 @@ public class SystemUpgraderTest {
         systemUpgrader.maintain();
         assertWantedVersion(SystemApplication.proxy, Version.emptyVersion, zone1);
         tester.computeVersionStatus();
-        assertEquals(version1, tester.controller().systemVersion());
+        assertEquals(version1, tester.controller().readSystemVersion());
     }
 
     /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */
@@ -333,11 +333,11 @@ public class SystemUpgraderTest {
     }
 
     private void assertSystemVersion(Version version) {
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readSystemVersion());
     }
 
     private void assertControllerVersion(Version version) {
-        assertEquals(version, tester.controller().versionStatus().controllerVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().controllerVersion().get().versionNumber());
     }
 
     private void assertWantedVersion(SystemApplication application, Version version, ZoneApi first, ZoneApi... rest) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 522d44ed667..eb8e154ee0c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -70,7 +70,7 @@ public class UpgraderTest {
         // --- Next version released - everything goes smoothly
         Version version1 = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(version1);
-        assertEquals(version1, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version1, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -86,7 +86,7 @@ public class UpgraderTest {
         canary1.deployPlatform(version1);
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         assertEquals("Canaries done: Should upgrade defaults", 6, tester.jobs().active().size());
@@ -96,7 +96,7 @@ public class UpgraderTest {
         default2.deployPlatform(version1);
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         assertEquals("Normals done: Should upgrade conservatives", 2, tester.jobs().active().size());
@@ -110,7 +110,7 @@ public class UpgraderTest {
         // --- Next version released - which fails a Canary
         Version version2 = Version.fromString("6.4");
         tester.controllerTester().upgradeSystem(version2);
-        assertEquals(version2, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version2, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -119,7 +119,7 @@ public class UpgraderTest {
         canary0.failDeployment(stagingTest);
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         assertEquals("Version broken, but Canaries should keep trying", 3, tester.jobs().active().size());
@@ -127,7 +127,7 @@ public class UpgraderTest {
         // --- Next version released - which repairs the Canary app and fails a default
         Version version3 = Version.fromString("6.5");
         tester.controllerTester().upgradeSystem(version3);
-        assertEquals(version3, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version3, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         canary0.abortJob(stagingTest);
         canary1.abortJob(systemTest);
@@ -146,7 +146,7 @@ public class UpgraderTest {
         canary1.deployPlatform(version3);
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -159,7 +159,7 @@ public class UpgraderTest {
 
         tester.controllerTester().computeVersionStatus();
         assertEquals("Not enough evidence to mark this as neither broken nor high",
-                     VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+                     VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         tester.triggerJobs();
         assertEquals("Upgrade with error should retry", 1, tester.jobs().active().size());
@@ -172,7 +172,7 @@ public class UpgraderTest {
         default0.deploy();
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         assertEquals("Normals done: Should upgrade conservatives", 2, tester.jobs().active().size());
@@ -193,7 +193,7 @@ public class UpgraderTest {
         canary0.deployPlatform(version4);
         canary1.deployPlatform(version4);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -211,7 +211,7 @@ public class UpgraderTest {
         canary0.deployPlatform(version5);
         canary1.deployPlatform(version5);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -245,7 +245,7 @@ public class UpgraderTest {
                 .runJob(stagingTest)
                 .failDeployment(productionUsWest1);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
 
         tester.upgrader().maintain();
@@ -286,7 +286,7 @@ public class UpgraderTest {
         // --- A new version is released
         version = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -302,7 +302,7 @@ public class UpgraderTest {
         canary1.deployPlatform(version);
 
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         assertEquals("Canaries done: Should upgrade defaults", 20, tester.jobs().active().size());
@@ -316,7 +316,7 @@ public class UpgraderTest {
         tester.upgrader().maintain();
         tester.abortAll();
         tester.triggerJobs();
-        assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
         assertEquals("Upgrades are cancelled", 0, tester.jobs().active().size());
     }
 
@@ -343,7 +343,7 @@ public class UpgraderTest {
         // V1 is released
         Version v1 = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(v1);
-        assertEquals(v1, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(v1, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -351,14 +351,14 @@ public class UpgraderTest {
         canary0.deployPlatform(v1);
         canary1.deployPlatform(v1);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
         // V2 is released
         Version v2 = Version.fromString("6.4");
         tester.controllerTester().upgradeSystem(v2);
-        assertEquals(v2, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(v2, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -366,7 +366,7 @@ public class UpgraderTest {
         canary0.deployPlatform(v2);
         canary1.deployPlatform(v2);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         // We "manually" cancel upgrades to V1 so that we can use the applications to make V2 fail instead
         // But we keep one (default4) to avoid V1 being garbage collected
@@ -391,7 +391,7 @@ public class UpgraderTest {
         default2.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1).failDeployment(productionUsEast3);
         default3.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1).failDeployment(productionUsEast3);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.broken, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         assertEquals(v2, default0.deployment(ZoneId.from("prod.us-west-1")).version());
         assertEquals(v0, default0.deployment(ZoneId.from("prod.us-east-3")).version());
@@ -428,7 +428,7 @@ public class UpgraderTest {
         // New version is released
         version = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -436,7 +436,7 @@ public class UpgraderTest {
         canary0.deployPlatform(version);
         canary1.deployPlatform(version);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.normal, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         // All applications upgrade successfully
         tester.upgrader().maintain();
@@ -444,7 +444,7 @@ public class UpgraderTest {
         for (var context : List.of(default0, default1, default2, default3, default4))
             context.deployPlatform(version);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
 
         // Multiple application changes are triggered and fail, but does not affect version confidence as upgrade has
         // completed successfully
@@ -453,7 +453,7 @@ public class UpgraderTest {
         default2.submit(applicationPackage("default")).failDeployment(systemTest);
         default3.submit(applicationPackage("default")).failDeployment(stagingTest);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+        assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence());
     }
 
     @Test
@@ -635,7 +635,7 @@ public class UpgraderTest {
         // New version is released and canaries upgrade
         version = Version.fromString("6.3");
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         upgrader.maintain();
         tester.triggerJobs();
 
@@ -675,7 +675,7 @@ public class UpgraderTest {
         version = Version.fromString("7.0");
         tester.controllerTester().upgradeSystem(version);
         tester.upgrader().maintain();
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.triggerJobs();
 
         // ... canary upgrade to it
@@ -705,7 +705,7 @@ public class UpgraderTest {
         // New major version is released
         version = Version.fromString("7.0");
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
@@ -746,7 +746,7 @@ public class UpgraderTest {
         tester.upgrader().setTargetMajorVersion(Optional.of(6));
         version = Version.fromString("7.0");
         tester.controllerTester().upgradeSystem(version);
-        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
         tester.triggerJobs();
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java
index d287c025b42..0d2390f9d92 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java
@@ -26,12 +26,12 @@ public class VersionStatusUpdaterTest {
     public void testVersionUpdating() {
         ControllerTester tester = new ControllerTester();
         tester.controller().updateVersionStatus(new VersionStatus(Collections.emptyList()));
-        assertFalse(tester.controller().versionStatus().systemVersion().isPresent());
+        assertFalse(tester.controller().readVersionStatus().systemVersion().isPresent());
 
         VersionStatusUpdater updater = new VersionStatusUpdater(tester.controller(), Duration.ofDays(1)
         );
         updater.maintain();
-        assertTrue(tester.controller().versionStatus().systemVersion().isPresent());
+        assertTrue(tester.controller().readVersionStatus().systemVersion().isPresent());
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index af2c0f43d94..20caceee097 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -85,7 +85,7 @@ public class TenantSerializerTest {
                                              Optional.of(new SimplePrincipal("foobar-user")),
                                              ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"),
                                                                otherPublicKey, new SimplePrincipal("jane")),
-                                             TenantInfo.EmptyInfo);
+                                             TenantInfo.EMPTY);
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.name(), serialized.name());
         assertEquals(tenant.creator(), serialized.creator());
@@ -93,9 +93,21 @@ public class TenantSerializerTest {
     }
 
     @Test
+    public void cloud_tenant_with_info() {
+        CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"),
+                Optional.of(new SimplePrincipal("foobar-user")),
+                ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"),
+                        otherPublicKey, new SimplePrincipal("jane")),
+                TenantInfo.EMPTY.withName("Ofni Tnanet"));
+        CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
+        assertEquals(tenant.info(), serialized.info());
+    }
+
+
+    @Test
     public void cloud_tenant_with_tenant_info_partial() {
-        TenantInfo partialInfo = TenantInfo.EmptyInfo
-                .withAddress(TenantInfoAddress.EmptyAddress.withCity("Hønefoss"));
+        TenantInfo partialInfo = TenantInfo.EMPTY
+                .withAddress(TenantInfoAddress.EMPTY.withCity("Hønefoss"));
 
         Slime slime = new Slime();
         Cursor parentObject = slime.setObject();
@@ -105,24 +117,24 @@ public class TenantSerializerTest {
 
     @Test
     public void cloud_tenant_with_tenant_info_full() {
-        TenantInfo fullInfo = TenantInfo.EmptyInfo
+        TenantInfo fullInfo = TenantInfo.EMPTY
                 .withName("My Company")
                 .withEmail("email@mycomp.any")
                 .withWebsite("http://mycomp.any")
                 .withContactEmail("ceo@mycomp.any")
                 .withContactName("My Name")
                 .withInvoiceEmail("invoice@mycomp.any")
-                .withAddress(TenantInfoAddress.EmptyAddress
+                .withAddress(TenantInfoAddress.EMPTY
                         .withCity("Hønefoss")
                         .withAddressLines("Riperbakken 2")
                         .withCountry("Norway")
                         .withPostalCodeOrZip("3510")
                         .withStateRegionProvince("Viken"))
-                .withBillingContact(TenantInfoBillingContact.EmptyBillingContact
+                .withBillingContact(TenantInfoBillingContact.EMPTY
                         .withEmail("thomas@sodor.com")
                         .withName("Thomas The Tank Engine")
                         .withPhone("NA")
-                        .withAddress(TenantInfoAddress.EmptyAddress
+                        .withAddress(TenantInfoAddress.EMPTY
                                 .withCity("Suddery")
                                 .withCountry("Sodor")
                                 .withAddressLines("Central Station")
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index dfe2930e2a8..8e74542ec23 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -954,7 +954,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .userIdentity(HOSTED_VESPA_OPERATOR),
                               "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Deployment of system applications during a system upgrade is not allowed\"}",
                               400);
-        deploymentTester.controllerTester().upgradeSystem(deploymentTester.controller().versionStatus().controllerVersion().get().versionNumber());
+        deploymentTester.controllerTester().upgradeSystem(deploymentTester.controller().readVersionStatus().controllerVersion().get().versionNumber());
         tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST)
                         .data(noAppEntity)
                         .userIdentity(HOSTED_VESPA_OPERATOR),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
index 5a7bbc07267..26c2e2d1175 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
@@ -11,7 +11,6 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
@@ -70,7 +69,7 @@ public class SignatureFilterTest {
         tester.curator().writeTenant(new CloudTenant(appId.tenant(),
                                                      Optional.empty(),
                                                      ImmutableBiMap.of(),
-                                                     TenantInfo.EmptyInfo));
+                                                     TenantInfo.EMPTY));
         tester.curator().writeApplication(new Application(appId, tester.clock().instant()));
     }
 
@@ -110,7 +109,7 @@ public class SignatureFilterTest {
         tester.curator().writeTenant(new CloudTenant(appId.tenant(),
                                                      Optional.empty(),
                                                      ImmutableBiMap.of(publicKey, () -> "user"),
-                                                     TenantInfo.EmptyInfo));
+                                                     TenantInfo.EMPTY));
         verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
                               new SecurityContext(new SimplePrincipal("user"),
                                                   Set.of(Role.reader(id.tenant()),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
index b0549662ab0..9bd8485db77 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.restapi.routing;
 
 import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.zone.RoutingMethod;
@@ -184,6 +185,21 @@ public class RoutingApiTest extends ControllerContainerTest {
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/zone-status-in.json"));
+
+        // Endpoint is removed
+        applicationPackage = new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
+                .region(westZone.region())
+                .region(eastZone.region())
+                .allow(ValidationId.globalEndpointChange)
+                .build();
+        context.submit(applicationPackage).deploy();
+
+        // GET deployment status. Now empty as no routing policies have global endpoints
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
+                                              "", Request.Method.GET),
+                              "{\"deployments\":[]}");
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 3ea9d038d99..a4c8bb8ef6d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -109,7 +109,7 @@ public class VersionStatusTest {
         ControllerTester tester = new ControllerTester();
         Version version0 = Version.fromString("6.2");
         tester.upgradeSystem(version0);
-        assertEquals(version0, tester.controller().systemVersion());
+        assertEquals(version0, tester.controller().readSystemVersion());
 
         // Downgrade one config server in each zone
         Version ancientVersion = Version.fromString("5.1");
@@ -122,7 +122,7 @@ public class VersionStatusTest {
         }
 
         tester.computeVersionStatus();
-        assertEquals(version0, tester.controller().systemVersion());
+        assertEquals(version0, tester.controller().readSystemVersion());
     }
 
     @Test
@@ -161,7 +161,7 @@ public class VersionStatusTest {
 
         tester.triggerJobs();
         tester.controllerTester().computeVersionStatus();
-        List<VespaVersion> versions = tester.controller().versionStatus().versions();
+        List<VespaVersion> versions = tester.controller().readVersionStatus().versions();
         assertEquals("The two versions above exist", 2, versions.size());
 
         VespaVersion v1 = versions.get(0);
@@ -336,8 +336,8 @@ public class VersionStatusTest {
         assertEquals("All canaries deployed + < 90% of defaults: Normal",
                      Confidence.normal, confidence(tester.controller(), version2));
         assertTrue("Status for version without applications is removed",
-                   tester.controller().versionStatus().versions().stream()
-                           .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1)));
+                   tester.controller().readVersionStatus().versions().stream()
+                         .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1)));
 
         // Another default application upgrades, raising confidence to high
         default8.deployPlatform(version2);
@@ -375,7 +375,7 @@ public class VersionStatusTest {
                      VespaVersion.Confidence.broken, confidence(tester.controller(), version3));
 
         // Test version order
-        List<VespaVersion> versions = tester.controller().versionStatus().versions();
+        List<VespaVersion> versions = tester.controller().readVersionStatus().versions();
         assertEquals(List.of("6.2", "6.4", "6.5"), versions.stream().map(version -> version.versionNumber().toString()).collect(Collectors.toList()));
 
         // Check release status is correct (static data in MockMavenRepository).
@@ -426,7 +426,7 @@ public class VersionStatusTest {
         canary0.abortJob(stagingTest);
         tester.controllerTester().computeVersionStatus();
         assertFalse("Previous version should be forgotten, as canary only had test jobs run on it",
-                    tester.controller().versionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version1)));
+                    tester.controller().readVersionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version1)));
 
         // App succeeds with tests, but fails production deployment
         canary0.runJob(systemTest)
@@ -459,7 +459,7 @@ public class VersionStatusTest {
         canary0.runJob(productionUsWest1);
         tester.controllerTester().computeVersionStatus();
         assertFalse("Previous version should be forgotten, as canary only had test jobs run on it",
-                    tester.controller().versionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version2)));
+                    tester.controller().readVersionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version2)));
         assertEquals("Canary OK, but not done upgrading, so confidence for version3: Low",
                      Confidence.low, confidence(tester.controller(), version3));
     }
@@ -510,9 +510,9 @@ public class VersionStatusTest {
         var commitSha0 = "badc0ffee";
         var commitDate0 = Instant.EPOCH;
         tester.controllerTester().upgradeSystem(version0);
-        assertEquals(version0, tester.controller().versionStatus().systemVersion().get().versionNumber());
-        assertEquals(commitSha0, tester.controller().versionStatus().systemVersion().get().releaseCommit());
-        assertEquals(commitDate0, tester.controller().versionStatus().systemVersion().get().committedAt());
+        assertEquals(version0, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
+        assertEquals(commitSha0, tester.controller().readVersionStatus().systemVersion().get().releaseCommit());
+        assertEquals(commitDate0, tester.controller().readVersionStatus().systemVersion().get().committedAt());
 
         // Deploy app on version0 to keep computing statistics for that version
         tester.newDeploymentContext().submit().deploy();
@@ -523,13 +523,13 @@ public class VersionStatusTest {
         var commitDate1 = Instant.ofEpochMilli(123);
         tester.controllerTester().upgradeController(version1, commitSha1, commitDate1);
         tester.controllerTester().upgradeSystemApplications(version1);
-        assertEquals(version1, tester.controller().versionStatus().systemVersion().get().versionNumber());
-        assertEquals(commitSha1, tester.controller().versionStatus().systemVersion().get().releaseCommit());
-        assertEquals(commitDate1, tester.controller().versionStatus().systemVersion().get().committedAt());
+        assertEquals(version1, tester.controller().readVersionStatus().systemVersion().get().versionNumber());
+        assertEquals(commitSha1, tester.controller().readVersionStatus().systemVersion().get().releaseCommit());
+        assertEquals(commitDate1, tester.controller().readVersionStatus().systemVersion().get().committedAt());
 
         // Commit details for previous version are preserved
-        assertEquals(commitSha0, tester.controller().versionStatus().version(version0).releaseCommit());
-        assertEquals(commitDate0, tester.controller().versionStatus().version(version0).committedAt());
+        assertEquals(commitSha0, tester.controller().readVersionStatus().version(version0).releaseCommit());
+        assertEquals(commitDate0, tester.controller().readVersionStatus().version(version0).committedAt());
     }
 
     @Test
@@ -548,7 +548,7 @@ public class VersionStatusTest {
                             .submit(new ApplicationPackageBuilder().upgradePolicy("default").region("us-west-1").build())
                             .deploy();
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.high, tester.controller().versionStatus().version(version0).confidence());
+        assertSame(Confidence.high, tester.controller().readVersionStatus().version(version0).confidence());
 
         // System and canary0 is upgraded within allowed time window
         Version version1 = Version.fromString("7.2");
@@ -556,25 +556,25 @@ public class VersionStatusTest {
         tester.upgrader().maintain();
         canary0.deployPlatform(version1);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.low, tester.controller().versionStatus().version(version1).confidence());
+        assertSame(Confidence.low, tester.controller().readVersionStatus().version(version1).confidence());
 
         // canary1 breaks just outside allowed upgrade window
         assertEquals(12, tester.controllerTester().hourOfDayAfter(Duration.ofHours(7)));
         canary1.failDeployment(systemTest);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.broken, tester.controller().versionStatus().version(version1).confidence());
+        assertSame(Confidence.broken, tester.controller().readVersionStatus().version(version1).confidence());
 
         // Second canary is fixed later in the day. All canaries are now fixed, but confidence is not raised as we're
         // outside the allowed time window
         assertEquals(20, tester.controllerTester().hourOfDayAfter(Duration.ofHours(8)));
         canary1.deployPlatform(version1);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.broken, tester.controller().versionStatus().version(version1).confidence());
+        assertSame(Confidence.broken, tester.controller().readVersionStatus().version(version1).confidence());
 
         // Early morning arrives, confidence is raised and normal application upgrades
         assertEquals(5, tester.controllerTester().hourOfDayAfter(Duration.ofHours(9)));
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.normal, tester.controller().versionStatus().version(version1).confidence());
+        assertSame(Confidence.normal, tester.controller().readVersionStatus().version(version1).confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         default0.deployPlatform(version1);
@@ -587,21 +587,21 @@ public class VersionStatusTest {
         canary0.deployPlatform(version2);
         canary1.deployPlatform(version2);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.low, tester.controller().versionStatus().version(version2).confidence());
+        assertSame(Confidence.low, tester.controller().readVersionStatus().version(version2).confidence());
 
         // Confidence override takes precedence over time window constraints
         tester.upgrader().overrideConfidence(version2, Confidence.normal);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.normal, tester.controller().versionStatus().version(version2).confidence());
+        assertSame(Confidence.normal, tester.controller().readVersionStatus().version(version2).confidence());
         tester.upgrader().overrideConfidence(version2, Confidence.low);
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.low, tester.controller().versionStatus().version(version2).confidence());
+        assertSame(Confidence.low, tester.controller().readVersionStatus().version(version2).confidence());
         tester.upgrader().removeConfidenceOverride(version2);
 
         // Next morning arrives, confidence is raised and normal application upgrades
         assertEquals(7, tester.controllerTester().hourOfDayAfter(Duration.ofHours(17)));
         tester.controllerTester().computeVersionStatus();
-        assertSame(Confidence.normal, tester.controller().versionStatus().version(version2).confidence());
+        assertSame(Confidence.normal, tester.controller().readVersionStatus().version(version2).confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
         default0.deployPlatform(version2);
@@ -650,7 +650,7 @@ public class VersionStatusTest {
         // Upgrade succeeds
         context.deployPlatform(version2);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(1, tester.controller().versionStatus().versions().size());
+        assertEquals(1, tester.controller().readVersionStatus().versions().size());
         assertOnVersion(version2, context.instanceId(), tester);
 
         // System is upgraded and application starts upgrading to next version
@@ -663,14 +663,14 @@ public class VersionStatusTest {
                .runJob(stagingTest)
                .failDeployment(productionUsWest1);
         tester.controllerTester().computeVersionStatus();
-        assertEquals(2, tester.controller().versionStatus().versions().size());
+        assertEquals(2, tester.controller().readVersionStatus().versions().size());
         for (var version : List.of(version2, version3)) {
             assertOnVersion(version, context.instanceId(), tester);
         }
     }
 
     private void assertOnVersion(Version version, ApplicationId instance, DeploymentTester tester) {
-        var vespaVersion = tester.controller().versionStatus().version(version);
+        var vespaVersion = tester.controller().readVersionStatus().version(version);
         assertNotNull("Statistics for version " + version + " exist", vespaVersion);
         var statistics = DeploymentStatistics.compute(List.of(version), tester.deploymentStatuses()).get(0);
         assertTrue("Application is on version " + version,
@@ -683,11 +683,11 @@ public class VersionStatusTest {
     }
 
     private Confidence confidence(Controller controller, Version version) {
-        return controller.versionStatus().versions().stream()
-                .filter(v -> v.versionNumber().equals(version))
-                .findFirst()
-                .map(VespaVersion::confidence)
-                .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version));
+        return controller.readVersionStatus().versions().stream()
+                         .filter(v -> v.versionNumber().equals(version))
+                         .findFirst()
+                         .map(VespaVersion::confidence)
+                         .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version));
     }
 
 }