Merge with master

60 files changed, 1153 insertions, 675 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index c17ac044136..c83f366cb67 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
+import java.security.PublicKey;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Comparator;
@@ -51,7 +52,7 @@ public class Application {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final Set<String> pemDeployKeys;
+    private final Set<PublicKey> deployKeys;
     private final Map<InstanceName, Instance> instances;
 
     /** Creates an empty application. */
@@ -64,7 +65,7 @@ public class Application {
     // DO NOT USE! For serialization purposes, only.
     public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        Change change, Change outstandingChange, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                       OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                       OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
                        OptionalLong projectId, boolean internal, Collection<Instance> instances) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
@@ -77,7 +78,7 @@ public class Application {
         this.owner = Objects.requireNonNull(owner, "owner cannot be null");
         this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null");
         this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
-        this.pemDeployKeys = Objects.requireNonNull(pemDeployKeys, "pemDeployKeys cannot be null");
+        this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null");
         this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null");
         this.internal = internal;
         this.instances = ImmutableSortedMap.copyOf(instances.stream().collect(Collectors.toMap(Instance::name, Function.identity())));
@@ -191,7 +192,7 @@ public class Application {
     }
 
     /** Returns the set of deploy keys for this application. */
-    public Set<String> pemDeployKeys() { return pemDeployKeys; }
+    public Set<PublicKey> deployKeys() { return deployKeys; }
 
     @Override
     public boolean equals(Object o) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 6f64237b2c4..0cf0f59102e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -2,12 +2,12 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
@@ -149,7 +150,6 @@ public class ApplicationController {
 
         // Update serialization format of all applications
         Once.after(Duration.ofMinutes(1), () -> {
-            curator.deleteOldApplicationData();
             Instant start = clock.instant();
             int count = 0;
             for (Application application : curator.readApplications()) {
@@ -213,6 +213,14 @@ public class ApplicationController {
 
     public ApplicationStore applicationStore() {  return applicationStore; }
 
+    /** Returns all content clusters in all current deployments of the given application. */
+    public Map<ZoneId, List<String>> contentClustersByZone(ApplicationId id, Iterable<ZoneId> zones) {
+        ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
+        for (ZoneId zone : zones)
+            clusters.put(zone, ImmutableList.copyOf(configServer.getContentClusters(new DeploymentId(id, zone))));
+        return clusters.build();
+    }
+
     /** Returns the oldest Vespa version installed on any active or reserved production node for the given application. */
     public Version oldestInstalledPlatform(TenantAndApplicationId id) {
         return requireApplication(id).instances().values().stream()
@@ -263,70 +271,86 @@ public class ApplicationController {
      *
      * @throws IllegalArgumentException if the application already exists
      */
-    // TODO jonmv: split in create application and create instance
-    public Application createApplication(ApplicationId id, Optional<Credentials> credentials) {
-        if (id.instance().isTester())
-            throw new IllegalArgumentException("'" + id + "' is a tester application!");
-        try (Lock lock = lock(TenantAndApplicationId.from(id))) {
-            // Validate only application names which do not already exist.
-            if (getApplication(TenantAndApplicationId.from(id)).isEmpty())
-                com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value());
+    public Application createApplication(TenantAndApplicationId id, Optional<Credentials> credentials) {
+        try (Lock lock = lock(id)) {
+            if (getApplication(id).isPresent())
+                throw new IllegalArgumentException("Could not create '" + id + "': Application already exists");
+            if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945
+                throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists");
+
+            com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value());
 
             Optional<Tenant> tenant = controller.tenants().get(id.tenant());
             if (tenant.isEmpty())
                 throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist");
-            if (getInstance(id).isPresent())
-                throw new IllegalArgumentException("Could not create '" + id + "': Application already exists");
-            if (getInstance(dashToUnderscore(id)).isPresent()) // VESPA-1945
-                throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists");
             if (tenant.get().type() != Tenant.Type.user) {
                 if (credentials.isEmpty())
                     throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided");
-
-                if ( ! id.instance().isTester()) // Only store the application permits for non-user applications.
-                    accessControl.createApplication(id, credentials.get());
+                accessControl.createApplication(id, credentials.get());
             }
-            Application application = getApplication(TenantAndApplicationId.from(id)).orElse(new Application(TenantAndApplicationId.from(id),
-                                                                                                             clock.instant()));
-            LockedApplication locked = new LockedApplication(application, lock).withNewInstance(id.instance());
+
+            LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock);
             store(locked);
             log.info("Created " + locked);
             return locked.get();
         }
     }
 
+    /**
+     * Creates a new instance for an existing application.
+     *
+     * @throws IllegalArgumentException if the instance already exists, or has an invalid instance name.
+     */
+    public void createInstance(ApplicationId id) {
+        if (id.instance().isTester())
+            throw new IllegalArgumentException("'" + id + "' is a tester application!");
+        lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
+            InstanceId.validate(id.instance().value());
+
+            if (getInstance(id).isPresent())
+                throw new IllegalArgumentException("Could not create '" + id + "': Instance already exists");
+            if (getInstance(dashToUnderscore(id)).isPresent()) // VESPA-1945
+                throw new IllegalArgumentException("Could not create '" + id + "': Instance " + dashToUnderscore(id) + " already exists");
+
+            store(application.withNewInstance(id.instance()));
+            log.info("Created " + id);
+        });
+    }
+
     public ActivateResult deploy(ApplicationId applicationId, ZoneId zone,
                                  Optional<ApplicationPackage> applicationPackageFromDeployer,
                                  DeployOptions options) {
-        return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options, Optional.empty());
+        return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options);
     }
 
     /** Deploys an application. If the application does not exist it is created. */
     // TODO: Get rid of the options arg
-    // TODO(jvenstad): Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter.
-    public ActivateResult deploy(ApplicationId applicationId, ZoneId zone,
+    // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter.
+    public ActivateResult deploy(ApplicationId instanceId, ZoneId zone,
                                  Optional<ApplicationPackage> applicationPackageFromDeployer,
                                  Optional<ApplicationVersion> applicationVersionFromDeployer,
-                                 DeployOptions options,
-                                 Optional<Principal> deployingIdentity) {
-        if (applicationId.instance().isTester())
-            throw new IllegalArgumentException("'" + applicationId + "' is a tester application!");
-
-        // TODO jonmv: Change this to create instances on demand.
-        Tenant tenant = controller.tenants().require(applicationId.tenant());
-        if (tenant.type() == Tenant.Type.user && getInstance(applicationId).isEmpty())
+                                 DeployOptions options) {
+        if (instanceId.instance().isTester())
+            throw new IllegalArgumentException("'" + instanceId + "' is a tester application!");
+
+        TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId);
+        if (   getApplication(applicationId).isEmpty()
+            && controller.tenants().require(instanceId.tenant()).type() == Tenant.Type.user)
             createApplication(applicationId, Optional.empty());
 
-        try (Lock deploymentLock = lockForDeployment(applicationId, zone)) {
+        if (getInstance(instanceId).isEmpty())
+            createInstance(instanceId);
+
+        try (Lock deploymentLock = lockForDeployment(instanceId, zone)) {
             Version platformVersion;
             ApplicationVersion applicationVersion;
             ApplicationPackage applicationPackage;
             Set<ContainerEndpoint> endpoints;
             Optional<ApplicationCertificate> applicationCertificate;
 
-            try (Lock lock = lock(TenantAndApplicationId.from(applicationId))) {
-                LockedApplication application = new LockedApplication(requireApplication(TenantAndApplicationId.from(applicationId)), lock);
-                InstanceName instance = applicationId.instance();
+            try (Lock lock = lock(applicationId)) {
+                LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
+                InstanceName instance = instanceId.instance();
 
                 boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed();
                 boolean preferOldestVersion = options.deployCurrentVersion;
@@ -348,25 +372,22 @@ public class ApplicationController {
                     if (   job.isEmpty()
                         || job.get().lastTriggered().isEmpty()
                         || job.get().lastCompleted().isPresent() && job.get().lastCompleted().get().at().isAfter(job.get().lastTriggered().get().at()))
-                        return unexpectedDeployment(applicationId, zone);
+                        return unexpectedDeployment(instanceId, zone);
                     JobRun triggered = job.get().lastTriggered().get();
                     platformVersion = preferOldestVersion ? triggered.sourcePlatform().orElse(triggered.platform())
                                                           : triggered.platform();
                     applicationVersion = preferOldestVersion ? triggered.sourceApplication().orElse(triggered.application())
                                                              : triggered.application();
 
-                    applicationPackage = getApplicationPackage(applicationId, application.get().internal(), applicationVersion);
-                    applicationPackage = withTesterCertificate(applicationPackage, applicationId, jobType);
+                    applicationPackage = getApplicationPackage(instanceId, application.get().internal(), applicationVersion);
+                    applicationPackage = withTesterCertificate(applicationPackage, instanceId, jobType);
                     validateRun(application.get(), instance, zone, platformVersion, applicationVersion);
                 }
 
-                // TODO jonmv: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...).
-                verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity);
-
                 if (zone.environment().isProduction()) // Assign and register endpoints
                     application = withRotation(applicationPackage.deploymentSpec(), application, instance);
 
-                endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(applicationId.instance()), zone);
+                endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone);
 
                 if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
                     // Provisions a new certificate if missing
@@ -385,11 +406,11 @@ public class ApplicationController {
 
             // Carry out deployment without holding the application lock.
             options = withVersion(platformVersion, options);
-            ActivateResult result = deploy(applicationId, applicationPackage, zone, options, endpoints,
+            ActivateResult result = deploy(instanceId, applicationPackage, zone, options, endpoints,
                                            applicationCertificate.orElse(null));
 
-            lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application ->
-                    store(application.with(applicationId.instance(),
+            lockApplicationOrThrow(applicationId, application ->
+                    store(application.with(instanceId.instance(),
                                            instance -> instance.withNewDeployment(zone, applicationVersion, platformVersion,
                                                                                   clock.instant(), warningsFrom(result)))));
             return result;
@@ -702,22 +723,24 @@ public class ApplicationController {
      *
      * @throws IllegalArgumentException if the application has deployments or the caller is not authorized
      */
-    public void deleteApplication(TenantName tenantName, ApplicationName applicationName, Optional<Credentials> credentials) {
-        Tenant tenant = controller.tenants().require(tenantName);
+    public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) {
+        Tenant tenant = controller.tenants().require(id.tenant());
         if (tenant.type() != Tenant.Type.user && credentials.isEmpty())
-            throw new IllegalArgumentException("Could not delete application '" + tenantName + "." + applicationName + "': No credentials provided");
+            throw new IllegalArgumentException("Could not delete application '" + id + "': No credentials provided");
 
         // Find all instances of the application
-        TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
         List<ApplicationId> instances = requireApplication(id).instances().keySet().stream()
                                                               .map(id::instance)
                                                               .collect(Collectors.toUnmodifiableList());
         if (instances.size() > 1)
             throw new IllegalArgumentException("Could not delete application; more than one instance present: " + instances);
 
-        // TODO: Make this one transaction when database is moved to ZooKeeper
         for (ApplicationId instance : instances)
-            deleteInstance(instance, credentials);
+            deleteInstance(instance);
+
+        if (tenant.type() != Tenant.Type.user)
+            accessControl.deleteApplication(id, credentials.get());
+        curator.removeApplication(id);
     }
 
     /**
@@ -726,24 +749,20 @@ public class ApplicationController {
      * @throws IllegalArgumentException if the application has deployments or the caller is not authorized
      * @throws NotExistsException if the instance does not exist
      */
-    public void deleteInstance(ApplicationId applicationId, Optional<Credentials> credentials) {
-        Tenant tenant = controller.tenants().require(applicationId.tenant());
-        if (tenant.type() != Tenant.Type.user && credentials.isEmpty())
-                throw new IllegalArgumentException("Could not delete application '" + applicationId + "': No credentials provided");
-
-        if (getInstance(applicationId).isEmpty())
-            throw new NotExistsException("Could not delete application '" + applicationId + "': Application not found");
+    public void deleteInstance(ApplicationId instanceId) {
+        if (getInstance(instanceId).isEmpty())
+            throw new NotExistsException("Could not delete instance '" + instanceId + "': Instance not found");
 
-        lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application -> {
-            if ( ! application.get().require(applicationId.instance()).deployments().isEmpty())
+        lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> {
+            if ( ! application.get().require(instanceId.instance()).deployments().isEmpty())
                 throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments in: " +
-                                                   application.get().require(applicationId.instance()).deployments().keySet().stream().map(ZoneId::toString)
+                                                   application.get().require(instanceId.instance()).deployments().keySet().stream().map(ZoneId::toString)
                                                               .sorted().collect(Collectors.joining(", ")));
 
-            applicationStore.removeAll(applicationId);
-            applicationStore.removeAll(TesterId.of(applicationId));
+            applicationStore.removeAll(instanceId);
+            applicationStore.removeAll(TesterId.of(instanceId));
 
-            Instance instance = application.get().require(applicationId.instance());
+            Instance instance = application.get().require(instanceId.instance());
             instance.rotations().forEach(assignedRotation -> {
                 var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId());
                 endpoints.asList().stream()
@@ -752,15 +771,10 @@ public class ApplicationController {
                              controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal);
                          });
             });
-            curator.storeWithoutInstance(application.without(applicationId.instance()).get());
+            curator.writeApplication(application.without(instanceId.instance()).get());
 
-            log.info("Deleted " + application);
+            log.info("Deleted " + instanceId);
         });
-
-
-        if (tenant.type() != Tenant.Type.user && getApplication(applicationId).isEmpty())
-            // TODO jonmv: Implementations ignore the instance — refactor to provide tenant and application names only.
-            accessControl.deleteApplication(applicationId, credentials.get());
     }
 
     /**
@@ -845,10 +859,12 @@ public class ApplicationController {
 
     public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; }
 
+    private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) {
+        return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_"));
+    }
+
     private ApplicationId dashToUnderscore(ApplicationId id) {
-        return ApplicationId.from(id.tenant().value(),
-                                  id.application().value().replaceAll("-", "_"),
-                                  id.instance().value());
+        return dashToUnderscore(TenantAndApplicationId.from(id)).instance(id.instance());
     }
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 5aa5a8e13de..19921595dc2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -11,11 +11,10 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 
+import java.security.PublicKey;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -43,7 +42,7 @@ public class LockedApplication {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final Set<String> pemDeployKeys;
+    private final Set<PublicKey> deployKeys;
     private final OptionalLong projectId;
     private final boolean internal;
     private final Map<InstanceName, Instance> instances;
@@ -58,14 +57,14 @@ public class LockedApplication {
         this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(),
              application.deploymentSpec(), application.validationOverrides(), application.change(),
              application.outstandingChange(), application.deploymentIssueId(), application.ownershipIssueId(),
-             application.owner(), application.majorVersion(), application.metrics(), application.pemDeployKeys(),
+             application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(),
              application.projectId(), application.internal(), application.instances());
     }
 
     private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec,
                               ValidationOverrides validationOverrides, Change change, Change outstandingChange,
                               Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                              OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                              OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
                               OptionalLong projectId, boolean internal,
                               Map<InstanceName, Instance> instances) {
         this.lock = lock;
@@ -80,7 +79,7 @@ public class LockedApplication {
         this.owner = owner;
         this.majorVersion = majorVersion;
         this.metrics = metrics;
-        this.pemDeployKeys = pemDeployKeys;
+        this.deployKeys = deployKeys;
         this.projectId = projectId;
         this.internal = internal;
         this.instances = Map.copyOf(instances);
@@ -89,7 +88,7 @@ public class LockedApplication {
     /** Returns a read-only copy of this */
     public Application get() {
         return new Application(id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                               deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                               deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                projectId, internal, instances.values());
     }
 
@@ -97,7 +96,7 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.put(instance, new Instance(id.instance(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -105,7 +104,7 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.put(instance, modification.apply(instances.get(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -113,61 +112,61 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.remove(instance);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withBuiltInternally(boolean builtInternally) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, builtInternally, instances);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withChange(Change change) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOutstandingChange(Change outstandingChange) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOwner(User owner) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -176,25 +175,25 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner,
                                      majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion),
-                                     metrics, pemDeployKeys, projectId, internal, instances);
+                                     metrics, deployKeys, projectId, internal, instances);
     }
 
     public LockedApplication with(ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
-    public LockedApplication withPemDeployKey(String pemDeployKey) {
-        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
+    public LockedApplication withDeployKey(PublicKey pemDeployKey) {
+        Set<PublicKey> keys = new LinkedHashSet<>(deployKeys);
         keys.add(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
                                      projectId, internal, instances);
     }
 
-    public LockedApplication withoutPemDeployKey(String pemDeployKey) {
-        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
+    public LockedApplication withoutDeployKey(PublicKey pemDeployKey) {
+        Set<PublicKey> keys = new LinkedHashSet<>(deployKeys);
         keys.remove(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
index ecc8bd65b72..6caf716aed4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
@@ -2,8 +2,10 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
@@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.Optional;
 
 import static java.util.Objects.requireNonNull;
@@ -126,44 +129,39 @@ public abstract class LockedTenant {
     public static class Cloud extends LockedTenant {
 
         private final BillingInfo billingInfo;
-        private final BiMap<String, Principal> pemDeveloperKeys;
+        private final BiMap<PublicKey, Principal> developerKeys;
 
-        private Cloud(TenantName name, BillingInfo billingInfo, BiMap<String, Principal> pemDeveloperKeys) {
+        private Cloud(TenantName name, BillingInfo billingInfo, BiMap<PublicKey, Principal> developerKeys) {
             super(name);
             this.billingInfo = billingInfo;
-            this.pemDeveloperKeys = pemDeveloperKeys;
+            this.developerKeys = ImmutableBiMap.copyOf(developerKeys);
         }
 
         private Cloud(CloudTenant tenant) {
-            this(tenant.name(), tenant.billingInfo(), tenant.pemDeveloperKeys());
+            this(tenant.name(), tenant.billingInfo(), tenant.developerKeys());
         }
 
         @Override
         public CloudTenant get() {
-            return new CloudTenant(name, billingInfo, pemDeveloperKeys);
+            return new CloudTenant(name, billingInfo, developerKeys);
         }
 
         public Cloud with(BillingInfo billingInfo) {
-            return new Cloud(name, billingInfo, pemDeveloperKeys);
-        }
-
-        public Cloud withPemDeveloperKey(String pemKey, Principal principal) {
-            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-            pemDeveloperKeys.forEach((key, user) -> {
-                if ( ! user.equals(principal))
-                    keys.put(key, user);
-            });
-            keys.put(pemKey, principal);
-            return new Cloud(name, billingInfo, keys.build());
-        }
-
-        public Cloud withoutPemDeveloperKey(String pemKey) {
-            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-            pemDeveloperKeys.forEach((key, user) -> {
-                if ( ! key.equals(pemKey))
-                    keys.put(key, user);
-            });
-            return new Cloud(name, billingInfo, keys.build());
+            return new Cloud(name, billingInfo, developerKeys);
+        }
+
+        public Cloud withDeveloperKey(PublicKey key, Principal principal) {
+            BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys);
+            if (keys.containsKey(key))
+                throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key));
+            keys.put(key, principal);
+            return new Cloud(name, billingInfo, keys);
+        }
+
+        public Cloud withoutDeveloperKey(PublicKey key) {
+            BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys);
+            keys.remove(key);
+            return new Cloud(name, billingInfo, keys);
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
index 9df918e3f20..5ff564f7ad3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
@@ -63,7 +63,7 @@ public enum SystemApplication {
                          .orElse(false);
     }
 
-    /** Returns the node types of this that should receive OS upgrades */
+    /** Returns whether this should receive OS upgrades */
     public boolean isEligibleForOsUpgrades() {
         return nodeType.isDockerHost();
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java
index 0b537535315..b4f0d6e2487 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java
@@ -48,6 +48,10 @@ public class TenantAndApplicationId implements Comparable<TenantAndApplicationId
         return instance(InstanceName.defaultName());
     }
 
+    public ApplicationId instance(String instance) {
+        return instance(InstanceName.from(instance));
+    }
+
     public ApplicationId instance(InstanceName instance) {
         return ApplicationId.from(tenant, application, instance);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
index 91f9e2d56d7..304a47044a1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.athenz.impl;
 
 import com.google.inject.Inject;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.log.LogLevel;
@@ -15,10 +14,12 @@ import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.api.OktaAccessToken;
 import com.yahoo.vespa.athenz.client.zms.RoleAction;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
+import com.yahoo.vespa.athenz.client.zms.ZmsClientException;
 import com.yahoo.vespa.athenz.client.zts.ZtsClient;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.security.AccessControl;
 import com.yahoo.vespa.hosted.controller.security.AthenzCredentials;
 import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec;
@@ -142,7 +143,7 @@ public class AthenzFacade implements AccessControl {
     }
 
     @Override
-    public void createApplication(ApplicationId id, Credentials credentials) {
+    public void createApplication(TenantAndApplicationId id, Credentials credentials) {
         AthenzCredentials athenzCredentials = (AthenzCredentials) credentials;
         createApplication(athenzCredentials.domain(), id.application(), athenzCredentials.token());
     }
@@ -152,11 +153,19 @@ public class AthenzFacade implements AccessControl {
         log("createProviderResourceGroup(" +
             "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)",
             domain, service.getDomain().getName(), service.getName(), application, tenantRoleActions);
-        zmsClient.createProviderResourceGroup(domain, service, application.value(), tenantRoleActions, token);
+        try {
+            zmsClient.createProviderResourceGroup(domain, service, application.value(), tenantRoleActions, token);
+        }
+        catch (ZmsClientException e) {
+            if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN)
+                throw new ForbiddenException("Not authorized to create application", e);
+            else
+                throw e;
+        }
     }
 
     @Override
-    public void deleteApplication(ApplicationId id, Credentials credentials) {
+    public void deleteApplication(TenantAndApplicationId id, Credentials credentials) {
         AthenzCredentials athenzCredentials = (AthenzCredentials) credentials;
         log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)",
             athenzCredentials.domain(), service.getDomain().getName(), service.getName(), id.application());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 1828a189cad..50af8bd8611 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -1,8 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
@@ -468,7 +466,7 @@ public class InternalStepRunner implements StepRunner {
                                                       testConfigSerializer.configJson(id.application(),
                                                                                       id.type(),
                                                                                       endpoints,
-                                                                                      listClusters(id.application(), zones)));
+                                                                                      controller.applications().contentClustersByZone(id.application(), zones)));
         return Optional.of(running);
     }
 
@@ -690,14 +688,6 @@ public class InternalStepRunner implements StepRunner {
         throw new IllegalStateException("No step deploys to the zone this run is for!");
     }
 
-    /** Returns all content clusters in all current deployments of the given real application. */
-    private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) {
-        ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
-        for (ZoneId zone : zones)
-            clusters.put(zone, ImmutableList.copyOf(controller.serviceRegistry().configServer().getContentClusters(new DeploymentId(id, zone))));
-        return clusters.build();
-    }
-
     /** Returns the generated services.xml content for the tester application. */
     static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, boolean useTesterCertificate,
                               NodeResources resources) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 0ecce359a02..54b5c339159 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.net.URI;
 import java.security.cert.X509Certificate;
@@ -351,12 +352,22 @@ public class JobController {
 
     /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */
     public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) {
+        if ( ! type.environment().isManuallyDeployed())
+            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
+
+        if (   controller.tenants().require(id.tenant()).type() == Tenant.Type.user
+            && controller.applications().getApplication(TenantAndApplicationId.from(id)).isEmpty())
+            controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty());
+
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             if ( ! application.get().internal())
-                controller.applications().store(registered(application));
+                application = registered(application);
+
+            if ( ! application.get().instances().containsKey(id.instance()))
+                application = application.withNewInstance(id.instance());
+
+            controller.applications().store(application);
         });
-        if ( ! type.environment().isManuallyDeployed())
-            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
 
         last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
         locked(id, type, __ -> {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 4e13a1c25e5..1743cad32e4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -67,7 +67,7 @@ public class ControllerMaintenance extends AbstractComponent {
         deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl);
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, controller.serviceRegistry().ownershipIssues());
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
-        jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl);
+        jobRunner = new JobRunner(controller, Duration.ofSeconds(90), jobControl);
         osUpgraders = osUpgraders(controller, jobControl);
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 79ababd20d3..9253e249765 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -1,7 +1,6 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.google.common.collect.ImmutableMap;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -13,17 +12,19 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobList;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
+ * This calculates and reports system-wide metrics based on data from a {@link Controller}.
+ *
  * @author mortent
  * @author mpolden
  */
@@ -34,9 +35,12 @@ public class MetricsReporter extends Maintainer {
     public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades";
     public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds";
     public static final String DEPLOYMENT_WARNINGS = "deployment.warnings";
+    public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade";
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
 
+    private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofHours(1);
+
     private final Metric metric;
     private final Clock clock;
 
@@ -51,12 +55,13 @@ public class MetricsReporter extends Maintainer {
         reportDeploymentMetrics();
         reportRemainingRotations();
         reportQueuedNameServiceRequests();
+        reportNodesFailingSystemUpgrade();
     }
 
     private void reportRemainingRotations() {
         try (RotationLock lock = controller().applications().rotationRepository().lock()) {
             int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size();
-            metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Collections.emptyMap()));
+            metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of()));
         }
     }
 
@@ -66,7 +71,7 @@ public class MetricsReporter extends Maintainer {
                                                   .flatMap(application -> application.instances().values().stream())
                                                   .collect(Collectors.toUnmodifiableList());
 
-        metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Collections.emptyMap()));
+        metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Map.of()));
 
         averageDeploymentDurations(instances, clock.instant()).forEach((application, duration) -> {
             metric.set(DEPLOYMENT_AVERAGE_DURATION, duration.getSeconds(), metric.createContext(dimensions(application)));
@@ -93,6 +98,24 @@ public class MetricsReporter extends Maintainer {
         metric.set(NAME_SERVICE_REQUESTS_QUEUED, controller().curator().readNameServiceQueue().requests().size(),
                    metric.createContext(Map.of()));
     }
+
+    private void reportNodesFailingSystemUpgrade() {
+        metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of()));
+    }
+
+    private int nodesFailingSystemUpgrade() {
+        if (!controller().versionStatus().isUpgrading()) return 0;
+        var nodesFailingUpgrade = 0;
+        var acceptableInstant = clock.instant().minus(NODE_UPGRADE_TIMEOUT);
+        for (var vespaVersion : controller().versionStatus().versions()) {
+            if (vespaVersion.confidence() == VespaVersion.Confidence.broken) continue;
+            for (var nodeVersion : vespaVersion.nodeVersions().asMap().values()) {
+                if (!nodeVersion.changing()) continue;
+                if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++;
+            }
+        }
+        return nodesFailingUpgrade;
+    }
     
     private static double deploymentFailRatio(List<Instance> instances) {
         return instances.stream()
@@ -149,10 +172,8 @@ public class MetricsReporter extends Maintainer {
     }
 
     private static Map<String, String> dimensions(ApplicationId application) {
-        return ImmutableMap.of(
-                "tenant", application.tenant().value(),
-                "app",application.application().value() + "." + application.instance().value()
-        );
+        return Map.of("tenant", application.tenant().value(),
+                      "app",application.application().value() + "." + application.instance().value());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 08b3355587f..61fd0b67ec9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -1,14 +1,13 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -21,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -39,7 +37,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
-import java.security.Principal;
+import java.security.PublicKey;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -54,7 +52,6 @@ import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Serializes {@link Application}s to/from slime.
@@ -192,7 +189,7 @@ public class ApplicationSerializer {
         application.majorVersion().ifPresent(majorVersion -> root.setLong(majorVersionField, majorVersion));
         root.setDouble(queryQualityField, application.metrics().queryServiceQuality());
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
-        deployKeysToSlime(application.pemDeployKeys().stream(), root.setArray(pemDeployKeysField));
+        deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField));
         instancesToSlime(application, root.setArray(instancesField));
         return slime;
     }
@@ -208,8 +205,8 @@ public class ApplicationSerializer {
         }
     }
 
-    private void deployKeysToSlime(Stream<String> pemDeployKeys, Cursor array) {
-        pemDeployKeys.forEach(array::addString);
+    private void deployKeysToSlime(Set<PublicKey> deployKeys, Cursor array) {
+        deployKeys.forEach(key -> array.addString(KeyUtils.toPem(key)));
     }
 
     private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) {
@@ -384,14 +381,14 @@ public class ApplicationSerializer {
         OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField));
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
-        Set<String> pemDeployKeys = pemDeployKeysFromSlime(root.field(pemDeployKeysField));
+        Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField));
         List<Instance> instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField));
         OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField));
         boolean builtInternally = root.field(builtInternallyField).asBool();
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides, deploying, outstandingChange,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics,
-                               pemDeployKeys, projectId, builtInternally, instances);
+                               deployKeys, projectId, builtInternally, instances);
     }
 
     private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) {
@@ -411,9 +408,9 @@ public class ApplicationSerializer {
         return instances;
     }
 
-    private Set<String> pemDeployKeysFromSlime(Inspector array) {
-        Set<String> keys = new LinkedHashSet<>();
-        array.traverse((ArrayTraverser) (__, key) -> keys.add(key.asString()));
+    private Set<PublicKey> deployKeysFromSlime(Inspector array) {
+        Set<PublicKey> keys = new LinkedHashSet<>();
+        array.traverse((ArrayTraverser) (__, key) -> keys.add(KeyUtils.fromPemEncodedPublicKey(key.asString())));
         return keys;
     }
 
@@ -428,7 +425,7 @@ public class ApplicationSerializer {
                               applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)),
                               Version.fromString(deploymentObject.field(versionField).asString()),
                               Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
-                              clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)),
+                              Map.of(),
                               clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
                               deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)),
                               DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)),
@@ -484,21 +481,6 @@ public class ApplicationSerializer {
         return map;
     }
 
-    private Map<ClusterSpec.Id, ClusterUtilization> clusterUtilsMapFromSlime(Inspector object) {
-        Map<ClusterSpec.Id, ClusterUtilization> map = new HashMap<>();
-        object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterUtililzationFromSlime(value)));
-        return map;
-    }
-
-    private ClusterUtilization clusterUtililzationFromSlime(Inspector object) {
-        double cpu = object.field(clusterUtilsCpuField).asDouble();
-        double mem = object.field(clusterUtilsMemField).asDouble();
-        double disk = object.field(clusterUtilsDiskField).asDouble();
-        double diskBusy = object.field(clusterUtilsDiskBusyField).asDouble();
-
-        return new ClusterUtilization(mem, cpu, disk, diskBusy);
-    }
-
     private ClusterInfo clusterInfoFromSlime(Inspector inspector) {
         String flavor = inspector.field(clusterInfoFlavorField).asString();
         int cost = (int)inspector.field(clusterInfoCostField).asLong();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 9501ac5a7f9..357dbb37b27 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -360,22 +360,11 @@ public class CuratorDb {
 
     private Stream<TenantAndApplicationId> readApplicationIds() {
         return curator.getChildren(applicationRoot).stream()
-                      .filter(id -> id.split(":").length == 2)
                       .map(TenantAndApplicationId::fromSerialized);
     }
 
-    public void deleteOldApplicationData() {
-        curator.getChildren(applicationRoot).stream()
-               .filter(id -> id.split(":").length == 3)
-               .forEach(id -> curator.delete(applicationRoot.append(id)));
-    }
-
-    // TODO jonmv: Refactor when instance split operation is done
-    public void storeWithoutInstance(Application application) {
-        if (application.instances().isEmpty())
-                curator.delete(applicationPath(application.id()));
-        else
-            writeApplication(application);
+    public void removeApplication(TenantAndApplicationId id) {
+        curator.delete(applicationPath(id));
     }
 
     // -------------- Job Runs ------------------------------------------------
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index 78d166607df..35128466e4d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.net.URI;
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -91,14 +93,14 @@ public class TenantSerializer {
     }
 
     private void toSlime(CloudTenant tenant, Cursor root) {
-        pemDeveloperKeysToSlime(tenant.pemDeveloperKeys(), root.setArray(pemDeveloperKeysField));
+        developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField));
         toSlime(tenant.billingInfo(), root.setObject(billingInfoField));
     }
 
-    private void pemDeveloperKeysToSlime(BiMap<String, Principal> keys, Cursor array) {
+    private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) {
         keys.forEach((key, user) -> {
             Cursor object = array.addObject();
-            object.setString("key", key);
+            object.setString("key", KeyUtils.toPem(key));
             object.setString("user", user.getName());
         });
     }
@@ -139,15 +141,16 @@ public class TenantSerializer {
     private CloudTenant cloudTenantFrom(Inspector tenantObject) {
         TenantName name = TenantName.from(tenantObject.field(nameField).asString());
         BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField));
-        BiMap<String, Principal> pemDeveloperKeys = pemDeveloperKeysFromSlime(tenantObject.field(pemDeveloperKeysField));
-        return new CloudTenant(name, billingInfo, pemDeveloperKeys);
+        BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField));
+        return new CloudTenant(name, billingInfo, developerKeys);
     }
 
-    private BiMap<String, Principal> pemDeveloperKeysFromSlime(Inspector array) {
-        ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-        array.traverse((ArrayTraverser) (__, keyObject) -> {
-            keys.put(keyObject.field("key").asString(), new SimplePrincipal(keyObject.field("user").asString()));
-        });
+    private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) {
+        ImmutableBiMap.Builder<PublicKey, Principal> keys = ImmutableBiMap.builder();
+        array.traverse((ArrayTraverser) (__, keyObject) ->
+                keys.put(KeyUtils.fromPemEncodedPublicKey(keyObject.field("key").asString()),
+                         new SimplePrincipal(keyObject.field("user").asString())));
+
         return keys.build();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
index 207a5f8dcf9..5061f32da68 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -1,6 +1,7 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
@@ -9,6 +10,8 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
@@ -47,6 +50,14 @@ public class VersionStatusSerializer {
     private static final String confidenceField = "confidence";
     private static final String configServersField = "configServerHostnames";
 
+    // NodeVersions fields
+    private static final String nodeVersionsField = "nodeVersions";
+
+    // NodeVersion fields
+    private static final String hostnameField = "hostname";
+    private static final String wantedVersionField = "wantedVersion";
+    private static final String changedAtField = "changedAt";
+
     // DeploymentStatistics fields
     private static final String versionField = "version";
     private static final String failingField = "failing";
@@ -77,9 +88,20 @@ public class VersionStatusSerializer {
         object.setBool(isReleasedField, version.isReleased());
         deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField));
         object.setString(confidenceField, version.confidence().name());
-        configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField));
+        configServersToSlime(version.nodeVersions().hostnames(), object.setArray(configServersField));
+        nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField));
+    }
+
+    private void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) {
+        for (NodeVersion nodeVersion : nodeVersions.asMap().values()) {
+            var nodeVersionObject = array.addObject();
+            nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value());
+            nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString());
+            nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli());
+        }
     }
 
+    // TODO(mpolden): Remove after October 2019
     private void configServersToSlime(Set<HostName> configServerHostnames, Cursor array) {
         configServerHostnames.stream().map(HostName::value).forEach(array::addString);
     }
@@ -102,17 +124,38 @@ public class VersionStatusSerializer {
     }
 
     private VespaVersion vespaVersionFromSlime(Inspector object) {
-        return new VespaVersion(deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)),
+        var deploymentStatistics = deploymentStatisticsFromSlime(object.field(deploymentStatisticsField));
+        return new VespaVersion(deploymentStatistics,
                                 object.field(releaseCommitField).asString(),
                                 Instant.ofEpochMilli(object.field(committedAtField).asLong()),
                                 object.field(isControllerVersionField).asBool(),
                                 object.field(isSystemVersionField).asBool(),
-                                object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true,
-                                configServersFromSlime(object.field(configServersField)),
+                                object.field(isReleasedField).asBool(),
+                                nodeVersionsFromSlime(object, deploymentStatistics.version()),
                                 VespaVersion.Confidence.valueOf(object.field(confidenceField).asString())
         );
     }
 
+    private NodeVersions nodeVersionsFromSlime(Inspector root, Version version) {
+        var nodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
+        var nodeVersionsRoot = root.field(nodeVersionsField);
+        if (nodeVersionsRoot.valid()) {
+            nodeVersionsRoot.traverse((ArrayTraverser) (i, entry) -> {
+                var hostname = HostName.from(entry.field(hostnameField).asString());
+                var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString());
+                var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong());
+                nodeVersions.put(hostname, new NodeVersion(hostname, version, wantedVersion, changedAt));
+            });
+        } else {
+            // TODO(mpolden): Remove after October 2019
+            var configServerHostnames = configServersFromSlime(root.field(configServersField));
+            for (var hostname : configServerHostnames) {
+                nodeVersions.put(hostname, NodeVersion.empty(hostname));
+            }
+        }
+        return new NodeVersions(nodeVersions.build());
+    }
+
     private Set<HostName> configServersFromSlime(Inspector array) {
         Set<HostName> configServerHostnames = new LinkedHashSet<>();
         array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(HostName.from(entry.asString())));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 4c4478c9af6..24819fda261 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -6,8 +6,8 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
 import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
@@ -16,19 +16,23 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
+import com.yahoo.restapi.ErrorResponse;
+import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.Path;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.restapi.ResourceResponse;
+import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
-import com.yahoo.vespa.athenz.client.zms.ZmsClientException;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.AlreadyExistsException;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.NotExistsException;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
@@ -69,10 +73,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
 import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer;
-import com.yahoo.restapi.ErrorResponse;
-import com.yahoo.restapi.MessageResponse;
-import com.yahoo.restapi.ResourceResponse;
-import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
@@ -96,12 +96,12 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.DigestInputStream;
 import java.security.Principal;
+import java.security.PublicKey;
 import java.time.DayOfWeek;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
@@ -132,6 +132,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private final Controller controller;
     private final AccessControlRequests accessControlRequests;
+    private final TestConfigSerializer testConfigSerializer;
 
     @Inject
     public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx,
@@ -140,6 +141,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         super(parentCtx);
         this.controller = controller;
         this.accessControlRequests = accessControlRequests;
+        this.testConfigSerializer = new TestConfigSerializer(controller.system());
     }
 
     @Override
@@ -240,14 +242,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse handlePOST(Path path, HttpRequest request) {
         if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/key")) return addDeveloperKey(path.get("tenant"), request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createInstance(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request);
@@ -377,9 +379,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Principal user = request.getJDiscRequest().getUserPrincipal();
         String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
-        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
-                controller.tenants().store(tenant.withPemDeveloperKey(pemDeveloperKey, user)));
-        return new MessageResponse("Set developer key " + pemDeveloperKey + " for " + user);
+        PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey);
+        Slime root = new Slime();
+        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> {
+            tenant = tenant.withDeveloperKey(developerKey, user);
+            toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys());
+            controller.tenants().store(tenant);
+        });
+        return new SlimeJsonResponse(root);
     }
 
     private HttpResponse removeDeveloperKey(String tenantName, HttpRequest request) {
@@ -387,26 +394,51 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant");
 
         String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
-        Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).pemDeveloperKeys().get(pemDeveloperKey);
-        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
-                controller.tenants().store(tenant.withoutPemDeveloperKey(pemDeveloperKey)));
-        return new MessageResponse("Removed developer key " + pemDeveloperKey + " for " + user);
+        PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey);
+        Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).developerKeys().get(developerKey);
+        Slime root = new Slime();
+        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> {
+            tenant = tenant.withoutDeveloperKey(developerKey);
+            toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys());
+            controller.tenants().store(tenant);
+        });
+        return new SlimeJsonResponse(root);
+    }
+
+    private void toSlime(Cursor keysArray, Map<PublicKey, Principal> keys) {
+        keys.forEach((key, principal) -> {
+            Cursor keyObject = keysArray.addObject();
+            keyObject.setString("key", KeyUtils.toPem(key));
+            keyObject.setString("user", principal.getName());
+        });
     }
 
     private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) {
         String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
+        PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+        Slime root = new Slime();
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
-            controller.applications().store(application.withPemDeployKey(pemDeployKey));
+            application = application.withDeployKey(deployKey);
+            application.get().deployKeys().stream()
+                       .map(KeyUtils::toPem)
+                       .forEach(root.setObject().setArray("keys")::addString);
+            controller.applications().store(application);
         });
-        return new MessageResponse("Added deploy key " + pemDeployKey);
+        return new SlimeJsonResponse(root);
     }
 
     private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) {
         String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
+        PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+        Slime root = new Slime();
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
-            controller.applications().store(application.withoutPemDeployKey(pemDeployKey));
+            application = application.withoutDeployKey(deployKey);
+            application.get().deployKeys().stream()
+                       .map(KeyUtils::toPem)
+                       .forEach(root.setObject().setArray("keys")::addString);
+            controller.applications().store(application);
         });
-        return new MessageResponse("Removed deploy key " + pemDeployKey);
+        return new SlimeJsonResponse(root);
     }
 
     private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) {
@@ -424,7 +456,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             Inspector pemDeployKeyField = requestObject.field("pemDeployKey");
             if (pemDeployKeyField.valid()) {
                 String pemDeployKey = pemDeployKeyField.asString();
-                application = application.withPemDeployKey(pemDeployKey);
+                PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+                application = application.withDeployKey(deployKey);
                 messageBuilder.add("Added deploy key " + pemDeployKey);
             }
 
@@ -654,9 +687,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
 
         // TODO jonmv: Remove when clients are updated
-        application.pemDeployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", key));
+        application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key)));
 
-        application.pemDeployKeys().forEach(object.setArray("pemDeployKeys")::addString);
+        application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString);
 
         // Metrics
         Cursor metricsObject = object.setObject("metrics");
@@ -1017,25 +1050,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return tenant(controller.tenants().require(TenantName.from(tenantName)), request);
     }
 
-    private HttpResponse createApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) {
+    private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) {
         Inspector requestObject = toSlime(request.getData()).get();
-        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
-        try {
-            Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
-                    ? Optional.empty()
-                    : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()));
-            Application application = controller.applications().createApplication(id, credentials);
-
-            Slime slime = new Slime();
-            toSlime(id, slime.setObject(), request);
-            return new SlimeJsonResponse(slime);
-        }
-        catch (ZmsClientException e) { // TODO: Push conversion down
-            if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN)
-                throw new ForbiddenException("Not authorized to create application", e);
-            else
-                throw e;
-        }
+        TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
+        Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
+                                            ? Optional.empty()
+                                            : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()));
+        Application application = controller.applications().createApplication(id, credentials);
+
+        Slime slime = new Slime();
+        toSlime(id, slime.setObject(), request);
+        return new SlimeJsonResponse(slime);
+    }
+
+    // TODO jonmv: Remove when clients are updated.
+    private HttpResponse createInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) {
+        TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenantName, applicationName);
+        if (controller.applications().getApplication(applicationId).isEmpty())
+            createApplication(tenantName, applicationName, request);
+
+        controller.applications().createInstance(applicationId.instance(instanceName));
+
+        Slime slime = new Slime();
+        toSlime(applicationId.instance(instanceName), slime.setObject(), request);
+        return new SlimeJsonResponse(slime);
     }
 
     /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */
@@ -1227,12 +1265,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                                  deployOptions.field("ignoreValidationErrors").asBool(),
                                                                  deployOptions.field("deployCurrentVersion").asBool());
 
+        applicationPackage.ifPresent(aPackage -> controller.applications().verifyApplicationIdentityConfiguration(applicationId.tenant(),
+                                                                                                                  aPackage,
+                                                                                                                  Optional.of(requireUserPrincipal(request))));
+
         ActivateResult result = controller.applications().deploy(applicationId,
                                                                  zone,
                                                                  applicationPackage,
                                                                  applicationVersion,
-                                                                 deployOptionsJsonClass,
-                                                                 Optional.of(requireUserPrincipal(request)));
+                                                                 deployOptionsJsonClass);
 
         return new SlimeJsonResponse(toSlime(result));
     }
@@ -1255,22 +1296,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) {
-        TenantName tenant = TenantName.from(tenantName);
-        ApplicationName application = ApplicationName.from(applicationName);
-        Optional<Credentials> credentials = controller.tenants().require(tenant).type() == Tenant.Type.user
+        TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
+        Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
                                             ? Optional.empty()
-                                            : Optional.of(accessControlRequests.credentials(tenant, toSlime(request.getData()).get(), request.getJDiscRequest()));
-        controller.applications().deleteApplication(tenant, application, credentials);
-        return new MessageResponse("Deleted application " + tenant + "." + application);
+                                            : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()));
+        controller.applications().deleteApplication(id, credentials);
+        return new MessageResponse("Deleted application " + id);
     }
 
     private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) {
-        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+        TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
         Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
                 ? Optional.empty()
                 : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()));
-        controller.applications().deleteInstance(id, credentials);
-        return new MessageResponse("Deleted instance " + id.toFullString());
+        controller.applications().deleteInstance(id.instance(instanceName));
+        if (controller.applications().requireApplication(id).instances().isEmpty())
+            controller.applications().deleteApplication(id, credentials);
+        return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString());
     }
 
     private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
@@ -1300,11 +1342,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse testConfig(ApplicationId id, JobType type) {
-        var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type));
-        return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id,
-                                                                                               type,
-                                                                                               endpoints,
-                                                                                               Collections.emptyMap()));
+        Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id, type);
+        return new SlimeJsonResponse(testConfigSerializer.configSlime(id,
+                                                                      type,
+                                                                      controller.applications().clusterEndpoints(id, zones),
+                                                                      controller.applications().contentClustersByZone(id, zones)));
     }
 
     private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) {
@@ -1366,18 +1408,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             case cloud: {
                 CloudTenant cloudTenant = (CloudTenant) tenant;
 
-                Cursor pemDeployKeysArray = object.setArray("pemDeployKeys");
-                for (Application application : applications)
-                    for (String key : application.pemDeployKeys()) {
-                        Cursor keyObject = pemDeployKeysArray.addObject();
-                        keyObject.setString("key", key);
-                        keyObject.setString("application", application.id().application().value());
-                    }
-
                 Cursor pemDeveloperKeysArray = object.setArray("pemDeveloperKeys");
-                cloudTenant.pemDeveloperKeys().forEach((key, user) -> {
+                cloudTenant.developerKeys().forEach((key, user) -> {
                     Cursor keyObject = pemDeveloperKeysArray.addObject();
-                    keyObject.setString("key", key);
+                    keyObject.setString("key", KeyUtils.toPem(key));
                     keyObject.setString("user", user.getName());
                 });
 
@@ -1470,6 +1504,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return Joiner.on("/").join(elements);
     }
 
+    private void toSlime(TenantAndApplicationId id, Cursor object, HttpRequest request) {
+        object.setString("tenant", id.tenant().value());
+        object.setString("application", id.application().value());
+        object.setString("url", withPath("/application/v4" +
+                                         "/tenant/" + id.tenant().value() +
+                                         "/application/" + id.application().value(),
+                                         request.getUri()).toString());
+    }
+
     private void toSlime(ApplicationId id, Cursor object, HttpRequest request) {
         object.setString("tenant", id.tenant().value());
         object.setString("application", id.application().value());
@@ -1639,6 +1682,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         long projectId = Math.max(1, submitOptions.field("projectId").asLong());
 
         ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP));
+        if (DeploymentSpec.empty.equals(applicationPackage.deploymentSpec()))
+            throw new IllegalArgumentException("Missing required file 'deployment.xml'");
+
         controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant),
                                                                          applicationPackage,
                                                                          Optional.of(requireUserPrincipal(request)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index ab92e38ee4b..49015f16cce 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -64,6 +64,9 @@ import static java.util.stream.Collectors.toMap;
  *
  * @see JobController
  * @see ApplicationApiHandler
+ *
+ * @author smorgrav
+ * @author jonmv
  */
 class JobControllerApiHandlerHelper {
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 86310ca2f6b..2adf6ce95e1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
 import com.yahoo.component.Version;
@@ -91,7 +91,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
             versionObject.setBool("systemVersion", version.isSystemVersion());
 
             Cursor configServerArray = versionObject.setArray("configServers");
-            for (HostName hostname : version.systemApplicationHostnames()) {
+            for (HostName hostname : version.nodeVersions().hostnames()) {
                 Cursor configServerObject = configServerArray.addObject();
                 configServerObject.setString("hostname", hostname.value());
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
index 6755110bb49..7ad2e03ef1d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
@@ -10,19 +10,27 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
 import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase;
 import com.yahoo.log.LogLevel;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.yolean.Exceptions;
 
 import java.security.Principal;
+import java.security.PublicKey;
+import java.util.Base64;
+import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.logging.Logger;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * Assigns the {@link Role#buildService(TenantName, ApplicationName)} role to requests with a
  * Authorization header signature matching the public key of the indicated application.
@@ -46,25 +54,11 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
         if (   request.getAttribute(SecurityContext.ATTRIBUTE_NAME) == null
             && request.getHeader("X-Authorization") != null)
             try {
-                ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
-                boolean verified = controller.applications().getApplication(TenantAndApplicationId.from(id)).stream()
-                                             .flatMap(application -> application.pemDeployKeys().stream())
-                                             .map(key -> new RequestVerifier(key, controller.clock()))
-                                             .anyMatch(verifier -> verifier.verify(Method.valueOf(request.getMethod()),
-                                                                                   request.getUri(),
-                                                                                   request.getHeader("X-Timestamp"),
-                                                                                   request.getHeader("X-Content-Hash"),
-                                                                                   request.getHeader("X-Authorization")));
-
-                if (verified) {
-                    Principal principal = new SimplePrincipal("buildService@" + id.tenant() + "." + id.application());
-                    request.setUserPrincipal(principal);
-                    request.setRemoteUser(principal.getName());
-                    request.setAttribute(SecurityContext.ATTRIBUTE_NAME,
-                                         new SecurityContext(principal,
-                                                             Set.of(Role.buildService(id.tenant(), id.application()),
-                                                                    Role.applicationDeveloper(id.tenant(), id.application()))));
-                }
+                getSecurityContext(request).ifPresent(securityContext -> {
+                    request.setUserPrincipal(securityContext.principal());
+                    request.setRemoteUser(securityContext.principal().getName());
+                    request.setAttribute(SecurityContext.ATTRIBUTE_NAME, securityContext);
+                });
             }
             catch (Exception e) {
                 logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e));
@@ -72,4 +66,48 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
         return Optional.empty();
     }
 
+    // TODO jonmv: Remove after October 2019.
+    private boolean anyDeployKeyMatches(TenantAndApplicationId id, DiscFilterRequest request) {
+        return controller.applications().getApplication(id).stream()
+                         .map(Application::deployKeys)
+                         .flatMap(Set::stream)
+                         .anyMatch(key -> keyVerifies(key, request));
+    }
+
+    private boolean keyVerifies(PublicKey key, DiscFilterRequest request) {
+        return new RequestVerifier(key, controller.clock()).verify(Method.valueOf(request.getMethod()),
+                                                                   request.getUri(),
+                                                                   request.getHeader("X-Timestamp"),
+                                                                   request.getHeader("X-Content-Hash"),
+                                                                   request.getHeader("X-Authorization"));
+    }
+
+    private Optional<SecurityContext> getSecurityContext(DiscFilterRequest request) {
+        ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
+        if (request.getHeader("X-Key") != null) { // TODO jonmv: Remove check and else branch after Oct 2019.
+            PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8));
+            if (keyVerifies(key, request)) {
+                Optional<CloudTenant> tenant = controller.tenants().get(id.tenant())
+                                                         .filter(CloudTenant.class::isInstance)
+                                                         .map(CloudTenant.class::cast);
+                if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key))
+                    return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key),
+                                                           Set.of(Role.reader(id.tenant()),
+                                                                  Role.developer(id.tenant()))));
+
+                Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id));
+                if (application.isPresent() && application.get().deployKeys().contains(key))
+                    return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
+                                                           Set.of(Role.reader(id.tenant()),
+                                                                  Role.developer(id.tenant())))); // TODO jonmv: Change to headless after Oct 10 2019.
+            }
+        }
+        else if (anyDeployKeyMatches(TenantAndApplicationId.from(id), request))
+            return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
+                                                   Set.of(Role.reader(id.tenant()),
+                                                          Role.developer(id.tenant()))));
+
+        return Optional.empty();
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
index 807e74b7c75..77622df4c4a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
@@ -26,10 +26,9 @@ import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
-import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.yolean.Exceptions;
 
-import java.security.Principal;
+import java.security.PublicKey;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
@@ -200,9 +199,9 @@ public class UserApiHandler extends LoggingRequestHandler {
         // TODO jonmv: Change to developer role, when this exists.
         if (role.definition().equals(RoleDefinition.tenantOperator))
             controller.tenants().lockIfPresent(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> {
-                String key = tenant.get().pemDeveloperKeys().inverse().get(new SimplePrincipal(user.value()));
+                PublicKey key = tenant.get().developerKeys().inverse().get(new SimplePrincipal(user.value()));
                 if (key != null)
-                    controller.tenants().store(tenant.withoutPemDeveloperKey(key));
+                    controller.tenants().store(tenant.withoutDeveloperKey(key));
             });
 
         users.removeUsers(role, List.of(user));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java
index 77ccce873fe..66c87a8eefd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.security;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.util.List;
@@ -52,7 +53,7 @@ public interface AccessControl {
      * @param id the ID of the application to create
      * @param credentials the credentials for the entity requesting the creation
      */
-    void createApplication(ApplicationId id, Credentials credentials);
+    void createApplication(TenantAndApplicationId id, Credentials credentials);
 
     /**
      * Deletes access control for the given tenant.
@@ -60,7 +61,7 @@ public interface AccessControl {
      * @param id the ID of the application to delete
      * @param credentials the credentials for the entity requesting the deletion
      */
-    void deleteApplication(ApplicationId id, Credentials credentials);
+    void deleteApplication(TenantAndApplicationId id, Credentials credentials);
 
     /**
      * Returns the list of tenants to which a user has access.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
index 7da3e43c9a5..a88e38e5f89 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
@@ -1,10 +1,8 @@
 package com.yahoo.vespa.hosted.controller.security;
 
 import com.google.inject.Inject;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.api.integration.user.Roles;
 import com.yahoo.vespa.hosted.controller.api.integration.user.UserId;
@@ -12,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement;
 import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.TenantRole;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
@@ -58,14 +57,14 @@ public class CloudAccessControl implements AccessControl {
     }
 
     @Override
-    public void createApplication(ApplicationId id, Credentials credentials) {
+    public void createApplication(TenantAndApplicationId id, Credentials credentials) {
         for (Role role : Roles.applicationRoles(id.tenant(), id.application()))
             userManagement.createRole(role);
         userManagement.addUsers(Role.applicationAdmin(id.tenant(), id.application()), List.of(new UserId(credentials.user().getName())));
     }
 
     @Override
-    public void deleteApplication(ApplicationId id, Credentials credentials) {
+    public void deleteApplication(TenantAndApplicationId id, Credentials credentials) {
         for (ApplicationRole role : Roles.applicationRoles(id.tenant(), id.application()))
             userManagement.deleteRole(role);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index 6ef9b5e6a4f..e230daf0c50 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -17,13 +18,13 @@ import java.util.Optional;
 public class CloudTenant extends Tenant {
 
     private final BillingInfo billingInfo;
-    private final BiMap<String, Principal> pemDeveloperKeys;
+    private final BiMap<PublicKey, Principal> developerKeys;
 
     /** Public for the serialization layer — do not use! */
-    public CloudTenant(TenantName name, BillingInfo info, BiMap<String, Principal> pemDeveloperKeys) {
+    public CloudTenant(TenantName name, BillingInfo info, BiMap<PublicKey, Principal> developerKeys) {
         super(name, Optional.empty());
         billingInfo = info;
-        this.pemDeveloperKeys = pemDeveloperKeys;
+        this.developerKeys = developerKeys;
     }
 
     /** Creates a tenant with the given name, provided it passes validation. */
@@ -37,7 +38,7 @@ public class CloudTenant extends Tenant {
     public BillingInfo billingInfo() { return billingInfo; }
 
     /** Returns the set of developer keys and their corresponding developers for this tenant. */
-    public BiMap<String, Principal> pemDeveloperKeys() { return pemDeveloperKeys; }
+    public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; }
 
     @Override
     public Type type() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
new file mode 100644
index 00000000000..0a690b90410
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
@@ -0,0 +1,93 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}.
+ *
+ * This is immutable.
+ *
+ * @author mpolden
+ */
+public class NodeVersion {
+
+    private final HostName hostname;
+    private final Version currentVersion;
+    private final Version wantedVersion;
+    private final Instant changedAt;
+
+    public NodeVersion(HostName hostname, Version currentVersion, Version wantedVersion, Instant changedAt) {
+        this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
+        this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null");
+        this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null");
+        this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null");
+    }
+
+    /** Hostname of this */
+    public HostName hostname() {
+        return hostname;
+    }
+
+    /** Current version of this */
+    public Version currentVersion() {
+        return currentVersion;
+    }
+
+    /** Wanted version of this */
+    public Version wantedVersion() {
+        return wantedVersion;
+    }
+
+    /** Returns whether this is changing (upgrading or downgrading) */
+    public boolean changing() {
+        return !currentVersion.equals(wantedVersion);
+    }
+
+    /** The most recent time the version of this changed */
+    public Instant changedAt() {
+        return changedAt;
+    }
+
+    /** Returns a copy of this with current version set to given version */
+    public NodeVersion withCurrentVersion(Version version, Instant changedAt) {
+        if (currentVersion.equals(version)) return this;
+        return new NodeVersion(hostname, version, wantedVersion, changedAt);
+    }
+
+    /** Returns a copy of this with wanted version set to given version */
+    public NodeVersion withWantedVersion(Version version) {
+        if (wantedVersion.equals(version)) return this;
+        return new NodeVersion(hostname, currentVersion, version, changedAt);
+    }
+
+    @Override
+    public String toString() {
+        return hostname + ": " + currentVersion + " -> " + wantedVersion + " [changedAt=" + changedAt + "]";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeVersion that = (NodeVersion) o;
+        return hostname.equals(that.hostname) &&
+               currentVersion.equals(that.currentVersion) &&
+               wantedVersion.equals(that.wantedVersion) &&
+               changedAt.equals(that.changedAt);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hostname, currentVersion, wantedVersion, changedAt);
+    }
+
+    public static NodeVersion empty(HostName hostname) {
+        return new NodeVersion(hostname, Version.emptyVersion, Version.emptyVersion, Instant.EPOCH);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
new file mode 100644
index 00000000000..3ab96e03bcd
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
@@ -0,0 +1,97 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ListMultimap;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * A filterable list of {@link NodeVersion}s. This is immutable.
+ *
+ * @author mpolden
+ */
+public class NodeVersions {
+
+    public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of());
+
+    private final ImmutableMap<HostName, NodeVersion> nodeVersions;
+
+    public NodeVersions(ImmutableMap<HostName, NodeVersion> nodeVersions) {
+        this.nodeVersions = Objects.requireNonNull(nodeVersions);
+    }
+
+    public Map<HostName, NodeVersion> asMap() {
+        return nodeVersions;
+    }
+
+    /** Returns host names in this, grouped by version */
+    public ListMultimap<Version, HostName> asVersionMap() {
+        var versions = ImmutableListMultimap.<Version, HostName>builder();
+        for (var kv : nodeVersions.entrySet()) {
+            versions.put(kv.getValue().currentVersion(), kv.getKey());
+        }
+        return versions.build();
+    }
+
+    /** Returns host names in this */
+    public Set<HostName> hostnames() {
+        return nodeVersions.keySet();
+    }
+
+    /** Returns a copy of this containing only node versions of given version */
+    public NodeVersions matching(Version version) {
+        return filter(nodeVersion -> nodeVersion.currentVersion().equals(version));
+    }
+
+    /** Returns number of node versions in this */
+    public int size() {
+        return nodeVersions.size();
+    }
+
+    /** Returns a copy of this containing only the given node versions */
+    public NodeVersions with(List<NodeVersion> nodeVersions) {
+        var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
+        for (var nodeVersion : nodeVersions) {
+            var existing = this.nodeVersions.get(nodeVersion.hostname());
+            if (existing != null) {
+                newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(),
+                                                                                        nodeVersion.changedAt())
+                                                                    .withWantedVersion(nodeVersion.wantedVersion()));
+            } else {
+                newNodeVersions.put(nodeVersion.hostname(), nodeVersion);
+            }
+        }
+        return new NodeVersions(newNodeVersions.build());
+    }
+
+    private NodeVersions filter(Predicate<NodeVersion> predicate) {
+        var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
+        for (var kv : nodeVersions.entrySet()) {
+            if (!predicate.test(kv.getValue())) continue;
+            newNodeVersions.put(kv.getKey(), kv.getValue());
+        }
+        return new NodeVersions(newNodeVersions.build());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeVersions that = (NodeVersions) o;
+        return nodeVersions.equals(that.nodeVersions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodeVersions);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 9dc6b86e4be..bb43ec20234 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -6,12 +6,10 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.JobList;
@@ -70,7 +68,8 @@ public class VersionStatus {
     /** Returns whether the system is currently upgrading */
     public boolean isUpgrading() {
         return systemVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion)
-                              .isBefore(controllerVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion));
+                              .isBefore(controllerVersion().map(VespaVersion::versionNumber)
+                                                           .orElse(Version.emptyVersion));
     }
 
     /** 
@@ -91,14 +90,14 @@ public class VersionStatus {
 
     /** Create a full, updated version status. This is expensive and should be done infrequently */
     public static VersionStatus compute(Controller controller) {
-        ListMultimap<Version, HostName> systemApplicationVersions = findSystemApplicationVersions(controller);
-        ListMultimap<ControllerVersion, HostName> controllerVersions = findControllerVersions(controller);
+        var systemApplicationVersions = findSystemApplicationVersions(controller);
+        var controllerVersions = findControllerVersions(controller);
 
-        ListMultimap<Version, HostName> infrastructureVersions = ArrayListMultimap.create();
+        var infrastructureVersions = ArrayListMultimap.<Version, HostName>create();
         for (var kv : controllerVersions.asMap().entrySet()) {
             infrastructureVersions.putAll(kv.getKey().version(), kv.getValue());
         }
-        infrastructureVersions.putAll(systemApplicationVersions);
+        infrastructureVersions.putAll(systemApplicationVersions.asVersionMap());
 
         // The controller version is the lowest controller version of all controllers
         ControllerVersion controllerVersion = controllerVersions.keySet().stream()
@@ -138,7 +137,7 @@ public class VersionStatus {
                                                           controllerVersion,
                                                           systemVersion,
                                                           isReleased,
-                                                          systemApplicationVersions.get(statistics.version()),
+                                                          systemApplicationVersions.matching(statistics.version()),
                                                           controller);
                 versions.add(vespaVersion);
             } catch (IllegalArgumentException e) {
@@ -152,29 +151,32 @@ public class VersionStatus {
         return new VersionStatus(versions);
     }
 
-    private static ListMultimap<Version, HostName> findSystemApplicationVersions(Controller controller) {
-        ListMultimap<Version, HostName> versions = ArrayListMultimap.create();
-        for (ZoneApi zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
-            for (SystemApplication application : SystemApplication.all()) {
-                List<Node> eligibleForUpgradeApplicationNodes = controller.serviceRegistry().configServer().nodeRepository()
-                        .list(zone.getId(), application.id()).stream()
-                        .filter(SystemUpgrader::eligibleForUpgrade)
-                        .collect(Collectors.toList());
-                if (eligibleForUpgradeApplicationNodes.isEmpty())
-                    continue;
-
-                boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
+    private static NodeVersions findSystemApplicationVersions(Controller controller) {
+        var nodeVersions = controller.versionStatus().systemVersion()
+                                     .map(VespaVersion::nodeVersions)
+                                     .orElse(NodeVersions.EMPTY);
+        var newNodeVersions = new ArrayList<NodeVersion>();
+        for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
+            for (var application : SystemApplication.all()) {
+                var nodes = controller.serviceRegistry().configServer().nodeRepository()
+                                      .list(zone.getId(), application.id()).stream()
+                                      .filter(SystemUpgrader::eligibleForUpgrade)
+                                      .collect(Collectors.toList());
+                if (nodes.isEmpty()) continue;
+                var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
                 if (!configConverged) {
-                    log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged");
+                    log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() +
+                                              " has not converged");
                 }
-                for (Node node : eligibleForUpgradeApplicationNodes) {
+                var now = controller.clock().instant();
+                for (var node : nodes) {
                     // Only use current node version if config has converged
-                    Version nodeVersion = configConverged ? node.currentVersion() : controller.systemVersion();
-                    versions.put(nodeVersion, node.hostname());
+                    Version version = configConverged ? node.currentVersion() : controller.systemVersion();
+                    newNodeVersions.add(new NodeVersion(node.hostname(), version, node.wantedVersion(), now));
                 }
             }
         }
-        return versions;
+        return nodeVersions.with(newNodeVersions);
     }
 
     private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) {
@@ -241,7 +243,7 @@ public class VersionStatus {
                                               ControllerVersion controllerVersion,
                                               Version systemVersion,
                                               boolean isReleased,
-                                              Collection<HostName> configServerHostnames,
+                                              NodeVersions nodeVersions,
                                               Controller controller) {
         var isSystemVersion = statistics.version().equals(systemVersion);
         var isControllerVersion = statistics.version().equals(controllerVersion.version());
@@ -279,7 +281,7 @@ public class VersionStatus {
                                 isControllerVersion,
                                 isSystemVersion,
                                 isReleased,
-                                configServerHostnames,
+                                nodeVersions,
                                 confidence);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index dc0b2c12d5c..0d144913022 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -1,16 +1,12 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
-import java.util.Collection;
-import java.util.Set;
 
 import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
 
@@ -30,12 +26,12 @@ public class VespaVersion implements Comparable<VespaVersion> {
     private final boolean isSystemVersion;
     private final boolean isReleased;
     private final DeploymentStatistics statistics;
-    private final ImmutableSet<HostName> systemApplicationHostnames;
+    private final NodeVersions nodeVersions;
     private final Confidence confidence;
 
     public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt,
                         boolean isControllerVersion, boolean isSystemVersion, boolean isReleased,
-                        Collection<HostName> systemApplicationHostnames,
+                        NodeVersions nodeVersions,
                         Confidence confidence) {
         this.statistics = statistics;
         this.releaseCommit = releaseCommit;
@@ -43,7 +39,7 @@ public class VespaVersion implements Comparable<VespaVersion> {
         this.isControllerVersion = isControllerVersion;
         this.isSystemVersion = isSystemVersion;
         this.isReleased = isReleased;
-        this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames);
+        this.nodeVersions = nodeVersions;
         this.confidence = confidence;
     }
 
@@ -108,9 +104,11 @@ public class VespaVersion implements Comparable<VespaVersion> {
     /** Returns whether the artifacts of this release are available in the configured maven repository. */
     public boolean isReleased() { return isReleased; }
 
-    /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */
-    public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; }
-    
+    /** Returns the versions of nodes allocated to system applications (across all zones) */
+    public NodeVersions nodeVersions() {
+        return nodeVersions;
+    }
+
     /** Returns the confidence we have in this versions suitability for production */
     public Confidence confidence() { return confidence; }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index fab1ed2ab20..e3682a78b7d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -507,7 +507,7 @@ public class ControllerTest {
             tester.deployAndNotify(tester.defaultInstance(app1.id()).id(), Optional.of(applicationPackage), true, systemTest);
             tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.test, RegionName.from("us-east-1")));
             tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.staging, RegionName.from("us-east-3")));
-            tester.applications().deleteApplication(app1.id().tenant(), app1.id().application(), tester.controllerTester().credentialsFor(app1.id()));
+            tester.applications().deleteApplication(app1.id(), tester.controllerTester().credentialsFor(app1.id()));
             try (RotationLock lock = tester.applications().rotationRepository().lock()) {
                 assertTrue("Rotation is unassigned",
                            tester.applications().rotationRepository().availableRotations(lock)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index cefdc3bed61..2c88d122e8f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -233,11 +233,12 @@ public final class ControllerTester {
     }
 
     public Application createApplication(TenantName tenant, String applicationName, String instanceName, long projectId) {
-        ApplicationId applicationId = ApplicationId.from(tenant.value(), applicationName, instanceName);
-        controller().applications().createApplication(applicationId, credentialsFor(TenantAndApplicationId.from(applicationId)));
-        controller().applications().lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application ->
+        TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenant.value(), applicationName);
+        controller().applications().createApplication(applicationId, credentialsFor(applicationId));
+        controller().applications().lockApplicationOrThrow(applicationId, application ->
                 controller().applications().store(application.withProjectId(OptionalLong.of(projectId))));
-        Application application = controller().applications().requireApplication(TenantAndApplicationId.from(applicationId));
+        controller().applications().createInstance(applicationId.instance(instanceName));
+        Application application = controller().applications().requireApplication(applicationId);
         assertTrue(application.projectId().isPresent());
         return application;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 5dc6fb183a2..61b393efbff 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -148,8 +148,13 @@ public class DeploymentTester {
 
     /** Upgrade system applications in all zones to given version */
     public void upgradeSystemApplications(Version version) {
+        upgradeSystemApplications(version, SystemApplication.all());
+    }
+
+    /** Upgrade given system applications in all zones to version */
+    public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) {
         for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) {
-            for (SystemApplication application : SystemApplication.all()) {
+            for (SystemApplication application : systemApplications) {
                 tester.configServer().setVersion(application.id(), zone.getId(), version);
                 tester.configServer().convergeServices(application.id(), zone.getId());
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 4a7ee8bcb63..6da77a967f1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.google.inject.Inject;
@@ -110,7 +110,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
                 List<Node> nodes = IntStream.rangeClosed(1, 3)
                                             .mapToObj(i -> new Node(
                                                     HostName.from("node-" + i + "-" + application.id().application()
-                                                                                                 .value()),
+                                                                                                 .value()
+                                                                  + "-" + zone.value()),
                                                     Node.State.active, application.nodeType(),
                                                     Optional.of(application.id()),
                                                     initialVersion,
@@ -150,9 +151,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     /** Set version for an application in a given zone */
     public void setVersion(ApplicationId application, ZoneId zone, Version version) {
+        setVersion(application, zone, version, -1);
+    }
+
+    /** Set version for nodeCount number of nodes in application in a given zone */
+    public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) {
+        int n = 0;
         for (Node node : nodeRepository().list(zone, application)) {
             nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(),
                                                           version, version));
+            if (++n == nodeCount) break;
         }
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index ff245e2e488..c6bd4bde410 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -218,7 +218,7 @@ public class JobRunnerTest {
 
         // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected.
         assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList()));
-        tester.controllerTester().controller().applications().deleteApplication(id.tenant(), id.application(), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id)));
+        tester.controllerTester().controller().applications().deleteApplication(TenantAndApplicationId.from(id), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id)));
         assertEquals(Collections.emptyList(), jobs.active());
         assertEquals(runId, jobs.last(id, systemTest).get().id());
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index 4fc952b0b15..9cb40d60677 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -1,9 +1,10 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -11,14 +12,17 @@ import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component;
@@ -213,6 +217,51 @@ public class MetricsReporterTest {
         assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue());
     }
 
+    @Test
+    public void test_nodes_failing_system_upgrade() {
+        var tester = new DeploymentTester();
+        var reporter = createReporter(tester.controller());
+        var zone1 = ZoneApiMock.fromId("prod.eu-west-1");
+        tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1));
+        var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1),
+                                                new JobControl(tester.controllerTester().curator()));
+        tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer);
+
+        // System on initial version
+        var version0 = Version.fromString("7.0");
+        tester.upgradeSystem(version0);
+        reporter.maintain();
+        assertEquals(0, getNodesFailingUpgrade());
+
+        for (var version : List.of(Version.fromString("7.1"), Version.fromString("7.2"))) {
+            // System starts upgrading to next version
+            tester.upgradeController(version);
+            reporter.maintain();
+            assertEquals(0, getNodesFailingUpgrade());
+            systemUpgrader.maintain();
+
+            // 30 minutes pass and nothing happens
+            tester.clock().advance(Duration.ofMinutes(30));
+            tester.computeVersionStatus();
+            reporter.maintain();
+            assertEquals(0, getNodesFailingUpgrade());
+
+            // 1/3 nodes upgrade within timeout
+            tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1);
+            tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+            tester.computeVersionStatus();
+            reporter.maintain();
+            assertEquals(2, getNodesFailingUpgrade());
+
+            // 3/3 nodes upgrade
+            tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version);
+            tester.computeVersionStatus();
+            reporter.maintain();
+            assertEquals(0, getNodesFailingUpgrade());
+            assertEquals(version, tester.controller().systemVersion());
+        }
+    }
+
     private Duration getAverageDeploymentDuration(ApplicationId id) {
         return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue());
     }
@@ -225,6 +274,10 @@ public class MetricsReporterTest {
         return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue();
     }
 
+    private int getNodesFailingUpgrade() {
+        return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue();
+    }
+
     private Number getMetric(String name, ApplicationId id) {
         return metrics.getMetric((dimensions) -> id.tenant().value().equals(dimensions.get("tenant")) &&
                                                  appDimension(id).equals(dimensions.get("app")),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 9677df6fd18..72b26aca588 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -1,10 +1,9 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -59,6 +58,7 @@ public class SystemUpgraderTest {
         systemUpgrader.maintain();
         assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4);
         assertCurrentVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4);
+        assertSystemVersion(version1);
 
         // Controller upgrades
         Version version2 = Version.fromString("6.6");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 3ba1181f762..08963b9fec7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -1,13 +1,13 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
-import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -37,6 +36,7 @@ import org.junit.Test;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.PublicKey;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -48,7 +48,6 @@ import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static java.util.Optional.empty;
@@ -64,6 +63,15 @@ public class ApplicationSerializerTest {
     private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/");
     private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1");
     private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3");
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
+
 
     @Test
     public void testSerialization() {
@@ -134,7 +142,7 @@ public class ApplicationSerializerTest {
                                                Optional.of(User.from("by-username")),
                                                OptionalInt.of(7),
                                                new ApplicationMetrics(0.5, 0.9),
-                                               Set.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"),
+                                               Set.of(publicKey, otherPublicKey),
                                                projectId,
                                                true,
                                                instances);
@@ -178,17 +186,14 @@ public class ApplicationSerializerTest {
         assertEquals(original.owner(), serialized.owner());
         assertEquals(original.majorVersion(), serialized.majorVersion());
         assertEquals(original.change(), serialized.change());
-        assertEquals(original.pemDeployKeys(), serialized.pemDeployKeys());
+        assertEquals(original.deployKeys(), serialized.deployKeys());
 
         assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations());
         assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus());
 
         // Test cluster utilization
         assertEquals(0, serialized.require(id1.instance()).deployments().get(zone1).clusterUtils().size());
-        assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size());
-        assertEquals(0.4, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01);
-        assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01);
-        assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01);
+        assertEquals(0, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size());
 
         // Test cluster info
         assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index 51df0e4b08b..ff1c952c2a5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Ho
 
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import org.junit.Test;
 
 import java.net.URI;
+import java.security.PublicKey;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -29,6 +31,14 @@ import static org.junit.Assert.assertTrue;
 public class TenantSerializerTest {
 
     private static final TenantSerializer serializer = new TenantSerializer();
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
 
     @Test
     public void athenz_tenant() {
@@ -78,12 +88,12 @@ public class TenantSerializerTest {
     public void cloud_tenant() {
         CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"),
                                              new BillingInfo("old cat lady", "vespa"),
-                                             ImmutableBiMap.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", new SimplePrincipal("joe"),
-                                                               "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", new SimplePrincipal("jane")));
+                                             ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"),
+                                                               otherPublicKey, new SimplePrincipal("jane")));
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.name(), serialized.name());
         assertEquals(tenant.billingInfo(), serialized.billingInfo());
-        assertEquals(tenant.pemDeveloperKeys(), serialized.pemDeveloperKeys());
+        assertEquals(tenant.developerKeys(), serialized.developerKeys());
     }
 
     private Contact contact() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
index a1e22b4fc64..5d65cf0381e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -1,20 +1,23 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static java.time.temporal.ChronoUnit.MILLIS;
 import static org.junit.Assert.assertEquals;
@@ -36,9 +39,11 @@ public class VersionStatusSerializerTest {
                         ApplicationId.from("tenant2", "success2", "default"))
         );
         vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false,
-                                           true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+                                           true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"),
+                                                              Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
         vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true,
-                                           false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+                                           false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"),
+                                                               Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
         VersionStatus status = new VersionStatus(vespaVersions);
         VersionStatusSerializer serializer = new VersionStatusSerializer();
         VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status));
@@ -53,14 +58,48 @@ public class VersionStatusSerializerTest {
             assertEquals(a.isSystemVersion(), b.isSystemVersion());
             assertEquals(a.isReleased(), b.isReleased());
             assertEquals(a.statistics(), b.statistics());
-            assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames());
+            assertEquals(a.nodeVersions(), b.nodeVersions());
             assertEquals(a.confidence(), b.confidence());
         }
 
     }
 
-    private static List<HostName> asHostnames(String... hostname) {
-        return Arrays.stream(hostname).map(HostName::from).collect(Collectors.toList());
+    @Test
+    public void testLegacySerialization() throws Exception {
+        var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json"));
+        var serializer = new VersionStatusSerializer();
+        var deserializedStatus = serializer.fromSlime(SlimeUtils.jsonToSlime(data));
+
+        var statistics = new DeploymentStatistics(
+                Version.fromString("7.0"),
+                List.of(),
+                List.of(),
+                List.of()
+        );
+        var vespaVersion = new VespaVersion(statistics, "badc0ffee",
+                                            Instant.ofEpochMilli(123), true,
+                                            true, true,
+                                            nodeVersions(Version.emptyVersion, Version.emptyVersion,
+                                                         Instant.EPOCH, "cfg1", "cfg2", "cfg3"),
+                                            VespaVersion.Confidence.normal);
+
+        VespaVersion deserialized = deserializedStatus.versions().get(0);
+        assertEquals(vespaVersion.releaseCommit(), deserialized.releaseCommit());
+        assertEquals(vespaVersion.committedAt().truncatedTo(MILLIS), deserialized.committedAt());
+        assertEquals(vespaVersion.isControllerVersion(), deserialized.isControllerVersion());
+        assertEquals(vespaVersion.isSystemVersion(), deserialized.isSystemVersion());
+        assertEquals(vespaVersion.isReleased(), deserialized.isReleased());
+        assertEquals(vespaVersion.statistics(), deserialized.statistics());
+        assertEquals(vespaVersion.nodeVersions(), deserialized.nodeVersions());
+        assertEquals(vespaVersion.confidence(), deserialized.confidence());
+    }
+
+    private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) {
+        var nodeVersions = new ArrayList<NodeVersion>();
+        for (var hostname : hostnames) {
+            nodeVersions.add(new NodeVersion(HostName.from(hostname), version, wantedVersion, changedAt));
+        }
+        return NodeVersions.EMPTY.with(nodeVersions);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
index 8ab277a3795..1c660726d61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
@@ -17,11 +17,11 @@
   "queryQuality": 100,
   "writeQuality": 99.99894341115082,
   "pemDeployKeys": [
-    "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"
+    "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----"
   ],
   "pemDeveloperKeys": [
     {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----",
       "user": "joe@dev"
     }
   ],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json
new file mode 100644
index 00000000000..96ca22e1c1a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json
@@ -0,0 +1,23 @@
+{
+  "versions": [
+    {
+      "releaseCommit": "badc0ffee",
+      "releasedAt": 123,
+      "isCurrentControllerVersion": true,
+      "isCurrentSystemVersion": true,
+      "isReleased": true,
+      "deploymentStatistics": {
+        "version": "7.0",
+        "failing": [],
+        "production": [],
+        "deploying": []
+      },
+      "confidence": "normal",
+      "configServerHostnames": [
+        "cfg1",
+        "cfg2",
+        "cfg3"
+      ]
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
index 2d8c937097a..80e52f373d7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
@@ -79,8 +79,10 @@ public class ContainerControllerTester {
                                                            Optional.of(new PropertyId("1234")));
         controller().tenants().create(tenantSpec, credentials);
 
-        ApplicationId app = ApplicationId.from(tenant, application, instance);
-        return controller().applications().createApplication(app, Optional.of(credentials));
+        TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application);
+        controller().applications().createApplication(id, Optional.of(credentials));
+        controller().applications().createInstance(id.instance(instance));
+        return controller().applications().requireApplication(id);
     }
 
     public void deploy(ApplicationId id, ApplicationPackage applicationPackage, ZoneId zone) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 3ff21bb2261..307496ace5a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -29,6 +29,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
@@ -49,11 +52,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
@@ -111,6 +111,11 @@ import static org.junit.Assert.assertTrue;
 public class ApplicationApiTest extends ControllerContainerTest {
 
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/";
+    private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                               "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                               "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                               "-----END PUBLIC KEY-----\n";
+    private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n");
 
     private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder()
             .instances("default")
@@ -320,7 +325,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                 .region("us-west-1")
                 .build();
 
-        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", POST)
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", POST)
                                       .userIdentity(USER_ID)
                                       .oktaAccessToken(OKTA_AT),
                               new File("application-reference-2.json"));
@@ -360,14 +365,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // POST a pem deploy key
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST)
                                       .userIdentity(USER_ID)
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"keys\":[\"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\"]}");
 
         // PATCH in a pem deploy key at deprecated path
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH)
                                       .userIdentity(USER_ID)
-                                      .data("{\"pemDeployKey\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"pemDeployKey\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}");
 
         // GET an application with a major version override
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
@@ -383,8 +388,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // DELETE the pem deploy key
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE)
                                       .userIdentity(USER_ID)
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"keys\":[]}");
 
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
                                       .userIdentity(USER_ID),
@@ -556,6 +561,11 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .oktaAccessToken(OKTA_AT),
                               new File("delete-with-active-deployments.json"), 400);
 
+        // GET test-config for local tests against a prod deployment
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/test-config", GET)
+                                      .userIdentity(USER_ID),
+                              new File("test-config.json"));
+
         // DELETE (deactivate) a deployment - dev
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE)
                                       .userIdentity(USER_ID),
@@ -1061,7 +1071,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST)
                                       .oktaAccessToken(OKTA_AT)
                                       .userIdentity(USER_ID),
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Application already exists\"}",
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Instance already exists\"}",
                               400);
 
         ConfigServerMock configServer = serviceRegistry().configServerMock();
@@ -1111,7 +1121,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE)
                                       .oktaAccessToken(OKTA_AT)
                                       .userIdentity(USER_ID),
-                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1.instance1': Application not found\"}",
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete instance 'tenant1.application1.instance1': Instance not found\"}",
                               404);
 
         // DELETE tenant
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
index 331aabd32d0..9d76654fbc0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
@@ -80,8 +80,8 @@
   "majorVersion": 7,
   "globalRotations": [],
   "instances": [],
-  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
-  "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"],
+  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
+  "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n"],
   "metrics": {
     "queryServiceQuality": 0.0,
     "writeServiceQuality": 0.0
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
index 25948e998f1..d62e39e42e7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
@@ -28,40 +28,10 @@
     "lastWritesPerSecond": 2.0
   },
   "cost": {
-    "tco": 74,
+    "tco": 0,
     "waste": 0,
-    "utilization": 2.9999999999999996,
-    "cluster": {
-      "cluster1": {
-        "count": 2,
-        "resource": "cpu",
-        "utilization": 2.9999999999999996,
-        "tco": 74,
-        "waste": 0,
-        "flavor": "flavor1",
-        "flavorCost":37.0,
-        "flavorCpu":2.0,
-        "flavorMem":4.0,
-        "flavorDisk":50.0,
-        "type": "content",
-        "util": {
-          "cpu": 2.9999999999999996,
-          "mem": 0.4285714285714286,
-          "disk": 0.5714285714285715,
-          "diskBusy": 1.0
-        },
-        "usage": {
-          "cpu": 0.6,
-          "mem": 0.3,
-          "disk": 0.4,
-          "diskBusy": 0.3
-        },
-        "hostnames": [
-          "host1",
-          "host2"
-        ]
-      }
-    }
+    "utilization": 0.0,
+    "cluster": {}
   },
   "metrics": {
     "queriesPerSecond": 1.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
index 1a2025e4de2..c56a269b9d4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
@@ -25,40 +25,10 @@
     "lastWritesPerSecond": 2.0
   },
   "cost": {
-    "tco": 74,
+    "tco": 0,
     "waste": 0,
-    "utilization": 2.9999999999999996,
-    "cluster": {
-      "cluster1": {
-        "count": 2,
-        "resource": "cpu",
-        "utilization": 2.9999999999999996,
-        "tco": 74,
-        "waste": 0,
-        "flavor": "flavor1",
-        "flavorCost": 37.0,
-        "flavorCpu": 2.0,
-        "flavorMem": 4.0,
-        "flavorDisk": 50.0,
-        "type": "content",
-        "util": {
-          "cpu": 2.9999999999999996,
-          "mem": 0.4285714285714286,
-          "disk": 0.5714285714285715,
-          "diskBusy": 1.0
-        },
-        "usage": {
-          "cpu": 0.6,
-          "mem": 0.3,
-          "disk": 0.4,
-          "diskBusy": 0.3
-        },
-        "hostnames": [
-          "host1",
-          "host2"
-        ]
-      }
-    }
+    "utilization": 0.0,
+    "cluster": {}
   },
   "metrics": {
     "queriesPerSecond": 1.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
index bb68904bee6..140be562fe9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
@@ -37,40 +37,10 @@
     "lastWritesPerSecond": 2.0
   },
   "cost": {
-    "tco": 74,
+    "tco": 0,
     "waste": 0,
-    "utilization": 2.9999999999999996,
-    "cluster": {
-      "cluster1": {
-        "count": 2,
-        "resource": "cpu",
-        "utilization": 2.9999999999999996,
-        "tco": 74,
-        "waste": 0,
-        "flavor": "flavor1",
-        "flavorCost": 37.0,
-        "flavorCpu": 2.0,
-        "flavorMem": 4.0,
-        "flavorDisk": 50.0,
-        "type": "content",
-        "util": {
-          "cpu": 2.9999999999999996,
-          "mem": 0.4285714285714286,
-          "disk": 0.5714285714285715,
-          "diskBusy": 1.0
-        },
-        "usage": {
-          "cpu": 0.6,
-          "mem": 0.3,
-          "disk": 0.4,
-          "diskBusy": 0.3
-        },
-        "hostnames": [
-          "host1",
-          "host2"
-        ]
-      }
-    }
+    "utilization": 0.0,
+    "cluster": {}
   },
   "metrics": {
     "queriesPerSecond": 1.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
new file mode 100644
index 00000000000..2338543b019
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
@@ -0,0 +1,20 @@
+{
+  "application": "tenant1:application1:instance1",
+  "zone": "prod.us-central-1",
+  "system": "main",
+  "endpoints": {
+    "prod.us-central-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ]
+  },
+  "zoneEndpoints": {
+    "prod.us-central-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    }
+  },
+  "clusters": {
+    "prod.us-central-1": [
+      "music"
+    ]
+  }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 084b235943e..0a4d046e318 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -1,26 +1,26 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
 import java.io.File;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * @author bratseth
@@ -69,16 +69,15 @@ public class DeploymentApiTest extends ControllerContainerTest {
     private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) {
         List<VespaVersion> censored = new ArrayList<>();
         for (VespaVersion version : versionStatus.versions()) {
-            if (!version.systemApplicationHostnames().isEmpty()) {
+            if (version.nodeVersions().size() > 0) {
                 version = new VespaVersion(version.statistics(),
                                            version.releaseCommit(),
                                            version.committedAt(),
                                            version.isControllerVersion(),
                                            version.isSystemVersion(),
                                            version.isReleased(),
-                                           ImmutableSet.of("config1.test", "config2.test").stream()
-                                                       .map(HostName::from)
-                                                       .collect(Collectors.toSet()),
+                                           NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH),
+                                                                           new NodeVersion(HostName.from("config2.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH))),
                                            VespaVersion.confidenceFrom(version.statistics(), controller)
                 );
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
index 3b7d55f8cef..0a1e996696b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
@@ -3,15 +3,21 @@ package com.yahoo.vespa.hosted.controller.restapi.filter;
 
 import ai.vespa.hosted.api.Method;
 import ai.vespa.hosted.api.RequestSigner;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -19,6 +25,8 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.http.HttpRequest;
+import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -27,21 +35,21 @@ import static org.junit.Assert.assertTrue;
 
 public class SignatureFilterTest {
 
-    private static final String publicKey = "-----BEGIN PUBLIC KEY-----\n" +
-                                                 "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
-                                                 "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
-                                                 "-----END PUBLIC KEY-----\n";
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
 
-    private static final String otherPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
-                                                 "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
-                                                 "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
-                                                 "-----END PUBLIC KEY-----\n";
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
 
-    private static final String privateKey = "-----BEGIN EC PRIVATE KEY-----\n" +
-                                                  "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" +
-                                                  "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" +
-                                                  "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
-                                                  "-----END EC PRIVATE KEY-----\n";
+    private static final PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey("-----BEGIN EC PRIVATE KEY-----\n" +
+                                                                                   "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" +
+                                                                                   "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" +
+                                                                                   "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                   "-----END EC PRIVATE KEY-----\n");
 
     private static final TenantAndApplicationId appId = TenantAndApplicationId.from("my-tenant", "my-app");
     private static final ApplicationId id = appId.defaultInstance();
@@ -58,10 +66,10 @@ public class SignatureFilterTest {
         filter = new SignatureFilter(tester.controller());
         signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock());
 
-        tester.createApplication(tester.createTenant(id.tenant().value(), "unused", 496L),
-                                 id.application().value(),
-                                 id.instance().value(),
-                                 28L);
+        tester.curator().writeTenant(new CloudTenant(appId.tenant(),
+                                                     new BillingInfo("id", "code"),
+                                                     ImmutableBiMap.of()));
+        tester.curator().writeApplication(new Application(appId, tester.clock().instant()));
     }
 
     @Test
@@ -69,42 +77,57 @@ public class SignatureFilterTest {
         // Unsigned request gets no role.
         HttpRequest.Builder request = HttpRequest.newBuilder(URI.create("https://host:123/path/./..//..%2F?query=empty&%3F=%26"));
         byte[] emptyBody = new byte[0];
-        DiscFilterRequest unsigned = requestOf(request.method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody);
-        filter.filter(unsigned);
-        assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME));
+        verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody),
+                              null);
 
         // Signed request gets no role when no key is stored for the application.
-        DiscFilterRequest signed = requestOf(signer.signed(request, Method.GET, InputStream::nullInputStream), emptyBody);
-        filter.filter(signed);
-        assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME));
-
-        // Signed request gets no role when a non-matching key is stored for the application.
-        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(otherPublicKey)));
-        filter.filter(signed);
-        assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME));
-
-        // Signed request gets a build service role when a matching key is stored for the application.
-        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(publicKey)));
-        assertTrue(filter.filter(signed).isEmpty());
-        SecurityContext securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME);
-        assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName());
-        assertEquals(Set.of(Role.buildService(id.tenant(), id.application()),
-                            Role.applicationDeveloper(id.tenant(), id.application())),
-                     securityContext.roles());
-
-        // Signed POST request also gets a build service role.
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                null);
+
+        // Signed request gets no role when only non-matching keys are stored for the application.
+        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(otherPublicKey)));
+        // Signed request gets no role when no key is stored for the application.
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              null);
+
+        // Signed request gets a headless role when a matching key is stored for the application.
+        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(publicKey)));
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
+
+        // TODO jonmv: remove after Oct 2019.
+        // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided.
+        verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant()))));
+
+        // Signed POST request with X-Key header gets a headless role.
         byte[] hiBytes = new byte[]{0x48, 0x69};
-        signed = requestOf(signer.signed(request, Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes);
-        filter.filter(signed);
-        securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME);
-        assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName());
-        assertEquals(Set.of(Role.buildService(id.tenant(), id.application()),
-                            Role.applicationDeveloper(id.tenant(), id.application())),
-                     securityContext.roles());
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
+
+        // Signed request gets a developer role when a matching developer key is stored for the tenant.
+        tester.curator().writeTenant(new CloudTenant(appId.tenant(),
+                                                     new BillingInfo("id", "code"),
+                                                     ImmutableBiMap.of(publicKey, () -> "user")));
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
+                              new SecurityContext(new SimplePrincipal("user"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant()))));
 
         // Unsigned requests still get no roles.
-        filter.filter(unsigned);
-        assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME));
+        verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody),
+                              null);
+    }
+
+    private void verifySecurityContext(DiscFilterRequest request, SecurityContext securityContext) {
+        assertTrue(filter.filter(request).isEmpty());
+        assertEquals(securityContext, request.getAttribute(SecurityContext.ATTRIBUTE_NAME));
     }
 
     private static DiscFilterRequest requestOf(HttpRequest request, byte[] body) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
index 17f90259fa8..01af1bd70dd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
@@ -6,92 +6,92 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -103,47 +103,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
index 86bc272fcd1..dbaa6623fae 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
@@ -6,92 +6,92 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -103,47 +103,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
index e8007fbf6c5..2b907c1156c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
@@ -6,47 +6,47 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -58,47 +58,47 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         }
@@ -110,47 +110,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
index b17dda7f810..b1f5f33b960 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
@@ -12,7 +12,6 @@ import java.io.File;
 import java.util.Set;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
-import static com.yahoo.application.container.handler.Request.Method.PATCH;
 import static com.yahoo.application.container.handler.Request.Method.POST;
 import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static org.junit.Assert.assertEquals;
@@ -23,6 +22,17 @@ import static org.junit.Assert.assertEquals;
 public class UserApiTest extends ControllerContainerCloudTest {
 
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/";
+    private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                               "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                               "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                               "-----END PUBLIC KEY-----\n";
+    private static final String otherPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                                    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                    "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                    "-----END PUBLIC KEY-----\n";
+    private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n");
+    private static final String otherQuotedPemPublicKey = otherPemPublicKey.replaceAll("\\n", "\\\\n");
+
 
     @Test
     public void testUserManagement() {
@@ -132,30 +142,30 @@ public class UserApiTest extends ControllerContainerCloudTest {
         // POST a pem deploy key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST)
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              new File("first-deploy-key.json"));
 
         // POST a pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("joe@dev")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              new File("first-developer-key.json"));
 
         // POST the same pem developer key for a different user is forbidden
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("operator@tenant")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Multiple entries with same key: -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=operator@tenant and -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=joe@dev\"}",
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Key "+  quotedPemPublicKey + " is already owned by joe@dev\"}",
                               400);
 
         // PATCH in a different pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("operator@tenant")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\nƪ(`▿▿▿▿´ƪ)\\n-----END PUBLIC KEY----- for operator@tenant\"}");
+                                      .data("{\"key\":\"" + otherPemPublicKey + "\"}"),
+                              new File("both-developer-keys.json"));
 
         // GET tenant information with keys
         tester.assertResponse(request("/application/v4/tenant/my-tenant/")
@@ -165,8 +175,8 @@ public class UserApiTest extends ControllerContainerCloudTest {
         // DELETE a pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE)
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Removed developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              new File("second-developer-key.json"));
 
         // DELETE an application role is allowed for an application admin.
         tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE)
@@ -180,8 +190,7 @@ public class UserApiTest extends ControllerContainerCloudTest {
                               "{\"message\":\"Deleted application my-tenant.my-app\"}");
 
         // DELETE a tenant role is available to tenant admins.
-        // DELETE the tenantOperator role clears any developer key.
-        // TODO jonmv: Change to developer, when this role exists.
+        // DELETE the developer role clears any developer key.
         tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE)
                                       .roles(Set.of(Role.tenantAdmin(id.tenant())))
                                       .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json
index 6cf4dc76173..31bdb07b26b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json
@@ -1,6 +1,5 @@
 {
   "tenant": "my-tenant",
   "application": "my-app",
-  "instance": "default",
-  "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default"
+  "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app"
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json
new file mode 100644
index 00000000000..2ff1c29fe29
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json
@@ -0,0 +1,12 @@
+{
+  "keys": [
+    {
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
+      "user": "joe@dev"
+    },
+    {
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n",
+      "user": "operator@tenant"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json
new file mode 100644
index 00000000000..1c86877b77d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json
@@ -0,0 +1,5 @@
+{
+  "keys": [
+    "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n"
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json
new file mode 100644
index 00000000000..b7d48f283f3
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json
@@ -0,0 +1,9 @@
+{
+  "keys": [
+    {
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
+      "user": "joe@dev"
+    }
+  ]
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json
new file mode 100644
index 00000000000..f7d90f31116
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json
@@ -0,0 +1,8 @@
+{
+  "keys": [
+    {
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n",
+      "user": "operator@tenant"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
index 5aaa900c3f0..b7970a48963 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
@@ -1,27 +1,14 @@
 {
   "tenant": "my-tenant",
   "type": "CLOUD",
-  "pemDeployKeys": [
-    {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
-      "application": "my-app"
-    }
-  ],
   "pemDeveloperKeys": [
     {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
       "user": "joe@dev"
     },
     {
-      "key": "-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n",
       "user": "operator@tenant"
     }],
-  "applications": [
-    {
-      "tenant":"my-tenant",
-      "application":"my-app",
-      "instance":"default",
-      "url":"http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default"
-    }
-  ]
+  "applications": []
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
index a89a0f5360c..39b6cccbab0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
@@ -1,7 +1,6 @@
 {
   "tenant": "my-tenant",
   "type": "CLOUD",
-  "pemDeployKeys": [],
   "pemDeveloperKeys": [],
   "applications": []
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index ba8309de286..83223b0e041 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;