Merge pull request #10851 from vespa-engine/jvenstad/differentiate-between-deploy-and-developer-keys

Jvenstad/differentiate between deploy and developer keys

22 files changed, 387 insertions, 222 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
index f36107db228..606db8a0f2f 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
@@ -58,6 +58,26 @@ public abstract class Role {
         return new TenantRole(RoleDefinition.tenantOperator, tenant);
     }
 
+    /** Returns a {@link RoleDefinition#reader} for the current system and given tenant. */
+    public static TenantRole reader(TenantName tenant) {
+        return new TenantRole(RoleDefinition.reader, tenant);
+    }
+
+    /** Returns a {@link RoleDefinition#developer} for the current system and given tenant. */
+    public static TenantRole developer(TenantName tenant) {
+        return new TenantRole(RoleDefinition.developer, tenant);
+    }
+
+    /** Returns a {@link RoleDefinition#administrator} for the current system and given tenant. */
+    public static TenantRole administrator(TenantName tenant) {
+        return new TenantRole(RoleDefinition.administrator, tenant);
+    }
+
+    /** Returns a {@link RoleDefinition#headless} for the current system, given tenant, and application */
+    public static ApplicationRole headless(TenantName tenant, ApplicationName application) {
+        return new ApplicationRole(RoleDefinition.headless, tenant, application);
+    }
+
     /** Returns a {@link RoleDefinition#applicationAdmin} for the current system and given tenant and application. */
     public static ApplicationRole applicationAdmin(TenantName tenant, ApplicationName application) {
         return new ApplicationRole(RoleDefinition.applicationAdmin, tenant, application);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
index 7bbd89404c7..8e3754777ea 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
@@ -70,6 +70,29 @@ public enum RoleDefinition {
     tenantOwner(tenantAdmin,
                 Policy.tenantDelete),
 
+    /** Reader — the base role for all tenant users */
+    reader(Policy.tenantRead,
+           Policy.applicationRead,
+           Policy.deploymentRead,
+           Policy.publicRead),
+
+    /** User — the dev.ops. role for normal Vespa tenant users */
+    developer(Policy.applicationCreate,
+              Policy.applicationUpdate,
+              Policy.applicationDelete,
+              Policy.applicationOperations,
+              Policy.developmentDeployment,
+              Policy.keyManagement,
+              Policy.submission),
+
+    /** Admin — the administrative function for user management etc. */
+    administrator(Policy.tenantUpdate,
+                  Policy.tenantManager,
+                  Policy.applicationManager),
+
+    /** Headless — the application specific role identified by deployment keys for production */
+    headless(Policy.submission),
+
     /** Build and continuous delivery service. */ // TODO replace with buildService, when everyone is on new pipeline.
     tenantPipeline(everyone,
                    Policy.submission,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index c17ac044136..c83f366cb67 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
+import java.security.PublicKey;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Comparator;
@@ -51,7 +52,7 @@ public class Application {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final Set<String> pemDeployKeys;
+    private final Set<PublicKey> deployKeys;
     private final Map<InstanceName, Instance> instances;
 
     /** Creates an empty application. */
@@ -64,7 +65,7 @@ public class Application {
     // DO NOT USE! For serialization purposes, only.
     public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        Change change, Change outstandingChange, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                       OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                       OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
                        OptionalLong projectId, boolean internal, Collection<Instance> instances) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
@@ -77,7 +78,7 @@ public class Application {
         this.owner = Objects.requireNonNull(owner, "owner cannot be null");
         this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null");
         this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
-        this.pemDeployKeys = Objects.requireNonNull(pemDeployKeys, "pemDeployKeys cannot be null");
+        this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null");
         this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null");
         this.internal = internal;
         this.instances = ImmutableSortedMap.copyOf(instances.stream().collect(Collectors.toMap(Instance::name, Function.identity())));
@@ -191,7 +192,7 @@ public class Application {
     }
 
     /** Returns the set of deploy keys for this application. */
-    public Set<String> pemDeployKeys() { return pemDeployKeys; }
+    public Set<PublicKey> deployKeys() { return deployKeys; }
 
     @Override
     public boolean equals(Object o) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 5aa5a8e13de..19921595dc2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -11,11 +11,10 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 
+import java.security.PublicKey;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -43,7 +42,7 @@ public class LockedApplication {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final Set<String> pemDeployKeys;
+    private final Set<PublicKey> deployKeys;
     private final OptionalLong projectId;
     private final boolean internal;
     private final Map<InstanceName, Instance> instances;
@@ -58,14 +57,14 @@ public class LockedApplication {
         this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(),
              application.deploymentSpec(), application.validationOverrides(), application.change(),
              application.outstandingChange(), application.deploymentIssueId(), application.ownershipIssueId(),
-             application.owner(), application.majorVersion(), application.metrics(), application.pemDeployKeys(),
+             application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(),
              application.projectId(), application.internal(), application.instances());
     }
 
     private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec,
                               ValidationOverrides validationOverrides, Change change, Change outstandingChange,
                               Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                              OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                              OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
                               OptionalLong projectId, boolean internal,
                               Map<InstanceName, Instance> instances) {
         this.lock = lock;
@@ -80,7 +79,7 @@ public class LockedApplication {
         this.owner = owner;
         this.majorVersion = majorVersion;
         this.metrics = metrics;
-        this.pemDeployKeys = pemDeployKeys;
+        this.deployKeys = deployKeys;
         this.projectId = projectId;
         this.internal = internal;
         this.instances = Map.copyOf(instances);
@@ -89,7 +88,7 @@ public class LockedApplication {
     /** Returns a read-only copy of this */
     public Application get() {
         return new Application(id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                               deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                               deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                projectId, internal, instances.values());
     }
 
@@ -97,7 +96,7 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.put(instance, new Instance(id.instance(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -105,7 +104,7 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.put(instance, modification.apply(instances.get(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -113,61 +112,61 @@ public class LockedApplication {
         var instances = new HashMap<>(this.instances);
         instances.remove(instance);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withBuiltInternally(boolean builtInternally) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, builtInternally, instances);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withChange(Change change) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOutstandingChange(Change outstandingChange) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
     public LockedApplication withOwner(User owner) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
@@ -176,25 +175,25 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner,
                                      majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion),
-                                     metrics, pemDeployKeys, projectId, internal, instances);
+                                     metrics, deployKeys, projectId, internal, instances);
     }
 
     public LockedApplication with(ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
-                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys,
+                                     deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
                                      projectId, internal, instances);
     }
 
-    public LockedApplication withPemDeployKey(String pemDeployKey) {
-        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
+    public LockedApplication withDeployKey(PublicKey pemDeployKey) {
+        Set<PublicKey> keys = new LinkedHashSet<>(deployKeys);
         keys.add(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
                                      projectId, internal, instances);
     }
 
-    public LockedApplication withoutPemDeployKey(String pemDeployKey) {
-        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
+    public LockedApplication withoutDeployKey(PublicKey pemDeployKey) {
+        Set<PublicKey> keys = new LinkedHashSet<>(deployKeys);
         keys.remove(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
index ecc8bd65b72..6caf716aed4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
@@ -2,8 +2,10 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
@@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.Optional;
 
 import static java.util.Objects.requireNonNull;
@@ -126,44 +129,39 @@ public abstract class LockedTenant {
     public static class Cloud extends LockedTenant {
 
         private final BillingInfo billingInfo;
-        private final BiMap<String, Principal> pemDeveloperKeys;
+        private final BiMap<PublicKey, Principal> developerKeys;
 
-        private Cloud(TenantName name, BillingInfo billingInfo, BiMap<String, Principal> pemDeveloperKeys) {
+        private Cloud(TenantName name, BillingInfo billingInfo, BiMap<PublicKey, Principal> developerKeys) {
             super(name);
             this.billingInfo = billingInfo;
-            this.pemDeveloperKeys = pemDeveloperKeys;
+            this.developerKeys = ImmutableBiMap.copyOf(developerKeys);
         }
 
         private Cloud(CloudTenant tenant) {
-            this(tenant.name(), tenant.billingInfo(), tenant.pemDeveloperKeys());
+            this(tenant.name(), tenant.billingInfo(), tenant.developerKeys());
         }
 
         @Override
         public CloudTenant get() {
-            return new CloudTenant(name, billingInfo, pemDeveloperKeys);
+            return new CloudTenant(name, billingInfo, developerKeys);
         }
 
         public Cloud with(BillingInfo billingInfo) {
-            return new Cloud(name, billingInfo, pemDeveloperKeys);
-        }
-
-        public Cloud withPemDeveloperKey(String pemKey, Principal principal) {
-            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-            pemDeveloperKeys.forEach((key, user) -> {
-                if ( ! user.equals(principal))
-                    keys.put(key, user);
-            });
-            keys.put(pemKey, principal);
-            return new Cloud(name, billingInfo, keys.build());
-        }
-
-        public Cloud withoutPemDeveloperKey(String pemKey) {
-            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-            pemDeveloperKeys.forEach((key, user) -> {
-                if ( ! key.equals(pemKey))
-                    keys.put(key, user);
-            });
-            return new Cloud(name, billingInfo, keys.build());
+            return new Cloud(name, billingInfo, developerKeys);
+        }
+
+        public Cloud withDeveloperKey(PublicKey key, Principal principal) {
+            BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys);
+            if (keys.containsKey(key))
+                throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key));
+            keys.put(key, principal);
+            return new Cloud(name, billingInfo, keys);
+        }
+
+        public Cloud withoutDeveloperKey(PublicKey key) {
+            BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys);
+            keys.remove(key);
+            return new Cloud(name, billingInfo, keys);
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 08b3355587f..8b6dc74fb87 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -1,14 +1,13 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -21,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -39,7 +37,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
-import java.security.Principal;
+import java.security.PublicKey;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -54,7 +52,6 @@ import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Serializes {@link Application}s to/from slime.
@@ -192,7 +189,7 @@ public class ApplicationSerializer {
         application.majorVersion().ifPresent(majorVersion -> root.setLong(majorVersionField, majorVersion));
         root.setDouble(queryQualityField, application.metrics().queryServiceQuality());
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
-        deployKeysToSlime(application.pemDeployKeys().stream(), root.setArray(pemDeployKeysField));
+        deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField));
         instancesToSlime(application, root.setArray(instancesField));
         return slime;
     }
@@ -208,8 +205,8 @@ public class ApplicationSerializer {
         }
     }
 
-    private void deployKeysToSlime(Stream<String> pemDeployKeys, Cursor array) {
-        pemDeployKeys.forEach(array::addString);
+    private void deployKeysToSlime(Set<PublicKey> deployKeys, Cursor array) {
+        deployKeys.forEach(key -> array.addString(KeyUtils.toPem(key)));
     }
 
     private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) {
@@ -384,14 +381,14 @@ public class ApplicationSerializer {
         OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField));
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
-        Set<String> pemDeployKeys = pemDeployKeysFromSlime(root.field(pemDeployKeysField));
+        Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField));
         List<Instance> instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField));
         OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField));
         boolean builtInternally = root.field(builtInternallyField).asBool();
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides, deploying, outstandingChange,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics,
-                               pemDeployKeys, projectId, builtInternally, instances);
+                               deployKeys, projectId, builtInternally, instances);
     }
 
     private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) {
@@ -411,9 +408,9 @@ public class ApplicationSerializer {
         return instances;
     }
 
-    private Set<String> pemDeployKeysFromSlime(Inspector array) {
-        Set<String> keys = new LinkedHashSet<>();
-        array.traverse((ArrayTraverser) (__, key) -> keys.add(key.asString()));
+    private Set<PublicKey> deployKeysFromSlime(Inspector array) {
+        Set<PublicKey> keys = new LinkedHashSet<>();
+        array.traverse((ArrayTraverser) (__, key) -> keys.add(KeyUtils.fromPemEncodedPublicKey(key.asString())));
         return keys;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index 78d166607df..35128466e4d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.net.URI;
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -91,14 +93,14 @@ public class TenantSerializer {
     }
 
     private void toSlime(CloudTenant tenant, Cursor root) {
-        pemDeveloperKeysToSlime(tenant.pemDeveloperKeys(), root.setArray(pemDeveloperKeysField));
+        developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField));
         toSlime(tenant.billingInfo(), root.setObject(billingInfoField));
     }
 
-    private void pemDeveloperKeysToSlime(BiMap<String, Principal> keys, Cursor array) {
+    private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) {
         keys.forEach((key, user) -> {
             Cursor object = array.addObject();
-            object.setString("key", key);
+            object.setString("key", KeyUtils.toPem(key));
             object.setString("user", user.getName());
         });
     }
@@ -139,15 +141,16 @@ public class TenantSerializer {
     private CloudTenant cloudTenantFrom(Inspector tenantObject) {
         TenantName name = TenantName.from(tenantObject.field(nameField).asString());
         BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField));
-        BiMap<String, Principal> pemDeveloperKeys = pemDeveloperKeysFromSlime(tenantObject.field(pemDeveloperKeysField));
-        return new CloudTenant(name, billingInfo, pemDeveloperKeys);
+        BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField));
+        return new CloudTenant(name, billingInfo, developerKeys);
     }
 
-    private BiMap<String, Principal> pemDeveloperKeysFromSlime(Inspector array) {
-        ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
-        array.traverse((ArrayTraverser) (__, keyObject) -> {
-            keys.put(keyObject.field("key").asString(), new SimplePrincipal(keyObject.field("user").asString()));
-        });
+    private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) {
+        ImmutableBiMap.Builder<PublicKey, Principal> keys = ImmutableBiMap.builder();
+        array.traverse((ArrayTraverser) (__, keyObject) ->
+                keys.put(KeyUtils.fromPemEncodedPublicKey(keyObject.field("key").asString()),
+                         new SimplePrincipal(keyObject.field("user").asString())));
+
         return keys.build();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 4c4478c9af6..96d21f113a4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -17,6 +17,7 @@ import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
 import com.yahoo.restapi.Path;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
@@ -96,6 +97,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.DigestInputStream;
 import java.security.Principal;
+import java.security.PublicKey;
 import java.time.DayOfWeek;
 import java.time.Duration;
 import java.time.Instant;
@@ -377,8 +379,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Principal user = request.getJDiscRequest().getUserPrincipal();
         String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
+        PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey);
         controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
-                controller.tenants().store(tenant.withPemDeveloperKey(pemDeveloperKey, user)));
+                controller.tenants().store(tenant.withDeveloperKey(developerKey, user)));
         return new MessageResponse("Set developer key " + pemDeveloperKey + " for " + user);
     }
 
@@ -387,25 +390,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant");
 
         String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
-        Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).pemDeveloperKeys().get(pemDeveloperKey);
+        PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey);
+        Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).developerKeys().get(developerKey);
         controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
-                controller.tenants().store(tenant.withoutPemDeveloperKey(pemDeveloperKey)));
+                controller.tenants().store(tenant.withoutDeveloperKey(developerKey)));
         return new MessageResponse("Removed developer key " + pemDeveloperKey + " for " + user);
     }
 
     private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) {
         String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
-        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
-            controller.applications().store(application.withPemDeployKey(pemDeployKey));
-        });
+        PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application ->
+                controller.applications().store(application.withDeployKey(deployKey)));
+
         return new MessageResponse("Added deploy key " + pemDeployKey);
     }
 
     private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) {
         String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
-        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
-            controller.applications().store(application.withoutPemDeployKey(pemDeployKey));
-        });
+        PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application ->
+                controller.applications().store(application.withoutDeployKey(deployKey)));
+
         return new MessageResponse("Removed deploy key " + pemDeployKey);
     }
 
@@ -424,7 +430,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             Inspector pemDeployKeyField = requestObject.field("pemDeployKey");
             if (pemDeployKeyField.valid()) {
                 String pemDeployKey = pemDeployKeyField.asString();
-                application = application.withPemDeployKey(pemDeployKey);
+                PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey);
+                application = application.withDeployKey(deployKey);
                 messageBuilder.add("Added deploy key " + pemDeployKey);
             }
 
@@ -654,9 +661,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
 
         // TODO jonmv: Remove when clients are updated
-        application.pemDeployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", key));
+        application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key)));
 
-        application.pemDeployKeys().forEach(object.setArray("pemDeployKeys")::addString);
+        application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString);
 
         // Metrics
         Cursor metricsObject = object.setObject("metrics");
@@ -1366,18 +1373,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             case cloud: {
                 CloudTenant cloudTenant = (CloudTenant) tenant;
 
-                Cursor pemDeployKeysArray = object.setArray("pemDeployKeys");
-                for (Application application : applications)
-                    for (String key : application.pemDeployKeys()) {
-                        Cursor keyObject = pemDeployKeysArray.addObject();
-                        keyObject.setString("key", key);
-                        keyObject.setString("application", application.id().application().value());
-                    }
-
                 Cursor pemDeveloperKeysArray = object.setArray("pemDeveloperKeys");
-                cloudTenant.pemDeveloperKeys().forEach((key, user) -> {
+                cloudTenant.developerKeys().forEach((key, user) -> {
                     Cursor keyObject = pemDeveloperKeysArray.addObject();
-                    keyObject.setString("key", key);
+                    keyObject.setString("key", KeyUtils.toPem(key));
                     keyObject.setString("user", user.getName());
                 });
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
index 6755110bb49..7ad2e03ef1d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
@@ -10,19 +10,27 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
 import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase;
 import com.yahoo.log.LogLevel;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.yolean.Exceptions;
 
 import java.security.Principal;
+import java.security.PublicKey;
+import java.util.Base64;
+import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.logging.Logger;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * Assigns the {@link Role#buildService(TenantName, ApplicationName)} role to requests with a
  * Authorization header signature matching the public key of the indicated application.
@@ -46,25 +54,11 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
         if (   request.getAttribute(SecurityContext.ATTRIBUTE_NAME) == null
             && request.getHeader("X-Authorization") != null)
             try {
-                ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
-                boolean verified = controller.applications().getApplication(TenantAndApplicationId.from(id)).stream()
-                                             .flatMap(application -> application.pemDeployKeys().stream())
-                                             .map(key -> new RequestVerifier(key, controller.clock()))
-                                             .anyMatch(verifier -> verifier.verify(Method.valueOf(request.getMethod()),
-                                                                                   request.getUri(),
-                                                                                   request.getHeader("X-Timestamp"),
-                                                                                   request.getHeader("X-Content-Hash"),
-                                                                                   request.getHeader("X-Authorization")));
-
-                if (verified) {
-                    Principal principal = new SimplePrincipal("buildService@" + id.tenant() + "." + id.application());
-                    request.setUserPrincipal(principal);
-                    request.setRemoteUser(principal.getName());
-                    request.setAttribute(SecurityContext.ATTRIBUTE_NAME,
-                                         new SecurityContext(principal,
-                                                             Set.of(Role.buildService(id.tenant(), id.application()),
-                                                                    Role.applicationDeveloper(id.tenant(), id.application()))));
-                }
+                getSecurityContext(request).ifPresent(securityContext -> {
+                    request.setUserPrincipal(securityContext.principal());
+                    request.setRemoteUser(securityContext.principal().getName());
+                    request.setAttribute(SecurityContext.ATTRIBUTE_NAME, securityContext);
+                });
             }
             catch (Exception e) {
                 logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e));
@@ -72,4 +66,48 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
         return Optional.empty();
     }
 
+    // TODO jonmv: Remove after October 2019.
+    private boolean anyDeployKeyMatches(TenantAndApplicationId id, DiscFilterRequest request) {
+        return controller.applications().getApplication(id).stream()
+                         .map(Application::deployKeys)
+                         .flatMap(Set::stream)
+                         .anyMatch(key -> keyVerifies(key, request));
+    }
+
+    private boolean keyVerifies(PublicKey key, DiscFilterRequest request) {
+        return new RequestVerifier(key, controller.clock()).verify(Method.valueOf(request.getMethod()),
+                                                                   request.getUri(),
+                                                                   request.getHeader("X-Timestamp"),
+                                                                   request.getHeader("X-Content-Hash"),
+                                                                   request.getHeader("X-Authorization"));
+    }
+
+    private Optional<SecurityContext> getSecurityContext(DiscFilterRequest request) {
+        ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
+        if (request.getHeader("X-Key") != null) { // TODO jonmv: Remove check and else branch after Oct 2019.
+            PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8));
+            if (keyVerifies(key, request)) {
+                Optional<CloudTenant> tenant = controller.tenants().get(id.tenant())
+                                                         .filter(CloudTenant.class::isInstance)
+                                                         .map(CloudTenant.class::cast);
+                if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key))
+                    return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key),
+                                                           Set.of(Role.reader(id.tenant()),
+                                                                  Role.developer(id.tenant()))));
+
+                Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id));
+                if (application.isPresent() && application.get().deployKeys().contains(key))
+                    return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
+                                                           Set.of(Role.reader(id.tenant()),
+                                                                  Role.developer(id.tenant())))); // TODO jonmv: Change to headless after Oct 10 2019.
+            }
+        }
+        else if (anyDeployKeyMatches(TenantAndApplicationId.from(id), request))
+            return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
+                                                   Set.of(Role.reader(id.tenant()),
+                                                          Role.developer(id.tenant()))));
+
+        return Optional.empty();
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
index 807e74b7c75..77622df4c4a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
@@ -26,10 +26,9 @@ import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
-import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.yolean.Exceptions;
 
-import java.security.Principal;
+import java.security.PublicKey;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
@@ -200,9 +199,9 @@ public class UserApiHandler extends LoggingRequestHandler {
         // TODO jonmv: Change to developer role, when this exists.
         if (role.definition().equals(RoleDefinition.tenantOperator))
             controller.tenants().lockIfPresent(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> {
-                String key = tenant.get().pemDeveloperKeys().inverse().get(new SimplePrincipal(user.value()));
+                PublicKey key = tenant.get().developerKeys().inverse().get(new SimplePrincipal(user.value()));
                 if (key != null)
-                    controller.tenants().store(tenant.withoutPemDeveloperKey(key));
+                    controller.tenants().store(tenant.withoutDeveloperKey(key));
             });
 
         users.removeUsers(role, List.of(user));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index 6ef9b5e6a4f..e230daf0c50 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 
 import java.security.Principal;
+import java.security.PublicKey;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -17,13 +18,13 @@ import java.util.Optional;
 public class CloudTenant extends Tenant {
 
     private final BillingInfo billingInfo;
-    private final BiMap<String, Principal> pemDeveloperKeys;
+    private final BiMap<PublicKey, Principal> developerKeys;
 
     /** Public for the serialization layer — do not use! */
-    public CloudTenant(TenantName name, BillingInfo info, BiMap<String, Principal> pemDeveloperKeys) {
+    public CloudTenant(TenantName name, BillingInfo info, BiMap<PublicKey, Principal> developerKeys) {
         super(name, Optional.empty());
         billingInfo = info;
-        this.pemDeveloperKeys = pemDeveloperKeys;
+        this.developerKeys = developerKeys;
     }
 
     /** Creates a tenant with the given name, provided it passes validation. */
@@ -37,7 +38,7 @@ public class CloudTenant extends Tenant {
     public BillingInfo billingInfo() { return billingInfo; }
 
     /** Returns the set of developer keys and their corresponding developers for this tenant. */
-    public BiMap<String, Principal> pemDeveloperKeys() { return pemDeveloperKeys; }
+    public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; }
 
     @Override
     public Type type() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 3ba1181f762..d68fe47b841 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -1,13 +1,13 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
-import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -37,6 +36,7 @@ import org.junit.Test;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.PublicKey;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -48,7 +48,6 @@ import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static java.util.Optional.empty;
@@ -64,6 +63,15 @@ public class ApplicationSerializerTest {
     private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/");
     private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1");
     private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3");
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
+
 
     @Test
     public void testSerialization() {
@@ -134,7 +142,7 @@ public class ApplicationSerializerTest {
                                                Optional.of(User.from("by-username")),
                                                OptionalInt.of(7),
                                                new ApplicationMetrics(0.5, 0.9),
-                                               Set.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"),
+                                               Set.of(publicKey, otherPublicKey),
                                                projectId,
                                                true,
                                                instances);
@@ -178,7 +186,7 @@ public class ApplicationSerializerTest {
         assertEquals(original.owner(), serialized.owner());
         assertEquals(original.majorVersion(), serialized.majorVersion());
         assertEquals(original.change(), serialized.change());
-        assertEquals(original.pemDeployKeys(), serialized.pemDeployKeys());
+        assertEquals(original.deployKeys(), serialized.deployKeys());
 
         assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations());
         assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index 51df0e4b08b..ff1c952c2a5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Ho
 
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.security.KeyUtils;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import org.junit.Test;
 
 import java.net.URI;
+import java.security.PublicKey;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -29,6 +31,14 @@ import static org.junit.Assert.assertTrue;
 public class TenantSerializerTest {
 
     private static final TenantSerializer serializer = new TenantSerializer();
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
 
     @Test
     public void athenz_tenant() {
@@ -78,12 +88,12 @@ public class TenantSerializerTest {
     public void cloud_tenant() {
         CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"),
                                              new BillingInfo("old cat lady", "vespa"),
-                                             ImmutableBiMap.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", new SimplePrincipal("joe"),
-                                                               "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", new SimplePrincipal("jane")));
+                                             ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"),
+                                                               otherPublicKey, new SimplePrincipal("jane")));
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.name(), serialized.name());
         assertEquals(tenant.billingInfo(), serialized.billingInfo());
-        assertEquals(tenant.pemDeveloperKeys(), serialized.pemDeveloperKeys());
+        assertEquals(tenant.developerKeys(), serialized.developerKeys());
     }
 
     private Contact contact() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
index 8ab277a3795..1c660726d61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
@@ -17,11 +17,11 @@
   "queryQuality": 100,
   "writeQuality": 99.99894341115082,
   "pemDeployKeys": [
-    "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"
+    "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----"
   ],
   "pemDeveloperKeys": [
     {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----",
       "user": "joe@dev"
     }
   ],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 7e00a2f9f64..0ae077f3fd7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -29,6 +29,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
@@ -49,11 +52,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
@@ -111,6 +111,11 @@ import static org.junit.Assert.assertTrue;
 public class ApplicationApiTest extends ControllerContainerTest {
 
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/";
+    private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                               "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                               "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                               "-----END PUBLIC KEY-----\n";
+    private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n");
 
     private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
             .environment(Environment.prod)
@@ -348,14 +353,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // POST a pem deploy key
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST)
                                       .userIdentity(USER_ID)
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}");
 
         // PATCH in a pem deploy key at deprecated path
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH)
                                       .userIdentity(USER_ID)
-                                      .data("{\"pemDeployKey\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"pemDeployKey\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}");
 
         // GET an application with a major version override
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
@@ -371,8 +376,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // DELETE the pem deploy key
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE)
                                       .userIdentity(USER_ID)
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Removed deploy key " + quotedPemPublicKey + "\"}");
 
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
                                       .userIdentity(USER_ID),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
index 331aabd32d0..9d76654fbc0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
@@ -80,8 +80,8 @@
   "majorVersion": 7,
   "globalRotations": [],
   "instances": [],
-  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
-  "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"],
+  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
+  "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n"],
   "metrics": {
     "queryServiceQuality": 0.0,
     "writeServiceQuality": 0.0
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
index 3b7d55f8cef..0a1e996696b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
@@ -3,15 +3,21 @@ package com.yahoo.vespa.hosted.controller.restapi.filter;
 
 import ai.vespa.hosted.api.Method;
 import ai.vespa.hosted.api.RequestSigner;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -19,6 +25,8 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.http.HttpRequest;
+import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -27,21 +35,21 @@ import static org.junit.Assert.assertTrue;
 
 public class SignatureFilterTest {
 
-    private static final String publicKey = "-----BEGIN PUBLIC KEY-----\n" +
-                                                 "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
-                                                 "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
-                                                 "-----END PUBLIC KEY-----\n";
+    private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                                                                "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                "-----END PUBLIC KEY-----\n");
 
-    private static final String otherPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
-                                                 "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
-                                                 "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
-                                                 "-----END PUBLIC KEY-----\n";
+    private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" +
+                                                                                     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                                                     "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                                                     "-----END PUBLIC KEY-----\n");
 
-    private static final String privateKey = "-----BEGIN EC PRIVATE KEY-----\n" +
-                                                  "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" +
-                                                  "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" +
-                                                  "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
-                                                  "-----END EC PRIVATE KEY-----\n";
+    private static final PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey("-----BEGIN EC PRIVATE KEY-----\n" +
+                                                                                   "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" +
+                                                                                   "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" +
+                                                                                   "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                                                                   "-----END EC PRIVATE KEY-----\n");
 
     private static final TenantAndApplicationId appId = TenantAndApplicationId.from("my-tenant", "my-app");
     private static final ApplicationId id = appId.defaultInstance();
@@ -58,10 +66,10 @@ public class SignatureFilterTest {
         filter = new SignatureFilter(tester.controller());
         signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock());
 
-        tester.createApplication(tester.createTenant(id.tenant().value(), "unused", 496L),
-                                 id.application().value(),
-                                 id.instance().value(),
-                                 28L);
+        tester.curator().writeTenant(new CloudTenant(appId.tenant(),
+                                                     new BillingInfo("id", "code"),
+                                                     ImmutableBiMap.of()));
+        tester.curator().writeApplication(new Application(appId, tester.clock().instant()));
     }
 
     @Test
@@ -69,42 +77,57 @@ public class SignatureFilterTest {
         // Unsigned request gets no role.
         HttpRequest.Builder request = HttpRequest.newBuilder(URI.create("https://host:123/path/./..//..%2F?query=empty&%3F=%26"));
         byte[] emptyBody = new byte[0];
-        DiscFilterRequest unsigned = requestOf(request.method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody);
-        filter.filter(unsigned);
-        assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME));
+        verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody),
+                              null);
 
         // Signed request gets no role when no key is stored for the application.
-        DiscFilterRequest signed = requestOf(signer.signed(request, Method.GET, InputStream::nullInputStream), emptyBody);
-        filter.filter(signed);
-        assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME));
-
-        // Signed request gets no role when a non-matching key is stored for the application.
-        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(otherPublicKey)));
-        filter.filter(signed);
-        assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME));
-
-        // Signed request gets a build service role when a matching key is stored for the application.
-        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(publicKey)));
-        assertTrue(filter.filter(signed).isEmpty());
-        SecurityContext securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME);
-        assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName());
-        assertEquals(Set.of(Role.buildService(id.tenant(), id.application()),
-                            Role.applicationDeveloper(id.tenant(), id.application())),
-                     securityContext.roles());
-
-        // Signed POST request also gets a build service role.
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                null);
+
+        // Signed request gets no role when only non-matching keys are stored for the application.
+        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(otherPublicKey)));
+        // Signed request gets no role when no key is stored for the application.
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              null);
+
+        // Signed request gets a headless role when a matching key is stored for the application.
+        applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(publicKey)));
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
+
+        // TODO jonmv: remove after Oct 2019.
+        // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided.
+        verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant()))));
+
+        // Signed POST request with X-Key header gets a headless role.
         byte[] hiBytes = new byte[]{0x48, 0x69};
-        signed = requestOf(signer.signed(request, Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes);
-        filter.filter(signed);
-        securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME);
-        assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName());
-        assertEquals(Set.of(Role.buildService(id.tenant(), id.application()),
-                            Role.applicationDeveloper(id.tenant(), id.application())),
-                     securityContext.roles());
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
+                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
+
+        // Signed request gets a developer role when a matching developer key is stored for the tenant.
+        tester.curator().writeTenant(new CloudTenant(appId.tenant(),
+                                                     new BillingInfo("id", "code"),
+                                                     ImmutableBiMap.of(publicKey, () -> "user")));
+        verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
+                              new SecurityContext(new SimplePrincipal("user"),
+                                                  Set.of(Role.reader(id.tenant()),
+                                                         Role.developer(id.tenant()))));
 
         // Unsigned requests still get no roles.
-        filter.filter(unsigned);
-        assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME));
+        verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody),
+                              null);
+    }
+
+    private void verifySecurityContext(DiscFilterRequest request, SecurityContext securityContext) {
+        assertTrue(filter.filter(request).isEmpty());
+        assertEquals(securityContext, request.getAttribute(SecurityContext.ATTRIBUTE_NAME));
     }
 
     private static DiscFilterRequest requestOf(HttpRequest request, byte[] body) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
index b17dda7f810..f2410c47908 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
@@ -12,7 +12,6 @@ import java.io.File;
 import java.util.Set;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
-import static com.yahoo.application.container.handler.Request.Method.PATCH;
 import static com.yahoo.application.container.handler.Request.Method.POST;
 import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static org.junit.Assert.assertEquals;
@@ -23,6 +22,17 @@ import static org.junit.Assert.assertEquals;
 public class UserApiTest extends ControllerContainerCloudTest {
 
     private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/";
+    private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                               "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" +
+                                               "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" +
+                                               "-----END PUBLIC KEY-----\n";
+    private static final String otherPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" +
+                                                    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" +
+                                                    "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" +
+                                                    "-----END PUBLIC KEY-----\n";
+    private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n");
+    private static final String otherQuotedPemPublicKey = otherPemPublicKey.replaceAll("\\n", "\\\\n");
+
 
     @Test
     public void testUserManagement() {
@@ -132,30 +142,30 @@ public class UserApiTest extends ControllerContainerCloudTest {
         // POST a pem deploy key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST)
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}");
 
         // POST a pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("joe@dev")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Set developer key " + quotedPemPublicKey + " for joe@dev\"}");
 
         // POST the same pem developer key for a different user is forbidden
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("operator@tenant")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Multiple entries with same key: -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=operator@tenant and -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=joe@dev\"}",
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Key "+  quotedPemPublicKey + " is already owned by joe@dev\"}",
                               400);
 
         // PATCH in a different pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
                                       .user("operator@tenant")
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\nƪ(`▿▿▿▿´ƪ)\\n-----END PUBLIC KEY----- for operator@tenant\"}");
+                                      .data("{\"key\":\"" + otherPemPublicKey + "\"}"),
+                              "{\"message\":\"Set developer key " + otherQuotedPemPublicKey + " for operator@tenant\"}");
 
         // GET tenant information with keys
         tester.assertResponse(request("/application/v4/tenant/my-tenant/")
@@ -165,8 +175,8 @@ public class UserApiTest extends ControllerContainerCloudTest {
         // DELETE a pem developer key
         tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE)
                                       .roles(Set.of(Role.tenantOperator(id.tenant())))
-                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Removed developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+                                      .data("{\"key\":\"" + pemPublicKey + "\"}"),
+                              "{\"message\":\"Removed developer key " + quotedPemPublicKey + " for joe@dev\"}");
 
         // DELETE an application role is allowed for an application admin.
         tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE)
@@ -180,8 +190,7 @@ public class UserApiTest extends ControllerContainerCloudTest {
                               "{\"message\":\"Deleted application my-tenant.my-app\"}");
 
         // DELETE a tenant role is available to tenant admins.
-        // DELETE the tenantOperator role clears any developer key.
-        // TODO jonmv: Change to developer, when this role exists.
+        // DELETE the developer role clears any developer key.
         tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE)
                                       .roles(Set.of(Role.tenantAdmin(id.tenant())))
                                       .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
index 5aaa900c3f0..f42534a4009 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json
@@ -1,19 +1,13 @@
 {
   "tenant": "my-tenant",
   "type": "CLOUD",
-  "pemDeployKeys": [
-    {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
-      "application": "my-app"
-    }
-  ],
   "pemDeveloperKeys": [
     {
-      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n",
       "user": "joe@dev"
     },
     {
-      "key": "-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----",
+      "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n",
       "user": "operator@tenant"
     }],
   "applications": [
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
index a89a0f5360c..39b6cccbab0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
@@ -1,7 +1,6 @@
 {
   "tenant": "my-tenant",
   "type": "CLOUD",
-  "pemDeployKeys": [],
   "pemDeveloperKeys": [],
   "applications": []
 }
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java
index b2fd16b7975..5d314d90356 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java
@@ -4,9 +4,10 @@ package ai.vespa.hosted.api;
 import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureUtils;
 
-import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.net.http.HttpRequest;
+import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.security.Signature;
 import java.security.SignatureException;
 import java.time.Clock;
@@ -15,6 +16,7 @@ import java.util.function.Supplier;
 
 import static ai.vespa.hosted.api.Signatures.sha256Digest;
 import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 /**
  * Signs HTTP request headers using a private key, for verification by the indicated public key.
@@ -25,6 +27,7 @@ public class RequestSigner {
 
     private final Signature signer;
     private final String keyId;
+    private final String base64PemPublicKey;
     private final Clock clock;
 
     /** Creates a new request signer from the given PEM encoded ECDSA key, with a public key with the given ID. */
@@ -34,8 +37,15 @@ public class RequestSigner {
 
     /** Creates a new request signer with a custom clock. */
     public RequestSigner(String pemPrivateKey, String keyId, Clock clock) {
-        this.signer = SignatureUtils.createSigner(KeyUtils.fromPemEncodedPrivateKey(pemPrivateKey), SHA256_WITH_ECDSA);
+        this(KeyUtils.fromPemEncodedPrivateKey(pemPrivateKey), keyId, clock);
+    }
+
+    /** Creates a new request signer with a custom clock. */
+    public RequestSigner(PrivateKey privateKey, String keyId, Clock clock) {
+        this.signer = SignatureUtils.createSigner(privateKey, SHA256_WITH_ECDSA);
         this.keyId = keyId;
+        this.base64PemPublicKey = Base64.getEncoder().encodeToString(KeyUtils.toPem(KeyUtils.extractPublicKey(privateKey)).getBytes(UTF_8));
+        PublicKey key = KeyUtils.extractPublicKey(privateKey);
         this.clock = clock;
     }
 
@@ -44,8 +54,8 @@ public class RequestSigner {
      * <br>
      * The request builder's method and data are set to the given arguments, and a hash of the
      * content is computed and added to a header, together with other meta data, like the URI
-     * of the request, the current UTC time, and the id of the public key which shall be used to
-     * verify this signature.
+     * of the request, the current UTC time, and the id and value of the public key which shall
+     * be used to * verify this signature.
      * Finally, a signature is computed from these fields, based on the private key of this, and
      * added to the request as another header.
      */
@@ -60,6 +70,29 @@ public class RequestSigner {
             request.setHeader("X-Timestamp", timestamp);
             request.setHeader("X-Content-Hash", contentHash);
             request.setHeader("X-Key-Id", keyId);
+            request.setHeader("X-Key", base64PemPublicKey);
+            request.setHeader("X-Authorization", signature);
+
+            request.method(method.name(), HttpRequest.BodyPublishers.ofInputStream(data));
+            return request.build();
+        }
+        catch (SignatureException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    // TODO jonmv: Simulates old clients — remove shortly (2 Oct 2019).
+    public HttpRequest legacySigned(HttpRequest.Builder request, Method method, Supplier<InputStream> data) {
+        try {
+            String timestamp = clock.instant().toString();
+            String contentHash = Base64.getEncoder().encodeToString(sha256Digest(data::get));
+            byte[] canonicalMessage = Signatures.canonicalMessageOf(method.name(), request.copy().build().uri(), timestamp, contentHash);
+            signer.update(canonicalMessage);
+            String signature = Base64.getEncoder().encodeToString(signer.sign());
+
+            request.setHeader("X-Timestamp", timestamp);
+            request.setHeader("X-Content-Hash", contentHash);
+            request.setHeader("X-Key-Id", keyId);
             request.setHeader("X-Authorization", signature);
 
             request.method(method.name(), HttpRequest.BodyPublishers.ofInputStream(data));
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java
index 9d85ec9bf6b..5a6bea54bce 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java
@@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureUtils;
 
 import java.net.URI;
+import java.security.PublicKey;
 import java.security.Signature;
 import java.security.SignatureException;
 import java.time.Clock;
@@ -31,7 +32,12 @@ public class RequestVerifier {
 
     /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */
     public RequestVerifier(String pemPublicKey, Clock clock) {
-        this.verifier = SignatureUtils.createVerifier(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), SHA256_WITH_ECDSA);
+        this(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), clock);
+    }
+
+    /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */
+    public RequestVerifier(PublicKey publicKey, Clock clock) {
+        this.verifier = SignatureUtils.createVerifier(publicKey, SHA256_WITH_ECDSA);
         this.clock = clock;
     }