Store developer keys <-> developers, and modify through application/v4

22 files changed, 280 insertions, 72 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
index 5ebea6c8d87..03eda33233d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
@@ -43,7 +43,7 @@ public class MockUserManagement implements UserManagement {
 
     @Override
     public void removeUsers(Role role, Collection<UserId> users) {
-        memberships.get(role).removeAll(users);
+        memberships.get(role).removeIf(user -> users.contains(new UserId(user.email())));
     }
 
     @Override
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
index 08702027264..958ded06c78 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
@@ -46,6 +46,15 @@ enum PathGroup {
                Optional.of("/api"),
                "/application/v4/tenant/{tenant}/application/"),
 
+    tenantKeys(Matcher.tenant,
+               Optional.of("/api"),
+               "/application/v4/tenant/{tenant}/key/"),
+
+    applicationKeys(Matcher.tenant,
+                    Matcher.application,
+                    Optional.of("/api"),
+                    "/application/v4/tenant/{tenant}/application/{application}/key/"),
+
     /** Path for the base application resource. */
     application(Matcher.tenant,
                 Matcher.application,
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
index 290382c6e6c..db7dd5909b3 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
@@ -83,6 +83,11 @@ enum Policy {
                                    .on(PathGroup.applicationInfo, PathGroup.productionRestart)
                                    .in(SystemName.all())),
 
+    /** Access to create and delete developer and deploy keys under a tenant. */
+    keyManagement(Privilege.grant(Action.write())
+                           .on(PathGroup.tenantKeys, PathGroup.applicationKeys)
+                           .in(SystemName.all())),
+
     /** Full access to application development deployments. */
     developmentDeployment(Privilege.grant(Action.all())
                                    .on(PathGroup.developmentDeployment, PathGroup.developmentRestart)
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
index 980b8bd316f..7bbd89404c7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
@@ -56,7 +56,8 @@ public enum RoleDefinition {
     /** Tenant operator with access to create application under a tenant, and to read the tenant's and public data. */
     tenantOperator(everyone,
                    Policy.tenantRead,
-                   Policy.applicationCreate),
+                   Policy.applicationCreate,
+                   Policy.keyManagement),
 
     /** Tenant admin with full access to all tenant resources, except deleting the tenant. */
     tenantAdmin(tenantOperator,
@@ -84,6 +85,7 @@ public enum RoleDefinition {
                       Policy.applicationUpdate,
                       Policy.applicationDelete,
                       Policy.applicationOperations,
+                      Policy.keyManagement,
                       Policy.developmentDeployment);
 
     private final Set<RoleDefinition> parents;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java
index 3378f9e0061..92f902dc0f7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java
@@ -49,4 +49,5 @@ public class SecurityContext {
                ", roles=" + roles +
                '}';
     }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index a944638105c..c17ac044136 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -25,6 +25,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -50,21 +51,21 @@ public class Application {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final List<String> pemDeployKeys;
+    private final Set<String> pemDeployKeys;
     private final Map<InstanceName, Instance> instances;
 
     /** Creates an empty application. */
     public Application(TenantAndApplicationId id, Instant now) {
         this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Change.empty(), Change.empty(),
              Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
-             new ApplicationMetrics(0, 0), List.of(), OptionalLong.empty(), false, List.of());
+             new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), false, List.of());
     }
 
     // DO NOT USE! For serialization purposes, only.
     public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        Change change, Change outstandingChange, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                       OptionalInt majorVersion, ApplicationMetrics metrics, List<String> pemDeployKeys, OptionalLong projectId,
-                       boolean internal, Collection<Instance> instances) {
+                       OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                       OptionalLong projectId, boolean internal, Collection<Instance> instances) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
         this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
@@ -76,7 +77,7 @@ public class Application {
         this.owner = Objects.requireNonNull(owner, "owner cannot be null");
         this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null");
         this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
-        this.pemDeployKeys = Objects.requireNonNull(pemDeployKeys, "pemDeployKey cannot be null");
+        this.pemDeployKeys = Objects.requireNonNull(pemDeployKeys, "pemDeployKeys cannot be null");
         this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null");
         this.internal = internal;
         this.instances = ImmutableSortedMap.copyOf(instances.stream().collect(Collectors.toMap(Instance::name, Function.identity())));
@@ -189,7 +190,8 @@ public class Application {
                                       .min(Comparator.naturalOrder());
     }
 
-    public List<String> pemDeployKeys() { return pemDeployKeys; }
+    /** Returns the set of deploy keys for this application. */
+    public Set<String> pemDeployKeys() { return pemDeployKeys; }
 
     @Override
     public boolean equals(Object o) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index cf7b8c8d48e..5aa5a8e13de 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -43,7 +43,7 @@ public class LockedApplication {
     private final Optional<User> owner;
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
-    private final List<String> pemDeployKeys;
+    private final Set<String> pemDeployKeys;
     private final OptionalLong projectId;
     private final boolean internal;
     private final Map<InstanceName, Instance> instances;
@@ -65,8 +65,9 @@ public class LockedApplication {
     private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec,
                               ValidationOverrides validationOverrides, Change change, Change outstandingChange,
                               Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
-                              OptionalInt majorVersion, ApplicationMetrics metrics, List<String> pemDeployKeys,
-                              OptionalLong projectId, boolean internal, Map<InstanceName, Instance> instances) {
+                              OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys,
+                              OptionalLong projectId, boolean internal,
+                              Map<InstanceName, Instance> instances) {
         this.lock = lock;
         this.id = id;
         this.createdAt = createdAt;
@@ -185,8 +186,7 @@ public class LockedApplication {
     }
 
     public LockedApplication withPemDeployKey(String pemDeployKey) {
-        List<String> keys = new ArrayList<>(pemDeployKeys);
-        keys.remove(pemDeployKey);
+        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
         keys.add(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
@@ -194,7 +194,7 @@ public class LockedApplication {
     }
 
     public LockedApplication withoutPemDeployKey(String pemDeployKey) {
-        List<String> keys = new ArrayList<>(pemDeployKeys);
+        Set<String> keys = new LinkedHashSet<>(pemDeployKeys);
         keys.remove(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
index f6ce63ca2ab..ecc8bd65b72 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
@@ -1,6 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.curator.Lock;
@@ -13,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
+import java.security.Principal;
 import java.util.Optional;
 
 import static java.util.Objects.requireNonNull;
@@ -123,23 +126,44 @@ public abstract class LockedTenant {
     public static class Cloud extends LockedTenant {
 
         private final BillingInfo billingInfo;
+        private final BiMap<String, Principal> pemDeveloperKeys;
 
-        private Cloud(TenantName name, BillingInfo billingInfo) {
+        private Cloud(TenantName name, BillingInfo billingInfo, BiMap<String, Principal> pemDeveloperKeys) {
             super(name);
             this.billingInfo = billingInfo;
+            this.pemDeveloperKeys = pemDeveloperKeys;
         }
 
         private Cloud(CloudTenant tenant) {
-            this(tenant.name(), tenant.billingInfo());
+            this(tenant.name(), tenant.billingInfo(), tenant.pemDeveloperKeys());
         }
 
         @Override
         public CloudTenant get() {
-            return new CloudTenant(name, billingInfo);
+            return new CloudTenant(name, billingInfo, pemDeveloperKeys);
         }
 
         public Cloud with(BillingInfo billingInfo) {
-            return new Cloud(name, billingInfo);
+            return new Cloud(name, billingInfo, pemDeveloperKeys);
+        }
+
+        public Cloud withPemDeveloperKey(String pemKey, Principal principal) {
+            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
+            pemDeveloperKeys.forEach((key, user) -> {
+                if ( ! user.equals(principal))
+                    keys.put(key, user);
+            });
+            keys.put(pemKey, principal);
+            return new Cloud(name, billingInfo, keys.build());
+        }
+
+        public Cloud withoutPemDeveloperKey(String pemKey) {
+            ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
+            pemDeveloperKeys.forEach((key, user) -> {
+                if ( ! key.equals(pemKey))
+                    keys.put(key, user);
+            });
+            return new Cloud(name, billingInfo, keys.build());
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 11435acbabe..08b3355587f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -1,6 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
@@ -19,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -36,17 +39,20 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
+import java.security.Principal;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -205,6 +211,7 @@ public class ApplicationSerializer {
     private void deployKeysToSlime(Stream<String> pemDeployKeys, Cursor array) {
         pemDeployKeys.forEach(array::addString);
     }
+
     private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) {
         for (Deployment deployment : deployments)
             deploymentToSlime(deployment, array.addObject());
@@ -377,7 +384,7 @@ public class ApplicationSerializer {
         OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField));
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
-        List<String> pemDeployKeys = pemDeployKeysFromSlime(root.field(pemDeployKeysField));
+        Set<String> pemDeployKeys = pemDeployKeysFromSlime(root.field(pemDeployKeysField));
         List<Instance> instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField));
         OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField));
         boolean builtInternally = root.field(builtInternallyField).asBool();
@@ -404,11 +411,12 @@ public class ApplicationSerializer {
         return instances;
     }
 
-    private List<String> pemDeployKeysFromSlime(Inspector array) {
-        List<String> keys = new ArrayList<>();
+    private Set<String> pemDeployKeysFromSlime(Inspector array) {
+        Set<String> keys = new LinkedHashSet<>();
         array.traverse((ArrayTraverser) (__, key) -> keys.add(key.asString()));
         return keys;
     }
+
     private List<Deployment> deploymentsFromSlime(Inspector array) {
         List<Deployment> deployments = new ArrayList<>();
         array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index 3a4e6c3954c..78d166607df 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -1,6 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
@@ -10,6 +12,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
@@ -18,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.net.URI;
+import java.security.Principal;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -52,6 +56,7 @@ public class TenantSerializer {
     private static final String billingInfoField = "billingInfo";
     private static final String customerIdField = "customerId";
     private static final String productCodeField = "productCode";
+    private static final String pemDeveloperKeysField = "pemDeveloperKeys";
 
     public Slime toSlime(Tenant tenant) {
         Slime slime = new Slime();
@@ -86,9 +91,18 @@ public class TenantSerializer {
     }
 
     private void toSlime(CloudTenant tenant, Cursor root) {
+        pemDeveloperKeysToSlime(tenant.pemDeveloperKeys(), root.setArray(pemDeveloperKeysField));
         toSlime(tenant.billingInfo(), root.setObject(billingInfoField));
     }
 
+    private void pemDeveloperKeysToSlime(BiMap<String, Principal> keys, Cursor array) {
+        keys.forEach((key, user) -> {
+            Cursor object = array.addObject();
+            object.setString("key", key);
+            object.setString("user", user.getName());
+        });
+    }
+
     private void toSlime(BillingInfo billingInfo, Cursor billingInfoObject) {
         billingInfoObject.setString(customerIdField, billingInfo.customerId());
         billingInfoObject.setString(productCodeField, billingInfo.productCode());
@@ -97,10 +111,7 @@ public class TenantSerializer {
     public Tenant tenantFrom(Slime slime) {
         Inspector tenantObject = slime.get();
         Tenant.Type type;
-        if (tenantObject.field(typeField).valid())
-            type = typeOf(tenantObject.field(typeField).asString());
-        else // TODO jvenstad: Remove once all tenants are stored on updated format.
-            type = tenantObject.field(nameField).asString().startsWith(Tenant.userPrefix) ? Tenant.Type.user : Tenant.Type.athenz;
+        type = typeOf(tenantObject.field(typeField).asString());
 
         switch (type) {
             case athenz: return athenzTenantFrom(tenantObject);
@@ -128,7 +139,16 @@ public class TenantSerializer {
     private CloudTenant cloudTenantFrom(Inspector tenantObject) {
         TenantName name = TenantName.from(tenantObject.field(nameField).asString());
         BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField));
-        return new CloudTenant(name, billingInfo);
+        BiMap<String, Principal> pemDeveloperKeys = pemDeveloperKeysFromSlime(tenantObject.field(pemDeveloperKeysField));
+        return new CloudTenant(name, billingInfo, pemDeveloperKeys);
+    }
+
+    private BiMap<String, Principal> pemDeveloperKeysFromSlime(Inspector array) {
+        ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder();
+        array.traverse((ArrayTraverser) (__, keyObject) -> {
+            keys.put(keyObject.field("key").asString(), new SimplePrincipal(keyObject.field("user").asString()));
+        });
+        return keys.build();
     }
 
     private Optional<Contact> contactFrom(Inspector object) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 1531df9c818..4c4478c9af6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -20,7 +20,6 @@ import com.yahoo.restapi.Path;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.slime.Type;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
@@ -30,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.AlreadyExistsException;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.NotExistsException;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
@@ -79,6 +79,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 import com.yahoo.vespa.hosted.controller.security.AccessControlRequests;
 import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
@@ -238,11 +239,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse handlePOST(Path path, HttpRequest request) {
         if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/key")) return addDeveloperKey(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request);
@@ -270,9 +273,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse handleDELETE(Path path, HttpRequest request) {
         if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default",  "all");
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice"));
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), "default");
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all");
@@ -366,6 +371,44 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new SlimeJsonResponse(slime);
     }
 
+    private HttpResponse addDeveloperKey(String tenantName, HttpRequest request) {
+        if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud)
+            throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant");
+
+        Principal user = request.getJDiscRequest().getUserPrincipal();
+        String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
+        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
+                controller.tenants().store(tenant.withPemDeveloperKey(pemDeveloperKey, user)));
+        return new MessageResponse("Set developer key " + pemDeveloperKey + " for " + user);
+    }
+
+    private HttpResponse removeDeveloperKey(String tenantName, HttpRequest request) {
+        if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud)
+            throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant");
+
+        String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString();
+        Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).pemDeveloperKeys().get(pemDeveloperKey);
+        controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant ->
+                controller.tenants().store(tenant.withoutPemDeveloperKey(pemDeveloperKey)));
+        return new MessageResponse("Removed developer key " + pemDeveloperKey + " for " + user);
+    }
+
+    private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) {
+        String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
+        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
+            controller.applications().store(application.withPemDeployKey(pemDeployKey));
+        });
+        return new MessageResponse("Added deploy key " + pemDeployKey);
+    }
+
+    private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) {
+        String pemDeployKey = toSlime(request.getData()).get().field("key").asString();
+        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> {
+            controller.applications().store(application.withoutPemDeployKey(pemDeployKey));
+        });
+        return new MessageResponse("Removed deploy key " + pemDeployKey);
+    }
+
     private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) {
         Inspector requestObject = toSlime(request.getData()).get();
         StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes.");
@@ -377,13 +420,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 messageBuilder.add("Set major version to " + (majorVersion == null ? "empty" : majorVersion));
             }
 
-            Inspector pemDeployKeyRemovalField = requestObject.field("pemDeployKeyRemoval");
-            if (pemDeployKeyRemovalField.valid()) {
-                String pemDeployKey = pemDeployKeyRemovalField.asString();
-                application = application.withoutPemDeployKey(pemDeployKey);
-                messageBuilder.add("Removed deploy key " + pemDeployKey);
-            }
-
+            // TODO jonmv: Remove when clients are updated.
             Inspector pemDeployKeyField = requestObject.field("pemDeployKey");
             if (pemDeployKeyField.valid()) {
                 String pemDeployKey = pemDeployKeyField.asString();
@@ -616,7 +653,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             }
         }
 
+        // TODO jonmv: Remove when clients are updated
         application.pemDeployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", key));
+
         application.pemDeployKeys().forEach(object.setArray("pemDeployKeys")::addString);
 
         // Metrics
@@ -1305,6 +1344,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private void toSlime(Cursor object, Tenant tenant, HttpRequest request) {
         object.setString("tenant", tenant.name().value());
         object.setString("type", tenantType(tenant));
+        List<Application> applications = controller.applications().asList(tenant.name());
         switch (tenant.type()) {
             case athenz:
                 AthenzTenant athenzTenant = (AthenzTenant) tenant;
@@ -1323,11 +1363,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 });
                 break;
             case user: break;
-            case cloud: break;
+            case cloud: {
+                CloudTenant cloudTenant = (CloudTenant) tenant;
+
+                Cursor pemDeployKeysArray = object.setArray("pemDeployKeys");
+                for (Application application : applications)
+                    for (String key : application.pemDeployKeys()) {
+                        Cursor keyObject = pemDeployKeysArray.addObject();
+                        keyObject.setString("key", key);
+                        keyObject.setString("application", application.id().application().value());
+                    }
+
+                Cursor pemDeveloperKeysArray = object.setArray("pemDeveloperKeys");
+                cloudTenant.pemDeveloperKeys().forEach((key, user) -> {
+                    Cursor keyObject = pemDeveloperKeysArray.addObject();
+                    keyObject.setString("key", key);
+                    keyObject.setString("user", user.getName());
+                });
+
+                break;
+            }
             default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
         }
         Cursor applicationArray = object.setArray("applications");
-        for (Application application : controller.applications().asList(tenant.name()))
+        for (Application application : applications)
             for (Instance instance : application.instances().values())
                 if (recurseOverApplications(request))
                     toSlime(applicationArray.addObject(), instance, application, request);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
index bf559085e5e..6755110bb49 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.yolean.Exceptions;
 
@@ -56,7 +57,7 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
                                                                                    request.getHeader("X-Authorization")));
 
                 if (verified) {
-                    Principal principal = () -> "buildService@" + id.tenant() + "." + id.application();
+                    Principal principal = new SimplePrincipal("buildService@" + id.tenant() + "." + id.application());
                     request.setUserPrincipal(principal);
                     request.setRemoteUser(principal.getName());
                     request.setAttribute(SecurityContext.ATTRIBUTE_NAME,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
index 410194a8ef7..807e74b7c75 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
@@ -13,6 +13,8 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.api.integration.user.Roles;
 import com.yahoo.vespa.hosted.controller.api.integration.user.User;
 import com.yahoo.vespa.hosted.controller.api.integration.user.UserId;
@@ -22,17 +24,22 @@ import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition;
 import com.yahoo.restapi.ErrorResponse;
 import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.yolean.Exceptions;
 
+import java.security.Principal;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * API for user management related to access control.
@@ -46,11 +53,13 @@ public class UserApiHandler extends LoggingRequestHandler {
     private static final String optionalPrefix = "/api";
 
     private final UserManagement users;
+    private final Controller controller;
 
     @Inject
-    public UserApiHandler(Context parentCtx, UserManagement users) {
+    public UserApiHandler(Context parentCtx, UserManagement users, Controller controller) {
         super(parentCtx);
         this.users = users;
+        this.controller = controller;
     }
 
     @Override
@@ -183,11 +192,18 @@ public class UserApiHandler extends LoggingRequestHandler {
         String roleName = require("roleName", Inspector::asString, requestObject);
         UserId user = new UserId(require("user", Inspector::asString, requestObject));
         Role role = Roles.toRole(TenantName.from(tenantName), roleName);
-        List<User> currentUsers = users.listUsers(role);
-        if (role.definition() == RoleDefinition.tenantOwner
-                && currentUsers.size() == 1
-                && currentUsers.get(0).email().equals(user.value()))
-            throw new IllegalArgumentException("Can't remove the last owner of a tenant.");
+
+        if (   role.definition() == RoleDefinition.tenantOwner
+            && Set.of(user.value()).equals(users.listUsers(role).stream().map(User::email).collect(Collectors.toSet())))
+        throw new IllegalArgumentException("Can't remove the last owner of a tenant.");
+
+        // TODO jonmv: Change to developer role, when this exists.
+        if (role.definition().equals(RoleDefinition.tenantOperator))
+            controller.tenants().lockIfPresent(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> {
+                String key = tenant.get().pemDeveloperKeys().inverse().get(new SimplePrincipal(user.value()));
+                if (key != null)
+                    controller.tenants().store(tenant.withoutPemDeveloperKey(key));
+            });
 
         users.removeUsers(role, List.of(user));
         return new MessageResponse(user+" is no longer a member of "+role);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
index 33529c342a3..7da3e43c9a5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
@@ -35,7 +35,7 @@ public class CloudAccessControl implements AccessControl {
     @Override
     public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) {
         CloudTenantSpec spec = (CloudTenantSpec) tenantSpec;
-        CloudTenant tenant = new CloudTenant(spec.tenant(), defaultBillingInfo);
+        CloudTenant tenant = CloudTenant.create(spec.tenant(), defaultBillingInfo);
 
         for (Role role : Roles.tenantRoles(spec.tenant()))
             userManagement.createRole(role);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index 2d59e539bbd..6ef9b5e6a4f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -1,35 +1,44 @@
 package com.yahoo.vespa.hosted.controller.tenant;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 
+import java.security.Principal;
+import java.util.Objects;
 import java.util.Optional;
 
 /**
- * A tenant as vague as its name.
- *
- * Only a reference to a cloud identity provider, and some billing info, is known for this tenant type.
+ * A paying tenant in a Vespa cloud service.
  *
  * @author jonmv
  */
 public class CloudTenant extends Tenant {
 
     private final BillingInfo billingInfo;
+    private final BiMap<String, Principal> pemDeveloperKeys;
 
     /** Public for the serialization layer — do not use! */
-    public CloudTenant(TenantName name, BillingInfo info) {
+    public CloudTenant(TenantName name, BillingInfo info, BiMap<String, Principal> pemDeveloperKeys) {
         super(name, Optional.empty());
         billingInfo = info;
+        this.pemDeveloperKeys = pemDeveloperKeys;
     }
 
     /** Creates a tenant with the given name, provided it passes validation. */
     public static CloudTenant create(TenantName tenantName, BillingInfo billingInfo) {
-        return new CloudTenant(requireName(tenantName), billingInfo);
+        return new CloudTenant(requireName(tenantName),
+                               Objects.requireNonNull(billingInfo),
+                               ImmutableBiMap.of());
     }
 
     /** Returns the billing info for this tenant. */
     public BillingInfo billingInfo() { return billingInfo; }
 
+    /** Returns the set of developer keys and their corresponding developers for this tenant. */
+    public BiMap<String, Principal> pemDeveloperKeys() { return pemDeveloperKeys; }
+
     @Override
     public Type type() {
         return Type.cloud;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index bc6a94470bc..3ba1181f762 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
@@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -46,6 +48,7 @@ import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static java.util.Optional.empty;
@@ -131,7 +134,7 @@ public class ApplicationSerializerTest {
                                                Optional.of(User.from("by-username")),
                                                OptionalInt.of(7),
                                                new ApplicationMetrics(0.5, 0.9),
-                                               List.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"),
+                                               Set.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"),
                                                projectId,
                                                true,
                                                instances);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index b262cd5cee7..51df0e4b08b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -1,14 +1,15 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
-import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import org.junit.Test;
@@ -75,17 +76,14 @@ public class TenantSerializerTest {
 
     @Test
     public void cloud_tenant() {
-        CloudTenant tenant = CloudTenant.create(TenantName.from("elderly-lady"),
-                                                new BillingInfo("old cat lady", "vespa"));
+        CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"),
+                                             new BillingInfo("old cat lady", "vespa"),
+                                             ImmutableBiMap.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", new SimplePrincipal("joe"),
+                                                               "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", new SimplePrincipal("jane")));
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.name(), serialized.name());
         assertEquals(tenant.billingInfo(), serialized.billingInfo());
-    }
-
-    @Test
-    public void legacy_deserialization() {
-        UserTenant legayUserTenant = (UserTenant) serializer.tenantFrom(SlimeUtils.jsonToSlime("{\"name\":\"by-someone\"}"));
-        assertTrue(legayUserTenant.is("someone"));
+        assertEquals(tenant.pemDeveloperKeys(), serialized.pemDeveloperKeys());
     }
 
     private Contact contact() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
index ec976c1b922..8ab277a3795 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
@@ -19,6 +19,12 @@
   "pemDeployKeys": [
     "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"
   ],
+  "pemDeveloperKeys": [
+    {
+      "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+      "user": "joe@dev"
+    }
+  ],
   "instances": [
     {
       "instanceName": "default",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
index b6154876dba..0c0d1f433cd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
@@ -4,6 +4,7 @@ import com.yahoo.application.container.handler.Request;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
+import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 
 import java.nio.charset.StandardCharsets;
 import java.security.Principal;
@@ -78,7 +79,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest {
 
         public RequestBuilder data(byte[] data) { this.data = data; return this; }
         public RequestBuilder data(String data) { this.data = data.getBytes(StandardCharsets.UTF_8); return this; }
-        public RequestBuilder user(String user) { this.user = () -> user; return this; }
+        public RequestBuilder user(String user) { this.user = new SimplePrincipal(user); return this; }
         public RequestBuilder roles(Set<Role> roles) { this.roles = roles; return this; }
 
         @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index af726dab552..7e00a2f9f64 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -345,7 +345,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data("{\"majorVersion\":7}"),
                               "{\"message\":\"Set major version to 7\"}");
 
-        // PATCH in a pem deploy key
+        // POST a pem deploy key
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST)
+                                      .userIdentity(USER_ID)
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
+                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+
+        // PATCH in a pem deploy key at deprecated path
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH)
                                       .userIdentity(USER_ID)
                                       .data("{\"pemDeployKey\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
@@ -362,16 +368,10 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data("{\"majorVersion\":null}"),
                               "{\"message\":\"Set major version to empty\"}");
 
-        // PATCH in removal of the pem deploy key
-        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", PATCH)
-                                      .userIdentity(USER_ID)
-                                      .data("{\"pemDeployKeyRemoval\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
-                              "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
-
-        // PATCH in removal of the pem deploy key on deprecated path
-        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH)
+        // DELETE the pem deploy key
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE)
                                       .userIdentity(USER_ID)
-                                      .data("{\"pemDeployKeyRemoval\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
                               "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
 
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
index d0e9ae77965..b17dda7f810 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
@@ -12,6 +12,7 @@ import java.io.File;
 import java.util.Set;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
+import static com.yahoo.application.container.handler.Request.Method.PATCH;
 import static com.yahoo.application.container.handler.Request.Method.POST;
 import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static org.junit.Assert.assertEquals;
@@ -128,6 +129,45 @@ public class UserApiTest extends ControllerContainerCloudTest {
                                       .roles(Set.of(Role.tenantOperator(id.tenant()))),
                               new File("application-roles.json"));
 
+        // POST a pem deploy key
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST)
+                                      .roles(Set.of(Role.tenantOperator(id.tenant())))
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
+                              "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}");
+
+        // POST a pem developer key
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
+                                      .user("joe@dev")
+                                      .roles(Set.of(Role.tenantOperator(id.tenant())))
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
+                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+
+        // POST the same pem developer key for a different user is forbidden
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
+                                      .user("operator@tenant")
+                                      .roles(Set.of(Role.tenantOperator(id.tenant())))
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Multiple entries with same key: -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=operator@tenant and -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=joe@dev\"}",
+                              400);
+
+        // PATCH in a different pem developer key
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST)
+                                      .user("operator@tenant")
+                                      .roles(Set.of(Role.tenantOperator(id.tenant())))
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----\"}"),
+                              "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\nƪ(`▿▿▿▿´ƪ)\\n-----END PUBLIC KEY----- for operator@tenant\"}");
+
+        // GET tenant information with keys
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/")
+                                      .roles(Set.of(Role.applicationReader(id.tenant(), id.application()))),
+                              new File("tenant-with-keys.json"));
+
+        // DELETE a pem developer key
+        tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE)
+                                      .roles(Set.of(Role.tenantOperator(id.tenant())))
+                                      .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"),
+                              "{\"message\":\"Removed developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}");
+
         // DELETE an application role is allowed for an application admin.
         tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE)
                                       .roles(Set.of(Role.applicationAdmin(id.tenant(), id.application())))
@@ -140,6 +180,8 @@ public class UserApiTest extends ControllerContainerCloudTest {
                               "{\"message\":\"Deleted application my-tenant.my-app\"}");
 
         // DELETE a tenant role is available to tenant admins.
+        // DELETE the tenantOperator role clears any developer key.
+        // TODO jonmv: Change to developer, when this role exists.
         tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE)
                                       .roles(Set.of(Role.tenantAdmin(id.tenant())))
                                       .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
index e4ca5a02446..a89a0f5360c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json
@@ -1,5 +1,7 @@
 {
   "tenant": "my-tenant",
   "type": "CLOUD",
+  "pemDeployKeys": [],
+  "pemDeveloperKeys": [],
   "applications": []
 }