Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources

 Conflicts:
	controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java

7 files changed, 148 insertions, 29 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java
index 69eda662692..46e7fb48553 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java
@@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive;
 
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess;
 
+import java.net.URI;
 import java.util.Map;
 import java.util.Set;
 
@@ -17,7 +19,9 @@ public interface ArchiveService {
 
     ArchiveBucket createArchiveBucketFor(ZoneId zoneId);
 
-    void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName);
+    void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName,ArchiveAccess> authorizeAccessByTenantName);
 
-    void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles);
+    boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket);
+
+    URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName);
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java
index ce7b56ad1f6..a2847439ce7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java
@@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive;
 
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess;
 
+import java.net.URI;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -15,8 +18,10 @@ import java.util.TreeMap;
  */
 public class MockArchiveService implements ArchiveService {
 
-    public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRolesForBucket = new HashMap<>();
-    public Map<String, Set<String>> authorizedIamRolesForKey = new TreeMap<>();
+
+    public Set<ArchiveBucket> archiveBuckets = new HashSet<>();
+    public Map<TenantName, ArchiveAccess> authorizeAccessByTenantName = new HashMap<>();
+
 
     @Override
     public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) {
@@ -24,12 +29,18 @@ public class MockArchiveService implements ArchiveService {
     }
 
     @Override
-    public void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) {
-        authorizedIamRolesForBucket.put(bucket, authorizeIamRoleByTenantName);
+    public void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName, ArchiveAccess> authorizeAccessByTenantName) {
+        this.archiveBuckets = new HashSet<>(buckets);
+        this.authorizeAccessByTenantName = new HashMap<>(authorizeAccessByTenantName);
+    }
+
+    @Override
+    public boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket) {
+        return bucket.tenants().size() < 5;
     }
 
     @Override
-    public void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles) {
-        authorizedIamRolesForKey.put(keyArn, tenantAuthorizedIamRoles);
+    public URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName) {
+        return URI.create(String.format("s3://%s/%s/", bucketName, tenantName.value()));
     }
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
index bff3fa14b89..1c6642f2dc6 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
@@ -64,7 +64,10 @@ enum PathGroup {
                "/application/v4/tenant/{tenant}/key/"),
 
     tenantArchiveAccess(Matcher.tenant,
-                       "/application/v4/tenant/{tenant}/archive-access"),
+                       "/application/v4/tenant/{tenant}/archive-access",
+                       "/application/v4/tenant/{tenant}/archive-access/aws",
+                       "/application/v4/tenant/{tenant}/archive-access/gcp"),
+
 
     billingToken(Matcher.tenant,
                  "/billing/v1/tenant/{tenant}/token"),
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
index a739a8e2b01..60950341a42 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
@@ -177,7 +177,7 @@ public class SystemFlagsDataArchive {
         if (!filenamesForSystem.isEmpty() && !filenamesForSystem.contains(filename)) {
             if (systemDefinition != null && filename.startsWith(systemDefinition.system().value() + '.')) {
                 throw new IllegalArgumentException(String.format(
-                        "Environment or zone in filename '%s' is does not exist", filename));
+                        "Environment or zone in filename '%s' does not exist", filename));
             }
             return; // Ignore files irrelevant for system
         }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java
new file mode 100644
index 00000000000..fba361f9223
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java
@@ -0,0 +1,105 @@
+package com.yahoo.vespa.hosted.controller.tenant;
+
+import com.yahoo.text.Text;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class ArchiveAccess {
+
+    private static final Pattern VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+");
+    private static final Pattern VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN = Pattern.compile("(?<prefix>[a-zA-Z]+):.+");
+
+    private static final Set<String> gcpMemberPrefixes = Set.of("user", "serviceAccount", "group", "domain");
+
+    // AWS IAM Role
+    private final Optional<String> awsRole;
+    // GCP Member
+    private final Optional<String> gcpMember;
+
+    public ArchiveAccess() {
+        this(Optional.empty(), Optional.empty());
+    }
+
+    private ArchiveAccess(Optional<String> awsRole, Optional<String> gcpMember) {
+        this.awsRole = awsRole;
+        this.gcpMember = gcpMember;
+
+        awsRole.ifPresent(role -> validateAWSIAMRole(role));
+        gcpMember.ifPresent(member -> validateGCPMember(member));
+    }
+
+    public ArchiveAccess withAWSRole(String role) {
+        return new ArchiveAccess(Optional.of(role), gcpMember());
+    }
+
+    public ArchiveAccess withGCPMember(String member) {
+        return new ArchiveAccess(awsRole(), Optional.of(member));
+    }
+
+    public ArchiveAccess withAWSRole(Optional<String> role) {
+        return new ArchiveAccess(role, gcpMember());
+    }
+
+    public ArchiveAccess withGCPMember(Optional<String> member) {
+        return new ArchiveAccess(awsRole(), member);
+    }
+
+    public ArchiveAccess removeAWSRole() {
+        return new ArchiveAccess(Optional.empty(), gcpMember());
+    }
+
+    public ArchiveAccess removeGCPMember() {
+        return new ArchiveAccess(awsRole(), Optional.empty());
+    }
+
+    private void validateAWSIAMRole(String role) {
+        if (!VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()) {
+            throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'",
+                    awsRole.get(), VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.pattern()));
+        }
+        if (role.length() > 100) {
+            throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters");
+        }
+    }
+
+    private void validateGCPMember(String member) {
+        var matcher = VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.matcher(member);
+        if (!matcher.matches()) {
+            throw new IllegalArgumentException(Text.format("Invalid GCP archive access member '%s': Must match expected pattern: '%s'",
+                    gcpMember.get(), VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.pattern()));
+        }
+        var prefix = matcher.group("prefix");
+        if (!gcpMemberPrefixes.contains(prefix)) {
+            throw new IllegalArgumentException(Text.format("Invalid GCP member prefix '%s', must be one of '%s'",
+                    prefix, gcpMemberPrefixes.stream().collect(Collectors.joining(", "))));
+        }
+        if (!"domain".equals(prefix) && !member.contains("@")) {
+            throw new IllegalArgumentException(Text.format("Invalid GCP member '%s', prefix '%s' must be followed by an email id", member, prefix));
+        }
+    }
+
+    public Optional<String> awsRole() {
+        return awsRole;
+    }
+
+    public Optional<String> gcpMember() {
+        return gcpMember;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ArchiveAccess that = (ArchiveAccess) o;
+        return awsRole.equals(that.awsRole) && gcpMember.equals(that.gcpMember);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(awsRole, gcpMember);
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index 9a4e04ebb3a..54924b9c456 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.tenant;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.ImmutableBiMap;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.text.Text;
 import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore;
 
 import java.security.Principal;
@@ -13,7 +12,6 @@ import java.time.Instant;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.regex.Pattern;
 
 /**
  * A paying tenant in a Vespa cloud service.
@@ -22,29 +20,22 @@ import java.util.regex.Pattern;
  */
 public class CloudTenant extends Tenant {
 
-    private static final Pattern VALID_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+");
-
     private final Optional<Principal> creator;
     private final BiMap<PublicKey, Principal> developerKeys;
     private final TenantInfo info;
     private final List<TenantSecretStore> tenantSecretStores;
-    private final Optional<String> archiveAccessRole;
+    private final ArchiveAccess archiveAccess;
 
     /** Public for the serialization layer — do not use! */
     public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator,
                        BiMap<PublicKey, Principal> developerKeys, TenantInfo info,
-                       List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) {
+                       List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess) {
         super(name, createdAt, lastLoginInfo, Optional.empty());
         this.creator = creator;
         this.developerKeys = developerKeys;
         this.info = Objects.requireNonNull(info);
         this.tenantSecretStores = tenantSecretStores;
-        this.archiveAccessRole = archiveAccessRole;
-        if (!archiveAccessRole.map(role -> VALID_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()).orElse(true))
-            throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'",
-                    archiveAccessRole.get(), VALID_ARCHIVE_ACCESS_ROLE_PATTERN.pattern()));
-        if (archiveAccessRole.map(role -> role.length() > 100).orElse(false))
-            throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters");
+        this.archiveAccess = Objects.requireNonNull(archiveAccess);
     }
 
     /** Creates a tenant with the given name, provided it passes validation. */
@@ -53,7 +44,7 @@ public class CloudTenant extends Tenant {
                                createdAt,
                                LastLoginInfo.EMPTY,
                                Optional.ofNullable(creator),
-                               ImmutableBiMap.of(), TenantInfo.empty(), List.of(), Optional.empty());
+                               ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess());
     }
 
     /** The user that created the tenant */
@@ -66,11 +57,6 @@ public class CloudTenant extends Tenant {
         return info;
     }
 
-    /** An iam role which is allowed to access the S3 (log, dump) archive) */
-    public Optional<String> archiveAccessRole() {
-        return archiveAccessRole;
-    }
-
     /** Returns the set of developer keys and their corresponding developers for this tenant. */
     public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; }
 
@@ -79,6 +65,16 @@ public class CloudTenant extends Tenant {
         return tenantSecretStores;
     }
 
+    /**
+     * Role or member that is allowed to access archive bucket (log, dump)
+     *
+     * For AWS is this the IAM role
+     * For GCP it is a GCP member
+     */
+    public ArchiveAccess archiveAccess() {
+        return archiveAccess;
+    }
+
     @Override
     public Type type() {
         return Type.cloud;
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
index e4c46cfed1a..ce2746f92e0 100644
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
@@ -135,7 +135,7 @@ public class SystemFlagsDataArchiveTest {
         Path directory = Paths.get("src/test/resources/system-flags-with-unknown-file-name/");
         expectedException.expect(IllegalArgumentException.class);
         expectedException.expectMessage(
-                "Environment or zone in filename 'main.prod.unknown-region.json' is does not exist");
+                "Environment or zone in filename 'main.prod.unknown-region.json' does not exist");
         SystemFlagsDataArchive.fromDirectoryAndSystem(directory, createZoneRegistryMock());
     }