diff options
author | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
commit | 1cce4cc2640fe7bce984b808b1571555f5df0c30 (patch) | |
tree | 65c8a1830e8e94c368fcd418dd0f7cabd94439a6 /controller-api | |
parent | 9fa7903909043d7b855f7e3ba315050ba5a12597 (diff) | |
parent | 970cd02608c75df65bcfee3b061f906a39566825 (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources
Conflicts:
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
Diffstat (limited to 'controller-api')
7 files changed, 148 insertions, 29 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index 69eda662692..46e7fb48553 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; +import java.net.URI; import java.util.Map; import java.util.Set; @@ -17,7 +19,9 @@ public interface ArchiveService { ArchiveBucket createArchiveBucketFor(ZoneId zoneId); - void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName); + void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName,ArchiveAccess> authorizeAccessByTenantName); - void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles); + boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket); + + URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index ce7b56ad1f6..a2847439ce7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; +import java.net.URI; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -15,8 +18,10 @@ import java.util.TreeMap; */ public class MockArchiveService implements ArchiveService { - public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRolesForBucket = new HashMap<>(); - public Map<String, Set<String>> authorizedIamRolesForKey = new TreeMap<>(); + + public Set<ArchiveBucket> archiveBuckets = new HashSet<>(); + public Map<TenantName, ArchiveAccess> authorizeAccessByTenantName = new HashMap<>(); + @Override public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { @@ -24,12 +29,18 @@ public class MockArchiveService implements ArchiveService { } @Override - public void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) { - authorizedIamRolesForBucket.put(bucket, authorizeIamRoleByTenantName); + public void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName, ArchiveAccess> authorizeAccessByTenantName) { + this.archiveBuckets = new HashSet<>(buckets); + this.authorizeAccessByTenantName = new HashMap<>(authorizeAccessByTenantName); + } + + @Override + public boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket) { + return bucket.tenants().size() < 5; } @Override - public void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles) { - authorizedIamRolesForKey.put(keyArn, tenantAuthorizedIamRoles); + public URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName) { + return URI.create(String.format("s3://%s/%s/", bucketName, tenantName.value())); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index bff3fa14b89..1c6642f2dc6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -64,7 +64,10 @@ enum PathGroup { "/application/v4/tenant/{tenant}/key/"), tenantArchiveAccess(Matcher.tenant, - "/application/v4/tenant/{tenant}/archive-access"), + "/application/v4/tenant/{tenant}/archive-access", + "/application/v4/tenant/{tenant}/archive-access/aws", + "/application/v4/tenant/{tenant}/archive-access/gcp"), + billingToken(Matcher.tenant, "/billing/v1/tenant/{tenant}/token"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index a739a8e2b01..60950341a42 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -177,7 +177,7 @@ public class SystemFlagsDataArchive { if (!filenamesForSystem.isEmpty() && !filenamesForSystem.contains(filename)) { if (systemDefinition != null && filename.startsWith(systemDefinition.system().value() + '.')) { throw new IllegalArgumentException(String.format( - "Environment or zone in filename '%s' is does not exist", filename)); + "Environment or zone in filename '%s' does not exist", filename)); } return; // Ignore files irrelevant for system } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java new file mode 100644 index 00000000000..fba361f9223 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java @@ -0,0 +1,105 @@ +package com.yahoo.vespa.hosted.controller.tenant; + +import com.yahoo.text.Text; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class ArchiveAccess { + + private static final Pattern VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); + private static final Pattern VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN = Pattern.compile("(?<prefix>[a-zA-Z]+):.+"); + + private static final Set<String> gcpMemberPrefixes = Set.of("user", "serviceAccount", "group", "domain"); + + // AWS IAM Role + private final Optional<String> awsRole; + // GCP Member + private final Optional<String> gcpMember; + + public ArchiveAccess() { + this(Optional.empty(), Optional.empty()); + } + + private ArchiveAccess(Optional<String> awsRole, Optional<String> gcpMember) { + this.awsRole = awsRole; + this.gcpMember = gcpMember; + + awsRole.ifPresent(role -> validateAWSIAMRole(role)); + gcpMember.ifPresent(member -> validateGCPMember(member)); + } + + public ArchiveAccess withAWSRole(String role) { + return new ArchiveAccess(Optional.of(role), gcpMember()); + } + + public ArchiveAccess withGCPMember(String member) { + return new ArchiveAccess(awsRole(), Optional.of(member)); + } + + public ArchiveAccess withAWSRole(Optional<String> role) { + return new ArchiveAccess(role, gcpMember()); + } + + public ArchiveAccess withGCPMember(Optional<String> member) { + return new ArchiveAccess(awsRole(), member); + } + + public ArchiveAccess removeAWSRole() { + return new ArchiveAccess(Optional.empty(), gcpMember()); + } + + public ArchiveAccess removeGCPMember() { + return new ArchiveAccess(awsRole(), Optional.empty()); + } + + private void validateAWSIAMRole(String role) { + if (!VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()) { + throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'", + awsRole.get(), VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); + } + if (role.length() > 100) { + throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters"); + } + } + + private void validateGCPMember(String member) { + var matcher = VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.matcher(member); + if (!matcher.matches()) { + throw new IllegalArgumentException(Text.format("Invalid GCP archive access member '%s': Must match expected pattern: '%s'", + gcpMember.get(), VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.pattern())); + } + var prefix = matcher.group("prefix"); + if (!gcpMemberPrefixes.contains(prefix)) { + throw new IllegalArgumentException(Text.format("Invalid GCP member prefix '%s', must be one of '%s'", + prefix, gcpMemberPrefixes.stream().collect(Collectors.joining(", ")))); + } + if (!"domain".equals(prefix) && !member.contains("@")) { + throw new IllegalArgumentException(Text.format("Invalid GCP member '%s', prefix '%s' must be followed by an email id", member, prefix)); + } + } + + public Optional<String> awsRole() { + return awsRole; + } + + public Optional<String> gcpMember() { + return gcpMember; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArchiveAccess that = (ArchiveAccess) o; + return awsRole.equals(that.awsRole) && gcpMember.equals(that.gcpMember); + } + + @Override + public int hashCode() { + return Objects.hash(awsRole, gcpMember); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 9a4e04ebb3a..54924b9c456 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; -import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import java.security.Principal; @@ -13,7 +12,6 @@ import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.regex.Pattern; /** * A paying tenant in a Vespa cloud service. @@ -22,29 +20,22 @@ import java.util.regex.Pattern; */ public class CloudTenant extends Tenant { - private static final Pattern VALID_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); - private final Optional<Principal> creator; private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; - private final Optional<String> archiveAccessRole; + private final ArchiveAccess archiveAccess; /** Public for the serialization layer — do not use! */ public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) { + List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess) { super(name, createdAt, lastLoginInfo, Optional.empty()); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); this.tenantSecretStores = tenantSecretStores; - this.archiveAccessRole = archiveAccessRole; - if (!archiveAccessRole.map(role -> VALID_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()).orElse(true)) - throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'", - archiveAccessRole.get(), VALID_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); - if (archiveAccessRole.map(role -> role.length() > 100).orElse(false)) - throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters"); + this.archiveAccess = Objects.requireNonNull(archiveAccess); } /** Creates a tenant with the given name, provided it passes validation. */ @@ -53,7 +44,7 @@ public class CloudTenant extends Tenant { createdAt, LastLoginInfo.EMPTY, Optional.ofNullable(creator), - ImmutableBiMap.of(), TenantInfo.empty(), List.of(), Optional.empty()); + ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess()); } /** The user that created the tenant */ @@ -66,11 +57,6 @@ public class CloudTenant extends Tenant { return info; } - /** An iam role which is allowed to access the S3 (log, dump) archive) */ - public Optional<String> archiveAccessRole() { - return archiveAccessRole; - } - /** Returns the set of developer keys and their corresponding developers for this tenant. */ public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } @@ -79,6 +65,16 @@ public class CloudTenant extends Tenant { return tenantSecretStores; } + /** + * Role or member that is allowed to access archive bucket (log, dump) + * + * For AWS is this the IAM role + * For GCP it is a GCP member + */ + public ArchiveAccess archiveAccess() { + return archiveAccess; + } + @Override public Type type() { return Type.cloud; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index e4c46cfed1a..ce2746f92e0 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -135,7 +135,7 @@ public class SystemFlagsDataArchiveTest { Path directory = Paths.get("src/test/resources/system-flags-with-unknown-file-name/"); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "Environment or zone in filename 'main.prod.unknown-region.json' is does not exist"); + "Environment or zone in filename 'main.prod.unknown-region.json' does not exist"); SystemFlagsDataArchive.fromDirectoryAndSystem(directory, createZoneRegistryMock()); } |