From 51dd527f03628a0fe37babdb9a105111a7c7d40a Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 2 Mar 2023 14:15:28 +0100 Subject: Store tenant managed archive buckets in ZK --- .../api/integration/archive/ArchiveBucket.java | 71 --------------------- .../api/integration/archive/ArchiveBuckets.java | 34 ++++++++++ .../api/integration/archive/ArchiveService.java | 6 +- .../integration/archive/MockArchiveService.java | 12 ++-- .../archive/TenantManagedArchiveBucket.java | 15 +++++ .../archive/VespaManagedArchiveBucket.java | 72 ++++++++++++++++++++++ 6 files changed, 129 insertions(+), 81 deletions(-) delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java deleted file mode 100644 index be3b87ddc5c..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.archive; - -import com.google.common.collect.Sets; -import com.yahoo.config.provision.TenantName; - -import java.util.Objects; -import java.util.Set; - -/** - * Represents an S3 bucket used to store archive data - logs, heap/core dumps, etc. - * - * @author andreer - */ -public class ArchiveBucket { - private final String bucketName; - private final String keyArn; - private final Set tenants; - - public ArchiveBucket(String bucketName, String keyArn) { - this(bucketName, keyArn, Set.of()); - } - - private ArchiveBucket(String bucketName, String keyArn, Set tenants) { - this.bucketName = bucketName; - this.keyArn = keyArn; - this.tenants = Set.copyOf(tenants); - } - - public String bucketName() { - return bucketName; - } - - public String keyArn() { - return keyArn; - } - - public Set tenants() { - return tenants; - } - - public ArchiveBucket withTenant(TenantName tenant) { - return withTenants(Set.of(tenant)); - } - - public ArchiveBucket withTenants(Set tenants) { - return new ArchiveBucket(bucketName, keyArn, Sets.union(this.tenants, tenants)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ArchiveBucket that = (ArchiveBucket) o; - return bucketName.equals(that.bucketName) && keyArn.equals(that.keyArn) && tenants.equals(that.tenants); - } - - @Override - public int hashCode() { - return Objects.hash(bucketName, keyArn, tenants); - } - - @Override - public String toString() { - return "ArchiveBucket{" + - "bucketName='" + bucketName + '\'' + - ", keyArn='" + keyArn + '\'' + - ", tenants=" + tenants + - '}'; - } -} \ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java new file mode 100644 index 00000000000..62e341c674c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBuckets.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author freva + */ +public record ArchiveBuckets(Set vespaManaged, + Set tenantManaged) { + public static final ArchiveBuckets EMPTY = new ArchiveBuckets(Set.of(), Set.of()); + + public ArchiveBuckets(Set vespaManaged, Set tenantManaged) { + this.vespaManaged = Set.copyOf(vespaManaged); + this.tenantManaged = Set.copyOf(tenantManaged); + } + + /** Adds or replaces a VespaManagedArchive bucket with the given archive bucket */ + public ArchiveBuckets with(VespaManagedArchiveBucket vespaManagedArchiveBucket) { + Set updated = new HashSet<>(vespaManaged); + updated.removeIf(bucket -> bucket.bucketName().equals(vespaManagedArchiveBucket.bucketName())); + updated.add(vespaManagedArchiveBucket); + return new ArchiveBuckets(updated, tenantManaged); + } + + /** Adds or replaces a TenantManagedArchive bucket with the given archive bucket */ + public ArchiveBuckets with(TenantManagedArchiveBucket tenantManagedArchiveBucket) { + Set updated = new HashSet<>(tenantManaged); + updated.removeIf(bucket -> bucket.cloudAccount().equals(tenantManagedArchiveBucket.cloudAccount())); + updated.add(tenantManagedArchiveBucket); + return new ArchiveBuckets(vespaManaged, updated); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index 71425ca4b79..ed965f4331e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -19,11 +19,11 @@ import java.util.Set; */ public interface ArchiveService { - ArchiveBucket createArchiveBucketFor(ZoneId zoneId); + VespaManagedArchiveBucket createArchiveBucketFor(ZoneId zoneId); - void updatePolicies(ZoneId zoneId, Set buckets, Map authorizeAccessByTenantName); + void updatePolicies(ZoneId zoneId, Set buckets, Map authorizeAccessByTenantName); - boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket); + boolean canAddTenantToBucket(ZoneId zoneId, VespaManagedArchiveBucket bucket); Optional findEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index 17f90f13988..4bed7c5177f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -19,24 +19,22 @@ import java.util.Set; */ public class MockArchiveService implements ArchiveService { - - public Set archiveBuckets = new HashSet<>(); + public Set archiveBuckets = new HashSet<>(); public Map authorizeAccessByTenantName = new HashMap<>(); - @Override - public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { - return new ArchiveBucket("bucketName", "keyArn"); + public VespaManagedArchiveBucket createArchiveBucketFor(ZoneId zoneId) { + return new VespaManagedArchiveBucket("bucketName", "keyArn"); } @Override - public void updatePolicies(ZoneId zoneId, Set buckets, Map authorizeAccessByTenantName) { + public void updatePolicies(ZoneId zoneId, Set buckets, Map authorizeAccessByTenantName) { this.archiveBuckets = new HashSet<>(buckets); this.authorizeAccessByTenantName = new HashMap<>(authorizeAccessByTenantName); } @Override - public boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket) { + public boolean canAddTenantToBucket(ZoneId zoneId, VespaManagedArchiveBucket bucket) { return bucket.tenants().size() < 5; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java new file mode 100644 index 00000000000..80e9762f84b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/TenantManagedArchiveBucket.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; + +import java.time.Instant; + +/** + * Represents a cloud storage bucket (e.g. AWS S3 or Google Storage) used to store archive data - logs, heap/core dumps, etc. + * that is managed by the tenant directly. + * + * @author freva + */ +public record TenantManagedArchiveBucket(String bucketName, CloudAccount cloudAccount, Instant updatedAt) { +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java new file mode 100644 index 00000000000..c80e9b3780d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/VespaManagedArchiveBucket.java @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.google.common.collect.Sets; +import com.yahoo.config.provision.TenantName; + +import java.util.Objects; +import java.util.Set; + +/** + * Represents a cloud storage bucket (e.g. AWS S3 or Google Storage) used to store archive data - logs, heap/core dumps, etc. + * that is managed by the Vespa controller. + * + * @author andreer + */ +public class VespaManagedArchiveBucket { + private final String bucketName; + private final String keyArn; + private final Set tenants; + + public VespaManagedArchiveBucket(String bucketName, String keyArn) { + this(bucketName, keyArn, Set.of()); + } + + private VespaManagedArchiveBucket(String bucketName, String keyArn, Set tenants) { + this.bucketName = bucketName; + this.keyArn = keyArn; + this.tenants = Set.copyOf(tenants); + } + + public String bucketName() { + return bucketName; + } + + public String keyArn() { + return keyArn; + } + + public Set tenants() { + return tenants; + } + + public VespaManagedArchiveBucket withTenant(TenantName tenant) { + return withTenants(Set.of(tenant)); + } + + public VespaManagedArchiveBucket withTenants(Set tenants) { + return new VespaManagedArchiveBucket(bucketName, keyArn, Sets.union(this.tenants, tenants)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VespaManagedArchiveBucket that = (VespaManagedArchiveBucket) o; + return bucketName.equals(that.bucketName) && keyArn.equals(that.keyArn) && tenants.equals(that.tenants); + } + + @Override + public int hashCode() { + return Objects.hash(bucketName, keyArn, tenants); + } + + @Override + public String toString() { + return "ArchiveBucket{" + + "bucketName='" + bucketName + '\'' + + ", keyArn='" + keyArn + '\'' + + ", tenants=" + tenants + + '}'; + } +} \ No newline at end of file -- cgit v1.2.3