diff options
| author | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
|---|---|---|
| committer | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
| commit | e9058b555d4dfea2f6c872d9a677e8678b569569 (patch) | |
| tree | fa1b67c6e39712c1e0d9f308b0dd55573b43f913 /controller-server/src | |
| parent | 0ad931fa86658904fe9212b014d810236b0e00e4 (diff) | |
| parent | 16030193ec04ee41e98779a3d7ee6a6c1d0d0d6f (diff) | |
Merge branch 'master' into bjormel/aws-main-controller
Diffstat (limited to 'controller-server/src')
38 files changed, 662 insertions, 154 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 6ec732a3815..7d19acfce80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; @@ -43,12 +44,14 @@ public abstract class LockedTenant { final Instant createdAt; final LastLoginInfo lastLoginInfo; final Instant tenantRolesLastMaintained; + final List<CloudAccountInfo> cloudAccounts; - private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { + private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { this.name = requireNonNull(name); this.createdAt = requireNonNull(createdAt); this.lastLoginInfo = requireNonNull(lastLoginInfo); this.tenantRolesLastMaintained = requireNonNull(tenantRolesLastMaintained); + this.cloudAccounts = requireNonNull(cloudAccounts); } static LockedTenant of(Tenant tenant, Mutex lock) { @@ -66,6 +69,8 @@ public abstract class LockedTenant { public abstract LockedTenant with(Instant tenantRolesLastMaintained); + public abstract LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts); + public Deleted deleted(Instant deletedAt) { return new Deleted(new DeletedTenant(name, createdAt, deletedAt)); } @@ -85,8 +90,8 @@ public abstract class LockedTenant { private final Optional<Contact> contact; private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.domain = domain; this.property = property; this.propertyId = propertyId; @@ -94,38 +99,43 @@ public abstract class LockedTenant { } private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained()); + this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts()); } @Override public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } } @@ -146,8 +156,8 @@ public abstract class LockedTenant { BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained, - Optional<BillingReference> billingReference) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; @@ -158,12 +168,12 @@ public abstract class LockedTenant { } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.billingReference()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -174,51 +184,56 @@ public abstract class LockedTenant { if (keys.inverse().containsKey(simplePrincipal)) throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); keys.put(key, simplePrincipal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud with(BillingReference billingReference) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, Optional.of(billingReference)); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, Optional.of(billingReference)); } } @@ -229,7 +244,7 @@ public abstract class LockedTenant { private final Instant deletedAt; private Deleted(DeletedTenant tenant) { - super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH); + super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH, List.of()); this.deletedAt = tenant.deletedAt(); } @@ -247,6 +262,11 @@ public abstract class LockedTenant { public LockedTenant with(Instant tenantRolesLastMaintained) { return this; } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return this; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 091836a1eea..b1ffce65852 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -82,7 +82,8 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag randomizedEndpoints; + private final BooleanFlag generatedEndpoints; + private final BooleanFlag legacyEndpoints; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -90,7 +91,8 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.randomizedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.generatedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.legacyEndpoints = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -228,13 +230,14 @@ public class RoutingController { .in(controller.system())); // Only a single region endpoint is needed, not one per auth method if (isProduction && generatedEndpoint.authMethod() == AuthMethod.mtls) { - endpoints.add(regionEndpoint.generatedFrom(generatedEndpoint) + GeneratedEndpoint weightedGeneratedEndpoint = generatedEndpoint.withClusterPart(weightedClusterPart(cluster, deployment)); + endpoints.add(regionEndpoint.generatedFrom(weightedGeneratedEndpoint) .authMethod(AuthMethod.none) .in(controller.system())); } } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(deployment.applicationId(), EndpointList.copyOf(endpoints)); } /** Read routing policies and return zone- and region-scoped endpoints for given deployment */ @@ -268,7 +271,7 @@ public class RoutingController { endpoints.add(builder.generatedFrom(ge).authMethod(ge.authMethod()).in(controller.system())); } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(routingId.instance(), EndpointList.copyOf(endpoints)); } /** Returns application endpoints pointing to given deployments */ @@ -424,6 +427,13 @@ public class RoutingController { Optional.of(application.id()))); } + private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { + if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { + return endpoints.generated(); + } + return endpoints; + } + private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId()); EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global); @@ -476,6 +486,22 @@ public class RoutingController { .toList(); } + /** Generate the cluster part of a {@link GeneratedEndpoint} for use in a {@link Endpoint.Scope#weighted} endpoint */ + private String weightedClusterPart(ClusterSpec.Id cluster, DeploymentId deployment) { + // This ID must be common for a given cluster in all deployments within the same cloud-native region + String cloudNativeRegion = controller.zoneRegistry().zones().all().get(deployment.zoneId()).get().getCloudNativeRegionName(); + HashCode hash = Hashing.sha256().newHasher() + .putString(cluster.value(), StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(cloudNativeRegion, StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(deployment.applicationId().serializedForm(), StandardCharsets.UTF_8) + .hash(); + String alphabet = "abcdef"; + char letter = alphabet.charAt(Math.abs(hash.asInt()) % alphabet.length()); + return letter + hash.toString().substring(0, 7); + } + /** Returns existing generated endpoints, grouped by their {@link Scope#multiDeployment()} endpoint */ private Map<EndpointId, GeneratedEndpointList> readDeclaredGeneratedEndpoints(TenantAndApplicationId application) { Map<EndpointId, GeneratedEndpointList> endpoints = new HashMap<>(); @@ -525,7 +551,17 @@ public class RoutingController { } public boolean generatedEndpointsEnabled(ApplicationId instance) { - return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()).value(); + return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); + } + + public boolean legacyEndpointsEnabled(ApplicationId instance) { + return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); } private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index bf2f2ab90eb..d11540b28dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -165,6 +166,14 @@ public class TenantController { } } + public void updateCloudAccounts(TenantName tenantName, List<CloudAccountInfo> cloudAccounts) { + try (Mutex lock = lock(tenantName)) { + var tenant = require(tenantName); + if (tenant.cloudAccounts().equals(cloudAccounts)) return; // no change + curator.writeTenant(LockedTenant.of(tenant, lock).withCloudAccounts(cloudAccounts).get()); + } + } + /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { try (Mutex lock = lock(tenant)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java index 8db4492356a..28f9963f24c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -38,6 +38,11 @@ public record GeneratedEndpoint(String clusterPart, String applicationPart, Auth return !declared(); } + /** Returns a copy of this with cluster part set to given value */ + public GeneratedEndpoint withClusterPart(String clusterPart) { + return new GeneratedEndpoint(clusterPart, applicationPart, authMethod, endpoint); + } + /** Create a new endpoint part, using random as a source of randomness */ public static String createPart(RandomGenerator random) { String alphabet = "abcdef0123456789"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index e01da00a27e..33af58a9790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,6 +11,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; @@ -57,6 +58,7 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; + private final BooleanFlag assignLegacyNames; private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, @@ -64,6 +66,7 @@ public class EndpointCertificates { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); + this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; @@ -140,10 +143,11 @@ public class EndpointCertificates { } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, candidate.get().certificate()), + EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); + curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), transaction); transaction.commit(); - return candidate.get().certificate(); + return certificate; } } } @@ -174,9 +178,12 @@ public class EndpointCertificates { } // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id + // Skip this validation for now if the cert has a randomized id and should not provision legacy names Optional<EndpointCertificate> currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - var requiredSansForZone = currentCertificate.get().randomizedId().isEmpty() ? + boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); + + var requiredSansForZone = legacyNames || currentCertificate.get().randomizedId().isEmpty() ? controller.routing().certificateDnsNames(deployment, deploymentSpec) : List.<String>of(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e247d6baa09..1b40781fe0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -276,7 +276,7 @@ public class DeploymentTrigger { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); - if(newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun()>=requiredEntry.requiredRun())) { + if (newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun() >= requiredEntry.requiredRun())) { newList.add(requiredEntry); } newList = newList.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 919facee0c1..11c47d8f481 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -62,7 +63,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -362,21 +362,24 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), - Optional.of(platform)); + // In manually deployed zones it is allowed for some model versions not being built (e.g due to incompatibility) + // but deployment still succeeding, so we cannot use version when checking for config convergence + Optional<Version> platformVersion = id.type().environment().isManuallyDeployed() ? Optional.empty() : Optional.of(platform); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), + platformVersion); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .applications(id.application()) - .states(active)); + List<Node> nodes = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .applications(id.application()) + .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); NodeList nodeList = NodeList.of(nodes, parents, services.get()); ConvergenceSummary summary = nodeList.summary(); @@ -496,8 +499,8 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), - Optional.of(platform)); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(testerId, zone), + Optional.of(platform)); if (services.isEmpty()) { if (run.stepInfo(installTester).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(30)))) { logger.log(WARNING, "Config status not available after 30 minutes; giving up!"); @@ -508,14 +511,14 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .applications(testerId) - .states(active, reserved)); + List<Node> nodes = configServer().nodeRepository().list(zone, + NodeFilter.all() + .applications(testerId) + .states(active, reserved)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(zone, + NodeFilter.all() + .hostnames(parentHostnames)); NodeList nodeList = NodeList.of(nodes, parents, services.get()); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, false)) @@ -534,6 +537,8 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private ConfigServer configServer() { return controller.serviceRegistry().configServer(); } + /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { DeploymentId deploymentId = new DeploymentId(id, zoneId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index d10e38fd990..e7ec6675a82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -2,23 +2,76 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; + private final BillingController billing; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { super(controller, interval, null, Set.of(SystemName.PublicCd)); this.reporter = controller.serviceRegistry().billingReporter(); + this.billing = controller.serviceRegistry().billingController(); + this.plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { - return this.reporter.maintain(); + maintainTenants(); + return 0.0; + } + + private void maintainTenants() { + var tenants = cloudTenants(); + var tenantNames = List.copyOf(tenants.keySet()); + var billableTenants = billableTenants(tenantNames); + + billableTenants.forEach(tenant -> { + controller().tenants().lockIfPresent(tenant, LockedTenant.Cloud.class, locked -> { + var ref = reporter.maintainTenant(locked.get()); + if (locked.get().billingReference().isEmpty() || ! locked.get().billingReference().get().equals(ref)) { + controller().tenants().store(locked.with(ref)); + } + }); + }); + } + + private Map<TenantName, CloudTenant> cloudTenants() { + return controller().tenants().asList() + .stream() + .filter(CloudTenant.class::isInstance) + .map(CloudTenant.class::cast) + .collect(Collectors.toMap( + Tenant::name, + Function.identity())); + } + + private List<Plan> billablePlans() { + return plans.all().stream() + .filter(Plan::isBilled) + .toList(); + } + + private List<TenantName> billableTenants(List<TenantName> tenants) { + return billablePlans().stream() + .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) + .toList(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 70eeb2b9f6c..ed383175cc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { // Create metric for available certificates in the pool as a fraction of configured size int poolSize = certPoolSize.value(); long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of())); + metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { provisionRandomizedCertificate(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java new file mode 100644 index 00000000000..f0fc8985bdf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.WARNING; + +/** + * Verifies the cloud accounts that may be used by a given user have applied the enclave template + * and extracts the version of the applied template. + * + * All maintainers that operate on external cloud accounts should use the list on the Tenant instance + * maintained by this class rather than the cloud-accounts feature flag. + * + * The template version can be used to determine if new features can be enabled for the cloud account. + * + * @author freva + */ +public class CloudAccountVerifier extends ControllerMaintainer { + + private static final Logger logger = Logger.getLogger(CloudAccountVerifier.class.getName()); + + CloudAccountVerifier(Controller controller, Duration interval) { + super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public)); + } + + @Override + protected double maintain() { + int attempts = 0, failures = 0; + for (Tenant tenant : controller().tenants().asList()) { + try { + attempts++; + List<CloudAccountInfo> cloudAccountInfos = controller().applications().accountsOf(tenant.name()).stream() + .flatMap(account -> controller().serviceRegistry() + .archiveService() + .getEnclaveTemplateVersion(account) + .map(version -> new CloudAccountInfo(account, version)) + .stream()) + .toList(); + controller().tenants().updateCloudAccounts(tenant.name(), cloudAccountInfos); + } catch (RuntimeException e) { + logger.log(WARNING, "Failed to verify cloud accounts for tenant " + tenant.name(), e); + failures++; + } + } + return asSuccessFactorDeviation(attempts, failures); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index f9c93a87c44..f6da3609fbb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -29,8 +29,8 @@ public class ContactInformationMaintainer extends ControllerMaintainer { private final ContactRetriever contactRetriever; - public ContactInformationMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + public ContactInformationMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)), successFactorBaseline); this.contactRetriever = controller.serviceRegistry().contactRetriever(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 6fae732df0a..7afa10ab8d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -59,7 +59,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); - maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); + maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer, successFactorBaseline.contactInformationMaintainerBaseline)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase())); @@ -85,6 +85,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval)); maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer)); + maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier)); } public Upgrader upgrader() { return upgrader; } @@ -147,6 +148,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration meteringMonitorMaintainer; private final Duration certificatePoolMaintainer; private final Duration billingReportMaintainer; + private final Duration cloudAccountVerifier; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -184,6 +186,7 @@ public class ControllerMaintenance extends AbstractComponent { this.meteringMonitorMaintainer = duration(30, MINUTES); this.certificatePoolMaintainer = duration(15, MINUTES); this.billingReportMaintainer = duration(60, MINUTES); + this.cloudAccountVerifier = duration(10, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { @@ -201,12 +204,14 @@ public class ControllerMaintenance extends AbstractComponent { private final Double deploymentMetricsMaintainerBaseline; private final Double trafficFractionUpdater; private final Double deploymentInfoMaintainerBaseline; + private final Double contactInformationMaintainerBaseline; public SuccessFactorBaseline(SystemName system) { Objects.requireNonNull(system); this.deploymentMetricsMaintainerBaseline = 0.90; this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65; this.deploymentInfoMaintainerBaseline = system.isCd() ? 0.5 : 0.95; + this.contactInformationMaintainerBaseline = 0.95; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 5218da91c46..6c1c4daa1bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -33,7 +33,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { private Set<CloudAccount> externalAccounts() { Set<CloudAccount> accounts = new HashSet<>(); for (Tenant tenant : controller().tenants().asList()) - accounts.addAll(controller().applications().accountsOf(tenant.name())); + tenant.cloudAccounts().forEach(accountInfo -> accounts.add(accountInfo.cloudAccount())); return accounts; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index c90fcb81c71..805bf3d7ada 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -67,7 +67,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - final BooleanFlag assignRandomizedId; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; private final IntFlag assignRandomizedIdRate; @@ -81,7 +80,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.assignRandomizedId = Flags.ASSIGN_RANDOMIZED_ID.bindTo(controller.flagSource()); this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); @@ -283,7 +281,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { assignedCertificates.stream() .filter(c -> c.instance().isPresent()) .filter(c -> c.certificate().randomizedId().isEmpty()) - .filter(c -> assignRandomizedId.with(FetchVector.Dimension.INSTANCE_ID, c.application().instance(c.instance().get()).serializedForm()).value()) .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion .limit(assignRandomizedIdRate.value()) .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a25aa9797ba..dc9c4650191 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -7,6 +7,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -602,7 +603,7 @@ public class CuratorDb { public List<DnsChallenge> readDnsChallenges(DeploymentId id) { return curator.getChildren(dnsChallengePath(id)).stream() - .map(cluster -> readDnsChallenge(new ClusterId(id, ClusterSpec.Id.from(cluster)))) + .map(cluster -> readDnsChallenge(new ClusterId(id, Id.from(cluster)))) .toList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index e3d61c81667..760fb9b0366 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -20,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; @@ -85,6 +88,9 @@ public class TenantSerializer { private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore"; private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained"; private static final String billingReferenceField = "billingReference"; + private static final String cloudAccountsField = "cloudAccounts"; + private static final String accountField = "account"; + private static final String templateVersionField = "templateVersion"; private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -97,6 +103,7 @@ public class TenantSerializer { tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); tenantObject.setLong(tenantRolesLastMaintainedField, tenant.tenantRolesLastMaintained().toEpochMilli()); + cloudAccountsToSlime(tenant.cloudAccounts(), tenantObject.setArray(cloudAccountsField)); switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; @@ -162,6 +169,14 @@ public class TenantSerializer { } } + private void cloudAccountsToSlime(List<CloudAccountInfo> cloudAccounts, Cursor cloudAccountsObject) { + cloudAccounts.forEach(cloudAccountInfo -> { + Cursor object = cloudAccountsObject.addObject(); + object.setString(accountField, cloudAccountInfo.cloudAccount().account()); + object.setString(templateVersionField, cloudAccountInfo.templateVersion().toFullString()); + }); + } + public Tenant tenantFrom(Slime slime) { Inspector tenantObject = slime.get(); Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); @@ -183,7 +198,8 @@ public class TenantSerializer { Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccountInfos); } private CloudTenant cloudTenantFrom(Inspector tenantObject) { @@ -197,8 +213,9 @@ public class TenantSerializer { ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); Optional<Instant> invalidateUserSessionsBefore = SlimeUtils.optionalInstant(tenantObject.field(invalidateUserSessionsBeforeField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); Optional<BillingReference> billingReference = billingReferenceFrom(tenantObject.field(billingReferenceField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccountInfos, billingReference); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -284,6 +301,14 @@ public class TenantSerializer { return new LastLoginInfo(lastLoginByUserLevel); } + private List<CloudAccountInfo> cloudAccountsFromSlime(Inspector cloudAccountsObject) { + return SlimeUtils.entriesStream(cloudAccountsObject) + .map(inspector -> new CloudAccountInfo( + CloudAccount.from(inspector.field(accountField).asString()), + Version.fromString(inspector.field(templateVersionField).asString()))) + .toList(); + } + void toSlime(TenantInfo info, Cursor parentCursor) { if (info.isEmpty()) return; Cursor infoCursor = parentCursor.setObject("info"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 46c81fc073f..16d862a66ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2915,6 +2915,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); + + if (!tenant.cloudAccounts().isEmpty()) { + Cursor cloudAccounts = object.setArray("cloudAccounts"); + tenant.cloudAccounts().forEach(accountInfo -> { + Cursor accountObject = cloudAccounts.addObject(); + accountObject.setString("cloudAccount", accountInfo.cloudAccount().value()); + accountObject.setString("templateVersion", accountInfo.templateVersion().toFullString()); + }); + } } private void toSlime(ArchiveAccess archiveAccess, Cursor object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 67dd172fd83..c5fb1afbae8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -85,6 +85,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") + .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") .get(self::plans)) .addExceptionMapper(RuntimeException.class, (c, e) -> ErrorResponses.logThrowing(c.request(), log, e)) @@ -262,6 +264,19 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return new SlimeJsonResponse(slime); } + private HttpResponse putAccountantInvoiceExport(RestApi.RequestContext ctx, Slime slime) { + var billId = ctx.attributes().get("invoice") + .map(id -> Bill.Id.of((String) id)) + .orElseThrow(() -> new RestApiException.BadRequest("Missing bill ID")); + + // TODO: try to find a way to retrieve the cloud tenant from BillingControllerImpl + var bill = billing.getBill(billId); + var cloudTenant = tenants.require(bill.tenant(), CloudTenant.class); + + var exportMethod = slime.get().field("method").asString(); + var result = billing.exportBill(bill, exportMethod, cloudTenant); + return new MessageResponse("Bill has been exported: " + result); + } // --------- INVOICE RENDERING ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index de25161c461..a21c6548a0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -263,8 +264,14 @@ public class RoutingPolicies { } else { weightedEndpoints = weightedEndpoints.not().generated(); } + if (generated && weightedEndpoints.isEmpty()) { + // Ignore this policy. If an instance has a global endpoint, and is switching from non-generated to + // generated endpoints we cannot update global DNS record for a deployment until it has been deployed at + // least once (which assigns a generated endpoint). + continue; + } if (weightedEndpoints.size() != 1) { - throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent); + throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent + ", got " + weightedEndpoints); } Endpoint endpoint = weightedEndpoints.first().get(); RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( @@ -410,24 +417,22 @@ public class RoutingPolicies { new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); nameServiceForwarder(endpoint).createRecord(record, Priority.normal, ownerOf(deploymentId)); - setPrivateDns(endpoint, loadBalancer, deploymentId); } + setPrivateDns(zoneEndpoints, loadBalancer, deploymentId); } - private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) { + private void setPrivateDns(EndpointList endpoints, LoadBalancer loadBalancer, DeploymentId deploymentId) { if (loadBalancer.service().isEmpty()) return; - // TODO(mpolden): Why is this done? Consider creating private DNS for all auth methods - boolean skipBasedOnAuthMethod = switch (endpoint.authMethod()) { - case token -> true; - case mtls -> false; - case none -> true; - }; - if (skipBasedOnAuthMethod) return; + // TODO(mpolden): Model one service for each endpoint (type), to allow private endpoints with tokens. + EndpointList mtlsEndpoints = endpoints.authMethod(AuthMethod.mtls); + if (mtlsEndpoints.isEmpty()) return; + Endpoint endpoint = mtlsEndpoints.generated().first().orElse(mtlsEndpoints.first().get()); if (endpoint.routingMethod() != RoutingMethod.exclusive) return; // Not supported for this routing method controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), - loadBalancer.cloudAccount()) + loadBalancer.cloudAccount(), + endpoint.generated().isPresent()) .ifPresent(challenge -> { try (Mutex lock = db.lockNameServiceQueue()) { controller.nameServiceForwarder().createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); @@ -436,10 +441,18 @@ public class RoutingPolicies { }); } + /** Deletes all DNS challenges, and corresponding TXT records, for the given deployment. */ + public void removeDnsChallenges(DeploymentId deploymentId) { + try (Mutex lock = db.lockNameServiceQueue()) { + db.readDnsChallenges(deploymentId).forEach(this::removeDnsChallenge); + } + } + /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ public boolean processDnsChallenges(DeploymentId deploymentId) { try (Mutex lock = db.lockNameServiceQueue()) { List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); + challenges.removeIf(challenge -> challenge.state() == ChallengeState.done); Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() .map(NameServiceRequest::name) .collect(Collectors.toSet()); @@ -450,14 +463,8 @@ public class RoutingPolicies { challenge = challenge.withState(ChallengeState.ready); } ChallengeState state = controller.serviceRegistry().vpcEndpointService().process(challenge); - if (state == ChallengeState.done) { - removeDnsChallenge(challenge); - return true; - } - else { - db.writeDnsChallenge(challenge.withState(state)); - return false; - } + db.writeDnsChallenge(challenge.withState(state)); + return state == ChallengeState.done; }); return challenges.isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index df0226176a2..99f60735f6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -57,6 +57,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { /** Deactivate routing configuration for the deployment in this context, using given deployment spec */ public final void deactivate(DeploymentSpec deploymentSpec) { routing.policies().refresh(deployment, deploymentSpec, EndpointList.EMPTY); + routing.policies().removeDnsChallenges(deployment); } /** Routing method of this context */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 1cb43453918..a6d3b435dcb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -306,6 +306,9 @@ public class EndpointCertificatesTest { fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} + // Advance clock to verify last requested time + clock.advance(Duration.ofDays(3)); + // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod @@ -315,6 +318,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().randomizedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } { // dev @@ -325,6 +329,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().randomizedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index ed5226ebc8b..0e5308fcef5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -42,6 +42,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter import com.yahoo.vespa.hosted.controller.api.integration.configserver.ProxyResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -103,6 +105,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<DeploymentId, TestReport> testReport = new HashMap<>(); private final Map<DeploymentId, CloudAccount> cloudAccounts = new HashMap<>(); private final Map<DeploymentId, List<X509Certificate>> additionalCertificates = new HashMap<>(); + private final Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokenFingerprints = new HashMap<>(); private List<SearchNodeMetrics> searchNodeMetrics; private Version lastPrepareVersion = null; @@ -319,6 +322,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return additionalCertificates.getOrDefault(deployment, List.of()); } + public void setActiveTokenFingerprints(HostName hostname, Map<TokenId, List<FingerPrint>> tokens) { + activeTokenFingerprints.put(hostname, tokens); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -585,6 +592,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return "{\"settings\":{\"name\":\"foo\",\"role\":\"vespa-secretstore-access\",\"awsId\":\"892075328880\",\"externalId\":\"*****\",\"region\":\"us-east-1\"},\"status\":\"ok\"}"; } + @Override + public Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokenFingerprints(DeploymentId deploymentId) { + return activeTokenFingerprints; + } + public static class Application { private final ApplicationId id; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index f2c827478c0..c6386509585 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingControll import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporterMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; @@ -52,9 +53,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import java.time.Instant; import java.util.Optional; +import java.util.UUID; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -316,6 +320,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg @Override public BillingReporter billingReporter() { - return () -> 0.0; + return new BillingReporterMock(clock()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java new file mode 100644 index 00000000000..b1e00ba0746 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class BillingReportMaintainerTest { + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); + private final BillingReportMaintainer maintainer = new BillingReportMaintainer(tester.controller(), Duration.ofMinutes(10)); + + @Test + void only_billable_tenants_are_maintained() { + var t1 = tester.createTenant("t1"); + var t2 = tester.createTenant("t2"); + + tester.controller().serviceRegistry().billingController().setPlan(t1, PlanRegistryMock.paidPlan.id(), false, true); + maintainer.maintain(); + + var b1 = billingReference(t1); + var b2 = billingReference(t2); + + assertFalse(b1.isEmpty()); + assertTrue(b2.isEmpty()); + + assertEquals(tester.clock().instant(), b1.orElseThrow().updated()); + assertNotNull(b1.orElseThrow().reference()); + } + + private Optional<BillingReference> billingReference(TenantName tenantName) { + var t = tester.controller().tenants().require(tenantName, CloudTenant.class); + return t.billingReference(); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java index 88c5ae9ff06..4257261b09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java @@ -53,12 +53,4 @@ public class CertificatePoolMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(n, tester.curator().readUnassignedCertificates().size()); } - - void old_unassigned_certs_are_refreshed() { - tester.flagSource().withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), 1); - assertNumCerts(1); - EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); - var request = endpointCertificateProvider.listCertificates().get(0); - - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index f0c11c0ddbd..2c54c0c9fb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -30,7 +30,7 @@ public class ContactInformationMaintainerTest { @BeforeEach public void before() { tester = new ControllerTester(); - maintainer = new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1)); + maintainer = new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1), 1.0); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java index 5bfac2866ce..1e1079a3314 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java @@ -21,17 +21,20 @@ class EnclaveAccessMaintainerTest { void test() { ControllerTester tester = new ControllerTester(); MockEnclaveAccessService amis = tester.serviceRegistry().enclaveAccessService(); - EnclaveAccessMaintainer sharer = new EnclaveAccessMaintainer(tester.controller(), Duration.ofMinutes(1)); + EnclaveAccessMaintainer sharer = new EnclaveAccessMaintainer(tester.controller(), Duration.ofHours(1)); + CloudAccountVerifier accountVerifier = new CloudAccountVerifier(tester.controller(), Duration.ofHours(1)); assertEquals(Set.of(), amis.currentAccounts()); assertEquals(1, sharer.maintain()); assertEquals(Set.of(), amis.currentAccounts()); tester.createTenant("tanten"); + accountVerifier.maintain(); assertEquals(1, sharer.maintain()); assertEquals(Set.of(), amis.currentAccounts()); tester.flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of("123123123123", "321321321321"), String.class); + accountVerifier.maintain(); assertEquals(1, sharer.maintain()); assertEquals(Set.of(CloudAccount.from("aws:123123123123"), CloudAccount.from("aws:321321321321")), amis.currentAccounts()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 918a4bed6f4..cbc69e52119 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -11,7 +11,9 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProviderMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import java.time.Duration; @@ -36,10 +39,13 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.perfUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -100,12 +106,15 @@ public class EndpointCertificateMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); var cert = tester.curator().readAssignedCertificate(appId).orElseThrow().certificate(); - tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(cert.rootRequestId()); // cert should not be deleted, the app is deployed! + tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(cert.leafRequestId().get()); // cert should not be deleted, the app is deployed! } @Test void refreshed_certificate_is_discovered_and_after_four_days_deployed() { - var appId = ApplicationId.from("tenant", "application", "default"); + prepareCertificatePool(1); + + var instanceId = ApplicationId.from("tenant", "application", "default"); + var applicationId = TenantAndApplicationId.from(instanceId); DeploymentTester deploymentTester = new DeploymentTester(tester); @@ -115,22 +124,25 @@ public class EndpointCertificateMaintainerTest { DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); - var assignedCertificate = tester.curator().readAssignedCertificate(appId).orElseThrow(); + var assignedCertificate = tester.curator().readAssignedCertificate(applicationId, Optional.empty()).orElseThrow(); // cert should not be deleted, the app is deployed! assertEquals(0.0, maintainer.maintain(), 0.0000001); - assertEquals(tester.curator().readAssignedCertificate(appId), Optional.of(assignedCertificate)); + assertEquals(tester.curator().readAssignedCertificate(applicationId, Optional.empty()).map(c->c.certificate().rootRequestId()), Optional.of(assignedCertificate.certificate().rootRequestId())); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(assignedCertificate.certificate().rootRequestId()); + // TODO: Remove this line when we have removed assignment of randomized id to application certificates + //assignedCertificate = tester.curator().readAssignedCertificate().orElseThrow(); // This simulates a cert refresh performed 3 days later tester.clock().advance(Duration.ofDays(3)); secretStore.setSecret(assignedCertificate.certificate().keyName(), "foo", 1); secretStore.setSecret(assignedCertificate.certificate().certName(), "bar", 1); - tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId.toFullString(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); + tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate("preprovisioned." + assignedCertificate.certificate().randomizedId().get(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); + // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertNotRunning(productionUsWest1); - var updatedCert = tester.curator().readAssignedCertificate(appId).orElseThrow().certificate(); + var updatedCert = tester.curator().readAssignedCertificate(applicationId, Optional.empty()).orElseThrow().certificate(); assertNotEquals(assignedCertificate.certificate().leafRequestId().orElseThrow(), updatedCert.leafRequestId().orElseThrow()); assertEquals(updatedCert.version(), assignedCertificate.certificate().version() + 1); @@ -179,24 +191,12 @@ public class EndpointCertificateMaintainerTest { } @Test - void certificates_are_not_assigned_random_id_when_flag_disabled() { - var app = ApplicationId.from("tenant", "app", "default"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - maintainer.maintain(); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - } - - @Test void production_deployment_certificates_are_assigned_random_id() { var app = ApplicationId.from("tenant", "app", "default"); DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); maintainer.maintain(); assertEquals(2, tester.curator().readAssignedCertificates().size()); @@ -223,7 +223,6 @@ public class EndpointCertificateMaintainerTest { DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, instance1, List.of(systemTest, stagingTest,productionUsWest1),Optional.of("instance1")); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); maintainer.maintain(); String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get(); @@ -241,7 +240,6 @@ public class EndpointCertificateMaintainerTest { DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); List<String> originalRequestedSans = tester.curator().readAssignedCertificate(devApp).get().certificate().requestedDnsSans(); maintainer.maintain(); assertEquals(1, tester.curator().readAssignedCertificates().size()); @@ -254,9 +252,64 @@ public class EndpointCertificateMaintainerTest { assertEquals(3, randomizedNames.size()); } + @Test + void deploy_to_other_manual_zone_refreshes_cert() { + String devSan = "*.foo.manual.tenant.us-east-1.dev.vespa.oath.cloud"; + String perfSan = "*.foo.manual.tenant.us-east-3.perf.vespa.oath.cloud"; + + var devApp = ApplicationId.from("tenant", "manual", "foo"); + DeploymentTester deploymentTester = new DeploymentTester(tester); + deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); + assertEquals(1, tester.curator().readAssignedCertificates().size()); + maintainer.maintain(); + Optional<AssignedCertificate> devCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(devApp), Optional.of(devApp.instance())); + List<String> devSans = devCertificate.get().certificate().requestedDnsSans(); + Assertions.assertThat(devSans).contains(devSan); + Assertions.assertThat(devSans).doesNotContain(perfSan); + + // Deploy to perf and verify that the certs are refreshed + deployToAssignCert(deploymentTester, devApp, List.of(perfUsEast3), Optional.empty()); + Optional<AssignedCertificate> devAndPerfCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(devApp), Optional.of(devApp.instance())); + List<String> devAndPerfSans = devAndPerfCertificate.get().certificate().requestedDnsSans(); + + assertNotEquals(devSans, devAndPerfSans); + Assertions.assertThat(devAndPerfSans).contains(devSan); + Assertions.assertThat(devAndPerfSans).contains(perfSan); + } + + @Test + void deploy_to_other_prod_zone_refreshes_cert() { + String westSan = "*.prod.tenant.us-west-1.vespa.oath.cloud"; + String centralSan = "*.prod.tenant.us-central-1.vespa.oath.cloud"; + + var prodApp = ApplicationId.from("tenant", "prod", "default"); + DeploymentTester deploymentTester = new DeploymentTester(tester); + deployToAssignCert(deploymentTester, prodApp, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); + assertEquals(1, tester.curator().readAssignedCertificates().size()); + maintainer.maintain(); + Optional<AssignedCertificate> usWestCert = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(prodApp), Optional.of(prodApp.instance())); + List<String> usWestSans = usWestCert.get().certificate().requestedDnsSans(); + Assertions.assertThat(usWestSans).contains(westSan); + Assertions.assertThat(usWestSans).doesNotContain(centralSan); + + // Deploy to perf and verify that the certs are refreshed + deployToAssignCert(deploymentTester, prodApp, List.of(systemTest, stagingTest, productionUsWest1, productionUsCentral1), Optional.empty()); + Optional<AssignedCertificate> usCentralWestCert = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(prodApp), Optional.of(prodApp.instance())); + List<String> usCentralWestSans = usCentralWestCert.get().certificate().requestedDnsSans(); + assertNotEquals(usWestSans, usCentralWestSans); + Assertions.assertThat(usCentralWestSans).contains(westSan); + Assertions.assertThat(usCentralWestSans).contains(centralSan); + } + + private void deploy() { + + } + private void deployToAssignCert(DeploymentTester tester, ApplicationId applicationId, List<JobType> jobTypes, Optional<String> instances) { - var applicationPackageBuilder = new ApplicationPackageBuilder() - .region("us-west-1"); + + var applicationPackageBuilder = new ApplicationPackageBuilder(); + jobTypes.stream().filter(JobType::isProduction).map(job -> job.zone().region().value()).forEach(applicationPackageBuilder::region); + instances.map(applicationPackageBuilder::instances); var applicationPackage = applicationPackageBuilder.build(); @@ -279,4 +332,23 @@ public class EndpointCertificateMaintainerTest { return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); } + private void prepareCertificatePool(int numCertificates) { + ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); + ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + + // Provision certificates + for (int i = 0; i < numCertificates; i++) { + certificatePoolMaintainer.maintain(); + } + + // Make certificate ready + EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); + List<EndpointCertificateRequest> endpointCertificateRequests = endpointCertificateProvider.listCertificates(); + endpointCertificateRequests.forEach(cert -> { + EndpointCertificateDetails details = endpointCertificateProvider.certificateDetails(cert.requestId()); + secretStore.setSecret(details.privateKeyKeyname(), "foo", 0); + secretStore.setSecret(details.certKeyKeyname(), "bar", 0); + }); + certificatePoolMaintainer.maintain(); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index bdbbc4b293f..228a61cebc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -69,6 +68,7 @@ public class NotificationsDbTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index ef1d9cd92e3..15524e2748c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -47,6 +46,7 @@ public class NotifierTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index dd7afa314ea..4369675ba3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.google.common.collect.ImmutableBiMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; @@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; @@ -91,7 +94,8 @@ public class TenantSerializerTest { Optional.of(contact()), Instant.EPOCH, lastLoginInfo(321L, 654L, 987L), - Instant.EPOCH); + Instant.EPOCH, + List.of()); AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.contact(), serialized.contact()); } @@ -109,6 +113,7 @@ public class TenantSerializerTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); @@ -133,6 +138,7 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role"), Optional.of(Instant.ofEpochMilli(1234567)), Instant.EPOCH, + List.of(), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); @@ -185,6 +191,8 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com"), Optional.empty(), Instant.EPOCH, + List.of(new CloudAccountInfo(CloudAccount.from("aws:123456789012"), Version.fromString("1.2.3")), + new CloudAccountInfo(CloudAccount.from("gcp:my-project"), Version.fromString("3.2.1"))), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role"); @@ -263,7 +271,8 @@ public class TenantSerializerTest { Optional.of(contact()), Instant.EPOCH, lastLoginInfo(321L, 654L, 987L), - Instant.ofEpochMilli(1_000_000)); + Instant.ofEpochMilli(1_000_000), + List.of()); assertEquals(tenant, serializer.tenantFrom(serializer.toSlime(tenant))); } @@ -281,6 +290,7 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com"), Optional.empty(), Instant.EPOCH, + List.of(), Optional.of(reference)); var slime = serializer.toSlime(tenant); var deserialized = serializer.tenantFrom(slime); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 4eb6e080737..3b74fea2b9c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -5,6 +5,7 @@ import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.RestApiException; @@ -26,12 +27,14 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -369,10 +372,10 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { new DeploymentTester(wrapped).newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) .submit() .deploy(); + tester.controller().tenants().updateCloudAccounts(tenantName, List.of(new CloudAccountInfo(CloudAccount.from("aws:123456789012"), new Version(1, 2, 4)))); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), - (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), - 200); + new File("tenant-cloud.json")); tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", PUT) .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ab70dfd6073..6b377e2069b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1372,7 +1372,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create legacy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, - new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY, Instant.EPOCH)); + new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY, Instant.EPOCH, List.of())); // POST (add) a Athenz tenant with dashes duplicates existing one with underscores tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json new file mode 100644 index 00000000000..c7258ab3aa6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json @@ -0,0 +1,35 @@ +{ + "tenant": "scoober", + "type": "CLOUD", + "creator": "developer@scoober", + "pemDeveloperKeys": [], + "secretStores": [], + "integrations": { + "aws": { + "tenantRole": "scoober-tenant-role", + "accounts": [] + } + }, + "quota": { + "budgetUsed": 1.304 + }, + "archiveAccess": {}, + "applications": [ + { + "tenant": "scoober", + "application": "albums", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/scoober/application/albums/instance/default" + } + ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastSubmissionToProdMillis": 1000 + }, + "cloudAccounts": [ + { + "cloudAccount": "aws:123456789012", + "templateVersion": "1.2.4" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index eb376a95c74..8b76613676c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -34,6 +34,9 @@ "name": "ChangeRequestMaintainer" }, { + "name": "CloudAccountVerifier" + }, + { "name": "CloudDatabaseMaintainer" }, { @@ -130,7 +133,5 @@ "name": "VersionStatusUpdater" } ], - "inactive": [ - "DeploymentExpirer" - ] + "inactive": ["DeploymentExpirer"] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 581f9704fc5..001e02e1b16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -70,17 +70,7 @@ public class SignatureFilterTest { filter = new SignatureFilter(tester.controller()); signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); - tester.curator().writeTenant(new CloudTenant(appId.tenant(), - Instant.EPOCH, - LastLoginInfo.EMPTY, - Optional.empty(), - ImmutableBiMap.of(), - TenantInfo.empty(), - List.of(), - new ArchiveAccess(), - Optional.empty(), - Instant.EPOCH, - Optional.empty())); + tester.curator().writeTenant(CloudTenant.create(appId.tenant(), Instant.EPOCH, null)); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @@ -129,6 +119,7 @@ public class SignatureFilterTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty())); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java index 779aee73dae..eb3f9daef53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -63,7 +63,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions are empty since this user is not authorized for any tenants - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of(), false, email1); @@ -72,7 +72,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions have filtered out tenant2 - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of("tenant1"), false, email1); @@ -81,7 +81,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match // Includes last value from DB which is not conditioned on email and the default from code - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code flagData, Set.of(), true, "operator@domain.tld"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 630de5137bb..b2b34441219 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -598,21 +598,28 @@ public class RoutingPoliciesTest { app.deploy(); - // TXT records are cleaned up as we go—the last challenge is the last to go here, and we must flush it ourselves. + // TXT records are cleaned up when deployments are deactivated. + // The last challenge is the last to go here, and we must flush it ourselves. assertEquals(List.of("a.t.aws-us-east-33a.vespa.oath.cloud", "challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), tester.recordNames()); app.flushDnsUpdates(); assertEquals(Set.of(new Record(Type.CNAME, RecordName.from("a.t.aws-us-east-33a.vespa.oath.cloud"), - RecordData.from("lb-0--t.a.default--prod.aws-us-east-33a."))), + RecordData.from("lb-0--t.a.default--prod.aws-us-east-33a.")), + new Record(Type.TXT, + RecordName.from("challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), + RecordData.from("system"))), tester.controllerTester().nameService().records()); + tester.controllerTester().controller().applications().deactivate(app.instanceId(), zone3); + app.flushDnsUpdates(); + assertEquals(Set.of(), + tester.controllerTester().nameService().records()); + // Deployment fails because challenge is not answered (immediately). tester.tester.controllerTester().serviceRegistry().vpcEndpointService().outcomes .put(RecordName.from("challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), ChallengeState.running); - - // Deployment fails because challenge is not answered (immediately). assertEquals("Status of run 2 of production-aws-us-east-33a for t.a ==> expected: <succeeded> but was: <unfinished>", assertThrows(AssertionError.class, () -> app.submit(appPackage).deploy()) @@ -1057,40 +1064,47 @@ public class RoutingPoliciesTest { int clustersPerZone = 2; var zone1 = ZoneId.from("prod", "aws-us-east-1c"); var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); + var zone3 = ZoneId.from("prod", "aws-us-east-1a"); // To test global endpoint pointing to two zones in same cloud-native region ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) + .region(zone3.region()) .container("c0", AuthMethod.mtls) .container("c1", AuthMethod.mtls, AuthMethod.token) .endpoint("foo", "c0") .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1))) .build(); - tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2, zone3); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates generated zone names List<String> expectedRecords = List.of( // save me, jebus! - "b36bf591.cafed00d.aws-us-east-1.w.vespa-app.cloud", + "a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", "b36bf591.cafed00d.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", "bc50b636.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c33db5ed.cafed00d.z.vespa-app.cloud", + "d467800f.cafed00d.z.vespa-app.cloud", "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.aws-eu-west-1.w.vespa-app.cloud", "dd0971b4.cafed00d.z.vespa-app.cloud", "eb48ad53.cafed00d.z.vespa-app.cloud", + "ec1e1288.cafed00d.z.vespa-app.cloud", "f2fa41ec.cafed00d.g.vespa-app.cloud", + "f411d177.cafed00d.z.vespa-app.cloud", "f4a4d111.cafed00d.a.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud", "foo.app1.tenant1.g.vespa-app.cloud" ); assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context.instanceId()).size()); + assertEquals(6, tester.policiesOf(context.instanceId()).size()); ClusterSpec.Id cluster0 = ClusterSpec.Id.from("c0"); ClusterSpec.Id cluster1 = ClusterSpec.Id.from("c1"); for (var zone : List.of(zone1, zone2)) { @@ -1107,13 +1121,17 @@ public class RoutingPoliciesTest { // Ordinary endpoints point to expected targets tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, - Map.of(zone1, 1L, zone2, 1L)); + ImmutableMap.of(zone1, 1L, + zone2, 1L, + zone3, 1L)); tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1)); // Generated endpoints point to expected targets tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, - Map.of(zone1, 1L, zone2, 1L), + ImmutableMap.of(zone1, 1L, + zone2, 1L, + zone3, 1L), true); tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1), @@ -1127,6 +1145,7 @@ public class RoutingPoliciesTest { // One endpoint is removed applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) + .region(zone3.region()) .container("c0", AuthMethod.mtls) .container("c1", AuthMethod.mtls, AuthMethod.token) .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1))) @@ -1138,13 +1157,18 @@ public class RoutingPoliciesTest { "bar.app1.tenant1.a.vespa-app.cloud", "bc50b636.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c33db5ed.cafed00d.z.vespa-app.cloud", + "d467800f.cafed00d.z.vespa-app.cloud", "d71005bf.cafed00d.z.vespa-app.cloud", "dd0971b4.cafed00d.z.vespa-app.cloud", "eb48ad53.cafed00d.z.vespa-app.cloud", + "ec1e1288.cafed00d.z.vespa-app.cloud", + "f411d177.cafed00d.z.vespa-app.cloud", "f4a4d111.cafed00d.a.vespa-app.cloud" ), tester.recordNames()); @@ -1157,6 +1181,35 @@ public class RoutingPoliciesTest { } @Test + public void generated_endpoints_only() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.controllerTester().flagSource() + .withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true) + .withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), zone1); + // ConfigServerMock provisions a load balancer for the "default" cluster, but in this scenario we need full + // control over the load balancer name because "default" has no special treatment when using generated endpoints + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), + 0, Map.of(zone1, 1L), true); + assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), + tester.recordNames()); + } + + @Test public void generated_endpoints_multi_instance() { var tester = new RoutingPoliciesTester(SystemName.Public); var context0 = tester.newDeploymentContext("tenant1", "app1", "default"); @@ -1213,6 +1266,32 @@ public class RoutingPoliciesTest { assertEquals(List.of(), tester.recordNames()); } + @Test + public void generated_endpoint_migration_with_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application + int clustersPerZone = 2; + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("foo"), 0, zone1, zone2); + + // Switch to generated + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), + 0, Map.of(zone1, 1L, zone2, 1L), true); + } + private void addCertificateToPool(String id, UnassignedCertificate.State state, RoutingPoliciesTester tester) { EndpointCertificate cert = new EndpointCertificate("testKey", "testCert", 1, 0, "request-id", @@ -1270,6 +1349,11 @@ public class RoutingPoliciesTest { .withCloudNativeRegionName("eu-west-1") .build(), ZoneApiMock.newBuilder() + .with(ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1a"))) + .with(CloudName.AWS) + .withCloudNativeRegionName("us-east-1") + .build(), + ZoneApiMock.newBuilder() .with(ZoneId.from(Environment.prod, RegionName.from("gcp-us-south1-b"))) .with(CloudName.GCP) .withCloudNativeRegionName("us-south1") |