diff options
author | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
---|---|---|
committer | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
commit | e9058b555d4dfea2f6c872d9a677e8678b569569 (patch) | |
tree | fa1b67c6e39712c1e0d9f308b0dd55573b43f913 /controller-server/src/main/java/com/yahoo | |
parent | 0ad931fa86658904fe9212b014d810236b0e00e4 (diff) | |
parent | 16030193ec04ee41e98779a3d7ee6a6c1d0d0d6f (diff) |
Merge branch 'master' into bjormel/aws-main-controller
Diffstat (limited to 'controller-server/src/main/java/com/yahoo')
20 files changed, 337 insertions, 87 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 6ec732a3815..7d19acfce80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; @@ -43,12 +44,14 @@ public abstract class LockedTenant { final Instant createdAt; final LastLoginInfo lastLoginInfo; final Instant tenantRolesLastMaintained; + final List<CloudAccountInfo> cloudAccounts; - private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { + private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { this.name = requireNonNull(name); this.createdAt = requireNonNull(createdAt); this.lastLoginInfo = requireNonNull(lastLoginInfo); this.tenantRolesLastMaintained = requireNonNull(tenantRolesLastMaintained); + this.cloudAccounts = requireNonNull(cloudAccounts); } static LockedTenant of(Tenant tenant, Mutex lock) { @@ -66,6 +69,8 @@ public abstract class LockedTenant { public abstract LockedTenant with(Instant tenantRolesLastMaintained); + public abstract LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts); + public Deleted deleted(Instant deletedAt) { return new Deleted(new DeletedTenant(name, createdAt, deletedAt)); } @@ -85,8 +90,8 @@ public abstract class LockedTenant { private final Optional<Contact> contact; private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.domain = domain; this.property = property; this.propertyId = propertyId; @@ -94,38 +99,43 @@ public abstract class LockedTenant { } private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained()); + this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts()); } @Override public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } } @@ -146,8 +156,8 @@ public abstract class LockedTenant { BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained, - Optional<BillingReference> billingReference) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; @@ -158,12 +168,12 @@ public abstract class LockedTenant { } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.billingReference()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -174,51 +184,56 @@ public abstract class LockedTenant { if (keys.inverse().containsKey(simplePrincipal)) throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); keys.put(key, simplePrincipal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud with(BillingReference billingReference) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, Optional.of(billingReference)); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, Optional.of(billingReference)); } } @@ -229,7 +244,7 @@ public abstract class LockedTenant { private final Instant deletedAt; private Deleted(DeletedTenant tenant) { - super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH); + super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH, List.of()); this.deletedAt = tenant.deletedAt(); } @@ -247,6 +262,11 @@ public abstract class LockedTenant { public LockedTenant with(Instant tenantRolesLastMaintained) { return this; } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return this; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 091836a1eea..b1ffce65852 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -82,7 +82,8 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag randomizedEndpoints; + private final BooleanFlag generatedEndpoints; + private final BooleanFlag legacyEndpoints; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -90,7 +91,8 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.randomizedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.generatedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.legacyEndpoints = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -228,13 +230,14 @@ public class RoutingController { .in(controller.system())); // Only a single region endpoint is needed, not one per auth method if (isProduction && generatedEndpoint.authMethod() == AuthMethod.mtls) { - endpoints.add(regionEndpoint.generatedFrom(generatedEndpoint) + GeneratedEndpoint weightedGeneratedEndpoint = generatedEndpoint.withClusterPart(weightedClusterPart(cluster, deployment)); + endpoints.add(regionEndpoint.generatedFrom(weightedGeneratedEndpoint) .authMethod(AuthMethod.none) .in(controller.system())); } } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(deployment.applicationId(), EndpointList.copyOf(endpoints)); } /** Read routing policies and return zone- and region-scoped endpoints for given deployment */ @@ -268,7 +271,7 @@ public class RoutingController { endpoints.add(builder.generatedFrom(ge).authMethod(ge.authMethod()).in(controller.system())); } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(routingId.instance(), EndpointList.copyOf(endpoints)); } /** Returns application endpoints pointing to given deployments */ @@ -424,6 +427,13 @@ public class RoutingController { Optional.of(application.id()))); } + private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { + if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { + return endpoints.generated(); + } + return endpoints; + } + private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId()); EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global); @@ -476,6 +486,22 @@ public class RoutingController { .toList(); } + /** Generate the cluster part of a {@link GeneratedEndpoint} for use in a {@link Endpoint.Scope#weighted} endpoint */ + private String weightedClusterPart(ClusterSpec.Id cluster, DeploymentId deployment) { + // This ID must be common for a given cluster in all deployments within the same cloud-native region + String cloudNativeRegion = controller.zoneRegistry().zones().all().get(deployment.zoneId()).get().getCloudNativeRegionName(); + HashCode hash = Hashing.sha256().newHasher() + .putString(cluster.value(), StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(cloudNativeRegion, StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(deployment.applicationId().serializedForm(), StandardCharsets.UTF_8) + .hash(); + String alphabet = "abcdef"; + char letter = alphabet.charAt(Math.abs(hash.asInt()) % alphabet.length()); + return letter + hash.toString().substring(0, 7); + } + /** Returns existing generated endpoints, grouped by their {@link Scope#multiDeployment()} endpoint */ private Map<EndpointId, GeneratedEndpointList> readDeclaredGeneratedEndpoints(TenantAndApplicationId application) { Map<EndpointId, GeneratedEndpointList> endpoints = new HashMap<>(); @@ -525,7 +551,17 @@ public class RoutingController { } public boolean generatedEndpointsEnabled(ApplicationId instance) { - return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()).value(); + return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); + } + + public boolean legacyEndpointsEnabled(ApplicationId instance) { + return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); } private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index bf2f2ab90eb..d11540b28dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -165,6 +166,14 @@ public class TenantController { } } + public void updateCloudAccounts(TenantName tenantName, List<CloudAccountInfo> cloudAccounts) { + try (Mutex lock = lock(tenantName)) { + var tenant = require(tenantName); + if (tenant.cloudAccounts().equals(cloudAccounts)) return; // no change + curator.writeTenant(LockedTenant.of(tenant, lock).withCloudAccounts(cloudAccounts).get()); + } + } + /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { try (Mutex lock = lock(tenant)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java index 8db4492356a..28f9963f24c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -38,6 +38,11 @@ public record GeneratedEndpoint(String clusterPart, String applicationPart, Auth return !declared(); } + /** Returns a copy of this with cluster part set to given value */ + public GeneratedEndpoint withClusterPart(String clusterPart) { + return new GeneratedEndpoint(clusterPart, applicationPart, authMethod, endpoint); + } + /** Create a new endpoint part, using random as a source of randomness */ public static String createPart(RandomGenerator random) { String alphabet = "abcdef0123456789"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index e01da00a27e..33af58a9790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,6 +11,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; @@ -57,6 +58,7 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; + private final BooleanFlag assignLegacyNames; private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, @@ -64,6 +66,7 @@ public class EndpointCertificates { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); + this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; @@ -140,10 +143,11 @@ public class EndpointCertificates { } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, candidate.get().certificate()), + EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); + curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), transaction); transaction.commit(); - return candidate.get().certificate(); + return certificate; } } } @@ -174,9 +178,12 @@ public class EndpointCertificates { } // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id + // Skip this validation for now if the cert has a randomized id and should not provision legacy names Optional<EndpointCertificate> currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - var requiredSansForZone = currentCertificate.get().randomizedId().isEmpty() ? + boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); + + var requiredSansForZone = legacyNames || currentCertificate.get().randomizedId().isEmpty() ? controller.routing().certificateDnsNames(deployment, deploymentSpec) : List.<String>of(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e247d6baa09..1b40781fe0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -276,7 +276,7 @@ public class DeploymentTrigger { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); - if(newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun()>=requiredEntry.requiredRun())) { + if (newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun() >= requiredEntry.requiredRun())) { newList.add(requiredEntry); } newList = newList.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 919facee0c1..11c47d8f481 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -62,7 +63,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -362,21 +362,24 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), - Optional.of(platform)); + // In manually deployed zones it is allowed for some model versions not being built (e.g due to incompatibility) + // but deployment still succeeding, so we cannot use version when checking for config convergence + Optional<Version> platformVersion = id.type().environment().isManuallyDeployed() ? Optional.empty() : Optional.of(platform); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), + platformVersion); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .applications(id.application()) - .states(active)); + List<Node> nodes = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .applications(id.application()) + .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); NodeList nodeList = NodeList.of(nodes, parents, services.get()); ConvergenceSummary summary = nodeList.summary(); @@ -496,8 +499,8 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), - Optional.of(platform)); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(testerId, zone), + Optional.of(platform)); if (services.isEmpty()) { if (run.stepInfo(installTester).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(30)))) { logger.log(WARNING, "Config status not available after 30 minutes; giving up!"); @@ -508,14 +511,14 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .applications(testerId) - .states(active, reserved)); + List<Node> nodes = configServer().nodeRepository().list(zone, + NodeFilter.all() + .applications(testerId) + .states(active, reserved)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(zone, + NodeFilter.all() + .hostnames(parentHostnames)); NodeList nodeList = NodeList.of(nodes, parents, services.get()); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, false)) @@ -534,6 +537,8 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private ConfigServer configServer() { return controller.serviceRegistry().configServer(); } + /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { DeploymentId deploymentId = new DeploymentId(id, zoneId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index d10e38fd990..e7ec6675a82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -2,23 +2,76 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; + private final BillingController billing; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { super(controller, interval, null, Set.of(SystemName.PublicCd)); this.reporter = controller.serviceRegistry().billingReporter(); + this.billing = controller.serviceRegistry().billingController(); + this.plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { - return this.reporter.maintain(); + maintainTenants(); + return 0.0; + } + + private void maintainTenants() { + var tenants = cloudTenants(); + var tenantNames = List.copyOf(tenants.keySet()); + var billableTenants = billableTenants(tenantNames); + + billableTenants.forEach(tenant -> { + controller().tenants().lockIfPresent(tenant, LockedTenant.Cloud.class, locked -> { + var ref = reporter.maintainTenant(locked.get()); + if (locked.get().billingReference().isEmpty() || ! locked.get().billingReference().get().equals(ref)) { + controller().tenants().store(locked.with(ref)); + } + }); + }); + } + + private Map<TenantName, CloudTenant> cloudTenants() { + return controller().tenants().asList() + .stream() + .filter(CloudTenant.class::isInstance) + .map(CloudTenant.class::cast) + .collect(Collectors.toMap( + Tenant::name, + Function.identity())); + } + + private List<Plan> billablePlans() { + return plans.all().stream() + .filter(Plan::isBilled) + .toList(); + } + + private List<TenantName> billableTenants(List<TenantName> tenants) { + return billablePlans().stream() + .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) + .toList(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 70eeb2b9f6c..ed383175cc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { // Create metric for available certificates in the pool as a fraction of configured size int poolSize = certPoolSize.value(); long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of())); + metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { provisionRandomizedCertificate(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java new file mode 100644 index 00000000000..f0fc8985bdf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.WARNING; + +/** + * Verifies the cloud accounts that may be used by a given user have applied the enclave template + * and extracts the version of the applied template. + * + * All maintainers that operate on external cloud accounts should use the list on the Tenant instance + * maintained by this class rather than the cloud-accounts feature flag. + * + * The template version can be used to determine if new features can be enabled for the cloud account. + * + * @author freva + */ +public class CloudAccountVerifier extends ControllerMaintainer { + + private static final Logger logger = Logger.getLogger(CloudAccountVerifier.class.getName()); + + CloudAccountVerifier(Controller controller, Duration interval) { + super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public)); + } + + @Override + protected double maintain() { + int attempts = 0, failures = 0; + for (Tenant tenant : controller().tenants().asList()) { + try { + attempts++; + List<CloudAccountInfo> cloudAccountInfos = controller().applications().accountsOf(tenant.name()).stream() + .flatMap(account -> controller().serviceRegistry() + .archiveService() + .getEnclaveTemplateVersion(account) + .map(version -> new CloudAccountInfo(account, version)) + .stream()) + .toList(); + controller().tenants().updateCloudAccounts(tenant.name(), cloudAccountInfos); + } catch (RuntimeException e) { + logger.log(WARNING, "Failed to verify cloud accounts for tenant " + tenant.name(), e); + failures++; + } + } + return asSuccessFactorDeviation(attempts, failures); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index f9c93a87c44..f6da3609fbb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -29,8 +29,8 @@ public class ContactInformationMaintainer extends ControllerMaintainer { private final ContactRetriever contactRetriever; - public ContactInformationMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + public ContactInformationMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)), successFactorBaseline); this.contactRetriever = controller.serviceRegistry().contactRetriever(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 6fae732df0a..7afa10ab8d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -59,7 +59,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); - maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); + maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer, successFactorBaseline.contactInformationMaintainerBaseline)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase())); @@ -85,6 +85,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval)); maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer)); + maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier)); } public Upgrader upgrader() { return upgrader; } @@ -147,6 +148,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration meteringMonitorMaintainer; private final Duration certificatePoolMaintainer; private final Duration billingReportMaintainer; + private final Duration cloudAccountVerifier; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -184,6 +186,7 @@ public class ControllerMaintenance extends AbstractComponent { this.meteringMonitorMaintainer = duration(30, MINUTES); this.certificatePoolMaintainer = duration(15, MINUTES); this.billingReportMaintainer = duration(60, MINUTES); + this.cloudAccountVerifier = duration(10, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { @@ -201,12 +204,14 @@ public class ControllerMaintenance extends AbstractComponent { private final Double deploymentMetricsMaintainerBaseline; private final Double trafficFractionUpdater; private final Double deploymentInfoMaintainerBaseline; + private final Double contactInformationMaintainerBaseline; public SuccessFactorBaseline(SystemName system) { Objects.requireNonNull(system); this.deploymentMetricsMaintainerBaseline = 0.90; this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65; this.deploymentInfoMaintainerBaseline = system.isCd() ? 0.5 : 0.95; + this.contactInformationMaintainerBaseline = 0.95; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 5218da91c46..6c1c4daa1bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -33,7 +33,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { private Set<CloudAccount> externalAccounts() { Set<CloudAccount> accounts = new HashSet<>(); for (Tenant tenant : controller().tenants().asList()) - accounts.addAll(controller().applications().accountsOf(tenant.name())); + tenant.cloudAccounts().forEach(accountInfo -> accounts.add(accountInfo.cloudAccount())); return accounts; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index c90fcb81c71..805bf3d7ada 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -67,7 +67,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - final BooleanFlag assignRandomizedId; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; private final IntFlag assignRandomizedIdRate; @@ -81,7 +80,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.assignRandomizedId = Flags.ASSIGN_RANDOMIZED_ID.bindTo(controller.flagSource()); this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); @@ -283,7 +281,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { assignedCertificates.stream() .filter(c -> c.instance().isPresent()) .filter(c -> c.certificate().randomizedId().isEmpty()) - .filter(c -> assignRandomizedId.with(FetchVector.Dimension.INSTANCE_ID, c.application().instance(c.instance().get()).serializedForm()).value()) .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion .limit(assignRandomizedIdRate.value()) .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a25aa9797ba..dc9c4650191 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -7,6 +7,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -602,7 +603,7 @@ public class CuratorDb { public List<DnsChallenge> readDnsChallenges(DeploymentId id) { return curator.getChildren(dnsChallengePath(id)).stream() - .map(cluster -> readDnsChallenge(new ClusterId(id, ClusterSpec.Id.from(cluster)))) + .map(cluster -> readDnsChallenge(new ClusterId(id, Id.from(cluster)))) .toList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index e3d61c81667..760fb9b0366 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -20,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; @@ -85,6 +88,9 @@ public class TenantSerializer { private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore"; private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained"; private static final String billingReferenceField = "billingReference"; + private static final String cloudAccountsField = "cloudAccounts"; + private static final String accountField = "account"; + private static final String templateVersionField = "templateVersion"; private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -97,6 +103,7 @@ public class TenantSerializer { tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); tenantObject.setLong(tenantRolesLastMaintainedField, tenant.tenantRolesLastMaintained().toEpochMilli()); + cloudAccountsToSlime(tenant.cloudAccounts(), tenantObject.setArray(cloudAccountsField)); switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; @@ -162,6 +169,14 @@ public class TenantSerializer { } } + private void cloudAccountsToSlime(List<CloudAccountInfo> cloudAccounts, Cursor cloudAccountsObject) { + cloudAccounts.forEach(cloudAccountInfo -> { + Cursor object = cloudAccountsObject.addObject(); + object.setString(accountField, cloudAccountInfo.cloudAccount().account()); + object.setString(templateVersionField, cloudAccountInfo.templateVersion().toFullString()); + }); + } + public Tenant tenantFrom(Slime slime) { Inspector tenantObject = slime.get(); Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); @@ -183,7 +198,8 @@ public class TenantSerializer { Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccountInfos); } private CloudTenant cloudTenantFrom(Inspector tenantObject) { @@ -197,8 +213,9 @@ public class TenantSerializer { ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); Optional<Instant> invalidateUserSessionsBefore = SlimeUtils.optionalInstant(tenantObject.field(invalidateUserSessionsBeforeField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); Optional<BillingReference> billingReference = billingReferenceFrom(tenantObject.field(billingReferenceField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccountInfos, billingReference); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -284,6 +301,14 @@ public class TenantSerializer { return new LastLoginInfo(lastLoginByUserLevel); } + private List<CloudAccountInfo> cloudAccountsFromSlime(Inspector cloudAccountsObject) { + return SlimeUtils.entriesStream(cloudAccountsObject) + .map(inspector -> new CloudAccountInfo( + CloudAccount.from(inspector.field(accountField).asString()), + Version.fromString(inspector.field(templateVersionField).asString()))) + .toList(); + } + void toSlime(TenantInfo info, Cursor parentCursor) { if (info.isEmpty()) return; Cursor infoCursor = parentCursor.setObject("info"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 46c81fc073f..16d862a66ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2915,6 +2915,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); + + if (!tenant.cloudAccounts().isEmpty()) { + Cursor cloudAccounts = object.setArray("cloudAccounts"); + tenant.cloudAccounts().forEach(accountInfo -> { + Cursor accountObject = cloudAccounts.addObject(); + accountObject.setString("cloudAccount", accountInfo.cloudAccount().value()); + accountObject.setString("templateVersion", accountInfo.templateVersion().toFullString()); + }); + } } private void toSlime(ArchiveAccess archiveAccess, Cursor object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 67dd172fd83..c5fb1afbae8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -85,6 +85,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") + .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") .get(self::plans)) .addExceptionMapper(RuntimeException.class, (c, e) -> ErrorResponses.logThrowing(c.request(), log, e)) @@ -262,6 +264,19 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return new SlimeJsonResponse(slime); } + private HttpResponse putAccountantInvoiceExport(RestApi.RequestContext ctx, Slime slime) { + var billId = ctx.attributes().get("invoice") + .map(id -> Bill.Id.of((String) id)) + .orElseThrow(() -> new RestApiException.BadRequest("Missing bill ID")); + + // TODO: try to find a way to retrieve the cloud tenant from BillingControllerImpl + var bill = billing.getBill(billId); + var cloudTenant = tenants.require(bill.tenant(), CloudTenant.class); + + var exportMethod = slime.get().field("method").asString(); + var result = billing.exportBill(bill, exportMethod, cloudTenant); + return new MessageResponse("Bill has been exported: " + result); + } // --------- INVOICE RENDERING ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index de25161c461..a21c6548a0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -263,8 +264,14 @@ public class RoutingPolicies { } else { weightedEndpoints = weightedEndpoints.not().generated(); } + if (generated && weightedEndpoints.isEmpty()) { + // Ignore this policy. If an instance has a global endpoint, and is switching from non-generated to + // generated endpoints we cannot update global DNS record for a deployment until it has been deployed at + // least once (which assigns a generated endpoint). + continue; + } if (weightedEndpoints.size() != 1) { - throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent); + throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent + ", got " + weightedEndpoints); } Endpoint endpoint = weightedEndpoints.first().get(); RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( @@ -410,24 +417,22 @@ public class RoutingPolicies { new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); nameServiceForwarder(endpoint).createRecord(record, Priority.normal, ownerOf(deploymentId)); - setPrivateDns(endpoint, loadBalancer, deploymentId); } + setPrivateDns(zoneEndpoints, loadBalancer, deploymentId); } - private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) { + private void setPrivateDns(EndpointList endpoints, LoadBalancer loadBalancer, DeploymentId deploymentId) { if (loadBalancer.service().isEmpty()) return; - // TODO(mpolden): Why is this done? Consider creating private DNS for all auth methods - boolean skipBasedOnAuthMethod = switch (endpoint.authMethod()) { - case token -> true; - case mtls -> false; - case none -> true; - }; - if (skipBasedOnAuthMethod) return; + // TODO(mpolden): Model one service for each endpoint (type), to allow private endpoints with tokens. + EndpointList mtlsEndpoints = endpoints.authMethod(AuthMethod.mtls); + if (mtlsEndpoints.isEmpty()) return; + Endpoint endpoint = mtlsEndpoints.generated().first().orElse(mtlsEndpoints.first().get()); if (endpoint.routingMethod() != RoutingMethod.exclusive) return; // Not supported for this routing method controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), - loadBalancer.cloudAccount()) + loadBalancer.cloudAccount(), + endpoint.generated().isPresent()) .ifPresent(challenge -> { try (Mutex lock = db.lockNameServiceQueue()) { controller.nameServiceForwarder().createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); @@ -436,10 +441,18 @@ public class RoutingPolicies { }); } + /** Deletes all DNS challenges, and corresponding TXT records, for the given deployment. */ + public void removeDnsChallenges(DeploymentId deploymentId) { + try (Mutex lock = db.lockNameServiceQueue()) { + db.readDnsChallenges(deploymentId).forEach(this::removeDnsChallenge); + } + } + /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ public boolean processDnsChallenges(DeploymentId deploymentId) { try (Mutex lock = db.lockNameServiceQueue()) { List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); + challenges.removeIf(challenge -> challenge.state() == ChallengeState.done); Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() .map(NameServiceRequest::name) .collect(Collectors.toSet()); @@ -450,14 +463,8 @@ public class RoutingPolicies { challenge = challenge.withState(ChallengeState.ready); } ChallengeState state = controller.serviceRegistry().vpcEndpointService().process(challenge); - if (state == ChallengeState.done) { - removeDnsChallenge(challenge); - return true; - } - else { - db.writeDnsChallenge(challenge.withState(state)); - return false; - } + db.writeDnsChallenge(challenge.withState(state)); + return state == ChallengeState.done; }); return challenges.isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index df0226176a2..99f60735f6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -57,6 +57,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { /** Deactivate routing configuration for the deployment in this context, using given deployment spec */ public final void deactivate(DeploymentSpec deploymentSpec) { routing.policies().refresh(deployment, deploymentSpec, EndpointList.EMPTY); + routing.policies().removeDnsChallenges(deployment); } /** Routing method of this context */ |