diff options
Diffstat (limited to 'controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance')
49 files changed, 368 insertions, 235 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java index c8c5a1834c7..29e251a9de3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 1a944cfd5d7..d998413e675 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 33a4802360e..b6f73d6e5e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index a8d025dbb6a..8913d6e7166 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudAccount; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 1a2fc2f71c2..02cf7a85445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index d06dcd8e0b6..92aaacaa1f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.Bcp; @@ -71,6 +71,24 @@ public class BcpGroupUpdater extends ControllerMaintainer { var patch = new ApplicationPatch(); addTrafficShare(deployment, bcpGroups, patch); addBcpGroupInfo(deployment.zone().region(), metrics.get(instance.id()), bcpGroups, patch); + + StringBuilder patchAsStringBuilder = new StringBuilder("Patch of instance ").append(instance.id().serializedForm()).append(": ") + .append("\n\tcurrentReadShare: ") + .append(patch.currentReadShare) + .append("\n\tmaxReadShare: ") + .append(patch.maxReadShare); + for (Map.Entry<String, ApplicationPatch.ClusterPatch> entry : patch.clusters.entrySet()) { + String key = entry.getKey(); + ApplicationPatch.ClusterPatch value = entry.getValue(); + patchAsStringBuilder.append("\n\tbcpGroupInfo for ").append(key).append(": ") + .append("\n\t\tcpuCostPerQuery: ") + .append(value.bcpGroupInfo.cpuCostPerQuery) + .append("\n\t\tqueryRate: ") + .append(value.bcpGroupInfo.queryRate) + .append("\n\t\tgrowthRateHeadroom: ") + .append(value.bcpGroupInfo.growthRateHeadroom); + } + log.log(Level.FINER, patchAsStringBuilder.toString()); nodeRepository.patchApplication(deployment.zone(), instance.id(), patch); } catch (Exception e) { @@ -84,7 +102,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactorDeviation < -0.1 ) + else if ( successFactorDeviation < 0 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); return successFactorDeviation; } @@ -103,7 +121,9 @@ public class BcpGroupUpdater extends ControllerMaintainer { currentReadShare += groupQps == 0 ? 0 : fraction * deploymentQps / groupQps; maxReadShare += group.size() == 1 ? currentReadShare - : fraction * ( deploymentQps + group.maxQpsExcluding(deployment.zone().region()) / (group.size() - 1) ) / groupQps; + : groupQps != 0 + ? fraction * (deploymentQps + group.maxQpsExcluding(deployment.zone().region()) / (group.size() - 1)) / groupQps + : 0; } patch.currentReadShare = currentReadShare; patch.maxReadShare = maxReadShare; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java index b40078eef51..426abb16549 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index e7ec6675a82..7868c3fe611 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -1,12 +1,15 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillStatus; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -23,18 +26,25 @@ public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; private final BillingController billing; + private final BillingDatabaseClient databaseClient; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, Set.of(SystemName.PublicCd)); - this.reporter = controller.serviceRegistry().billingReporter(); - this.billing = controller.serviceRegistry().billingController(); - this.plans = controller.serviceRegistry().planRegistry(); + super(controller, interval, null, Set.of(SystemName.Public, SystemName.PublicCd)); + reporter = controller.serviceRegistry().billingReporter(); + billing = controller.serviceRegistry().billingController(); + databaseClient = controller.serviceRegistry().billingDatabase(); + plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { maintainTenants(); + + var updates = maintainInvoices(); + log.fine("Updated invoices: " + updates); + return 0.0; } @@ -53,6 +63,19 @@ public class BillingReportMaintainer extends ControllerMaintainer { }); } + InvoiceUpdate maintainInvoices() { + var billsNeedingMaintenance = databaseClient.readBills().stream() + .filter(bill -> bill.getExportedId().isPresent()) + .filter(exported -> exported.status() == BillStatus.OPEN) + .toList(); + + var updates = new InvoiceUpdate.Counter(); + for (var bill : billsNeedingMaintenance) { + updates.add(reporter.maintainInvoice(bill)); + } + return updates.finish(); + } + private Map<TenantName, CloudTenant> cloudTenants() { return controller().tenants().asList() .stream() @@ -74,4 +97,5 @@ public class BillingReportMaintainer extends ControllerMaintainer { .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) .toList(); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index ed383175cc3..5e6e495e473 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -1,7 +1,10 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; @@ -11,9 +14,9 @@ import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; @@ -30,7 +33,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Manages pool of ready-to-use randomized endpoint certificates + * Manages a pool of ready-to-use endpoint certificates. * * @author andreer */ @@ -44,7 +47,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { private final Metric metric; private final Controller controller; private final IntFlag certPoolSize; - private final String dnsSuffix; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; @@ -58,7 +60,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { this.curator = controller.curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); this.metric = metric; - this.dnsSuffix = Endpoint.dnsSuffix(controller.system()); } protected double maintain() { @@ -72,10 +73,10 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { - provisionRandomizedCertificate(); + provisionCertificate(); } } catch (Exception e) { - log.log(Level.SEVERE, "Exception caught while maintaining pool of unused randomized endpoint certs", e); + log.log(Level.SEVERE, "Failed to maintain certificate pool", e); return 1.0; } return 0.0; @@ -90,46 +91,49 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { OptionalInt maxCertVersion = secretStore.listSecretVersions(cert.certificate().certName()).stream().mapToInt(i -> i).max(); if (maxKeyVersion.isPresent() && maxCertVersion.equals(maxKeyVersion)) { curator.writeUnassignedCertificate(cert.withState(UnassignedCertificate.State.ready)); - log.log(Level.INFO, "Randomized endpoint cert %s now ready for use".formatted(cert.id())); + log.log(Level.INFO, "Readied certificate %s".formatted(cert.id())); } } catch (SecretNotFoundException s) { // Likely because the certificate is very recently provisioned - ignore till next time - should we log? - log.log(Level.INFO, "Could not yet read secrets for randomized endpoint cert %s - maybe next time ...".formatted(cert.id())); + log.log(Level.INFO, "Cannot ready certificate %s yet, will retry in %s".formatted(cert.id(), interval())); } } } } - private void provisionRandomizedCertificate() { + private void provisionCertificate() { try (Mutex lock = controller.curator().lockCertificatePool()) { Set<String> existingNames = controller.curator().readUnassignedCertificates().stream().map(UnassignedCertificate::id).collect(Collectors.toSet()); curator.readAssignedCertificates().stream() .map(AssignedCertificate::certificate) - .map(EndpointCertificate::randomizedId) + .map(EndpointCertificate::generatedId) .forEach(id -> id.ifPresent(existingNames::add)); - String id = generateRandomId(); - while (existingNames.contains(id)) id = generateRandomId(); - - EndpointCertificate f = endpointCertificateProvider.requestCaSignedCertificate( - "preprovisioned.%s".formatted(id), - List.of( - "*.%s.z%s".formatted(id, dnsSuffix), - "*.%s.g%s".formatted(id, dnsSuffix), - "*.%s.a%s".formatted(id, dnsSuffix) - ), - Optional.empty(), - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withRandomizedId(id); - - UnassignedCertificate certificate = new UnassignedCertificate(f, UnassignedCertificate.State.requested); + String id = generateId(); + while (existingNames.contains(id)) id = generateId(); + List<String> dnsNames = wildcardDnsNames(id); + EndpointCertificate cert = endpointCertificateProvider.requestCaSignedCertificate( + "preprovisioned.%s".formatted(id), + dnsNames, + Optional.empty(), + endpointCertificateAlgo.value(), + useAlternateCertProvider.value()).withGeneratedId(id); + + UnassignedCertificate certificate = new UnassignedCertificate(cert, UnassignedCertificate.State.requested); curator.writeUnassignedCertificate(certificate); } } - private String generateRandomId() { + private List<String> wildcardDnsNames(String id) { + DeploymentId defaultDeployment = new DeploymentId(ApplicationId.defaultId(), ZoneId.defaultId()); + return controller.routing().certificateDnsNames(defaultDeployment, // Not used for non-legacy names + DeploymentSpec.empty, // Not used for non-legacy names + id, + false); + } + + private String generateId() { return GeneratedEndpoint.createPart(controller.random(true)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index 5be20f9a994..51720806371 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java index 320f15a8b18..9f687249f38 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java index f0fc8985bdf..fedfea792f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java index 68fd5c8bafe..73204fb1655 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 18ef47759f4..55428e80493 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -1,26 +1,42 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.notification.MailTemplating; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; +import com.yahoo.vespa.hosted.controller.persistence.TrialNotifications; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRED; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_IMMEDIATELY; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_SOON; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.MID_CHECK_IN; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.SIGNED_UP; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.UNKNOWN; + /** * Expires unused tenants from Vespa Cloud. - * <p> - * TODO: Should support sending notifications some time before the various expiry events happen. * * @author ogronnesby */ @@ -28,19 +44,22 @@ public class CloudTrialExpirer extends ControllerMaintainer { private static final Logger log = Logger.getLogger(CloudTrialExpirer.class.getName()); private static final Duration nonePlanAfter = Duration.ofDays(14); - private static final Duration tombstoneAfter = Duration.ofDays(183); + private static final Duration tombstoneAfter = Duration.ofDays(91); private final ListFlag<String> extendedTrialTenants; + private final BooleanFlag cloudTrialNotificationEnabled; public CloudTrialExpirer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); this.extendedTrialTenants = PermanentFlags.EXTENDED_TRIAL_TENANTS.bindTo(controller().flagSource()); + this.cloudTrialNotificationEnabled = Flags.CLOUD_TRIAL_NOTIFICATIONS.bindTo(controller().flagSource()); } @Override protected double maintain() { var a = tombstoneNonePlanTenants(); var b = moveInactiveTenantsToNonePlan(); - return (a ? 0.5 : 0.0) + (b ? 0.5 : 0.0); + var c = notifyTenants(); + return (a ? 0.0 : -(1D/3)) + (b ? 0.0 : -(1D/3) + (c ? 0.0 : -(1D/3))); } private boolean moveInactiveTenantsToNonePlan() { @@ -76,6 +95,116 @@ public class CloudTrialExpirer extends ControllerMaintainer { return tombstoneTenants(idleOldPlanTenants); } + private boolean notifyTenants() { + try { + var currentStatus = controller().curator().readTrialNotifications() + .map(TrialNotifications::tenants).orElse(List.of()); + log.fine(() -> "Current: %s".formatted(currentStatus)); + var currentStatusByTenant = new HashMap<TenantName, TrialNotifications.Status>(); + currentStatus.forEach(status -> currentStatusByTenant.put(status.tenant(), status)); + var updatedStatus = new ArrayList<TrialNotifications.Status>(); + var now = controller().clock().instant(); + + for (var tenant : controller().tenants().asList()) { + + var status = currentStatusByTenant.get(tenant.name()); + var state = status == null ? UNKNOWN : status.state(); + var plan = controller().serviceRegistry().billingController().getPlan(tenant.name()).value(); + var ageInDays = Duration.between(tenant.createdAt(), now).toDays(); + + // TODO Replace stubs with proper email content stored in templates. + + var enabled = cloudTrialNotificationEnabled.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()).value(); + if (!enabled) { + if (status != null) updatedStatus.add(status); + } else if (!List.of("none", "trial").contains(plan)) { + // Ignore tenants that are on a paid plan and skip from inclusion in updated data structure + } else if (status == null && "trial".equals(plan) && ageInDays <= 1) { + updatedStatus.add(updatedStatus(tenant, now, SIGNED_UP)); + notifySignup(tenant); + } else if ("none".equals(plan) && !List.of(EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRED)); + notifyExpired(tenant); + } else if ("trial".equals(plan) && ageInDays >= 13 + && !List.of(EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_IMMEDIATELY)); + notifyExpiresImmediately(tenant); + } else if ("trial".equals(plan) && ageInDays >= 12 + && !List.of(EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_SOON)); + notifyExpiresSoon(tenant); + } else if ("trial".equals(plan) && ageInDays >= 7 + && !List.of(MID_CHECK_IN, EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, MID_CHECK_IN)); + notifyMidCheckIn(tenant); + } else { + updatedStatus.add(status); + } + } + log.fine(() -> "Updated: %s".formatted(updatedStatus)); + controller().curator().writeTrialNotifications(new TrialNotifications(updatedStatus)); + return true; + } catch (Exception e) { + log.log(Level.WARNING, "Failed to process trial notifications", e); + return false; + } + } + + private void notifySignup(Tenant tenant) { + var consoleMsg = "Welcome to Vespa Cloud trial! [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Welcome to Vespa Cloud", + "Welcome to Vespa Cloud! We hope you will enjoy your trial. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyMidCheckIn(Tenant tenant) { + var consoleMsg = "You're halfway through the **14 day** trial period. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "How is your Vespa Cloud trial going?", + "How is your Vespa Cloud trial going? " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpiresSoon(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial expires in **2** days. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial expires in 2 days", + "Your Vespa Cloud trial expires in 2 days. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpiresImmediately(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial expires **tomorrow**. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial expires tomorrow", + "Your Vespa Cloud trial expires tomorrow. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpired(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial has expired. [Upgrade plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial has expired", + "Your Vespa Cloud trial has expired. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void queueNotification(Tenant tenant, String consoleMsg, String emailSubject, String emailMsg) { + var mail = Optional.of(Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) + .subject(emailSubject) + .with("mailMessageTemplate", "cloud-trial-notification") + .with("cloudTrialMessage", emailMsg) + .with("mailTitle", emailSubject) + .with("consoleLink", controller().serviceRegistry().consoleUrls().tenantOverview(tenant.name())) + .build()); + var source = NotificationSource.from(tenant.name()); + // Remove previous notification to ensure new notification is sent by email + controller().notificationsDb().removeNotification(source, Notification.Type.account); + controller().notificationsDb().setNotification( + source, Notification.Type.account, Notification.Level.info, consoleMsg, List.of(), mail); + } + + private String billingUrl(Tenant t) { return controller().serviceRegistry().consoleUrls().tenantBilling(t.name()); } + + private static TrialNotifications.Status updatedStatus(Tenant t, Instant i, TrialNotifications.State s) { + return new TrialNotifications.Status(t.name(), s, i); + } private boolean tenantIsCloudTenant(Tenant tenant) { return tenant.type() == Tenant.Type.cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index f6da3609fbb..e0db7780fbb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index ff8fdf7ace4..3bc9126f835 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.concurrent.maintenance.JobMetrics; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 7afa10ab8d5..8d45fcb8878 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; @@ -86,6 +86,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer)); maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier)); + maintainers.add(new DataPlaneTokenRedeployer(controller, intervals.dataPlaneTokenRedeployer)); } public Upgrader upgrader() { return upgrader; } @@ -149,6 +150,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration certificatePoolMaintainer; private final Duration billingReportMaintainer; private final Duration cloudAccountVerifier; + private final Duration dataPlaneTokenRedeployer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -187,6 +189,7 @@ public class ControllerMaintenance extends AbstractComponent { this.certificatePoolMaintainer = duration(15, MINUTES); this.billingReportMaintainer = duration(60, MINUTES); this.cloudAccountVerifier = duration(10, MINUTES); + this.dataPlaneTokenRedeployer = duration(1, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 668893d5a7e..af8248c399c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java new file mode 100644 index 00000000000..e9d2dc0714b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DataPlaneTokenRedeployer.java @@ -0,0 +1,24 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Controller; + +import java.time.Duration; + +/** + * @author jonmv + */ +public class DataPlaneTokenRedeployer extends ControllerMaintainer { + + public DataPlaneTokenRedeployer(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected double maintain() { + controller().dataplaneTokenService().triggerTokenChangeDeployments(); + return 0; + } + + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index c22cb1efdb3..aea23e6def8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index b2b06cf281f..7b4ed9e1e98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index cd0f4be7a48..ae9eb1dc2b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 29266a25c5e..df1f793914e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.text.Text; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 82cac1e7520..270c388d73c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 6c1c4daa1bb..8fd9dc919fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudAccount; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 805bf3d7ada..e3e3e347c04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -1,35 +1,26 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; import com.yahoo.component.annotation.Inject; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -43,11 +34,9 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -67,9 +56,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - private final StringFlag endpointCertificateAlgo; - private final BooleanFlag useAlternateCertProvider; - private final IntFlag assignRandomizedIdRate; @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -80,9 +66,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); } @Override @@ -93,12 +76,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { updateRefreshedCertificates(); deleteUnusedCertificates(); deleteOrReportUnmanagedCertificates(); - assignRandomizedIds(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); return 1.0; } - return 0.0; } @@ -270,115 +251,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } } - private void assignRandomizedIds() { - List<AssignedCertificate> assignedCertificates = curator.readAssignedCertificates(); - /* - only assign randomized id if: - * instance is present - * randomized id is not already assigned - * feature flag is enabled - */ - assignedCertificates.stream() - .filter(c -> c.instance().isPresent()) - .filter(c -> c.certificate().randomizedId().isEmpty()) - .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion - .limit(assignRandomizedIdRate.value()) - .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); - } - - /* - Assign randomized id according to these rules: - * Instance is not mentioned in the deployment spec for this application - -> assume this is a manual deployment. Assign a randomized id to the certificate, save using instance only - * Instance is mentioned in deployment spec: - -> If there is a random endpoint assigned to tenant:application -> use this also for the "instance" certificate - -> Otherwise assign a random endpoint and write to the application and the instance. - */ - private void assignRandomizedId(TenantAndApplicationId tenantAndApplicationId, InstanceName instanceName) { - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.of(instanceName)); - if (assignedCertificate.isEmpty()) { - log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id"); - } - // Verify that the assigned certificate still does not have randomized id assigned - if (assignedCertificate.get().certificate().randomizedId().isPresent()) return; - - controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> { - DeploymentSpec deploymentSpec = application.get().deploymentSpec(); - if (deploymentSpec.instance(instanceName).isPresent()) { - Optional<AssignedCertificate> applicationLevelAssignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.empty()); - assignApplicationRandomId(assignedCertificate.get(), applicationLevelAssignedCertificate); - } else { - assignInstanceRandomId(assignedCertificate.get()); - } - }); - } - - private void assignApplicationRandomId(AssignedCertificate instanceLevelAssignedCertificate, Optional<AssignedCertificate> applicationLevelAssignedCertificate) { - TenantAndApplicationId tenantAndApplicationId = instanceLevelAssignedCertificate.application(); - if (applicationLevelAssignedCertificate.isPresent()) { - // Application level assigned certificate with randomized id already exists. Copy randomized id to instance level certificate and request with random names. - EndpointCertificate withRandomNames = requestRandomNames( - tenantAndApplicationId, - instanceLevelAssignedCertificate.instance(), - applicationLevelAssignedCertificate.get().certificate().randomizedId() - .orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing")), - Optional.of(instanceLevelAssignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } else { - // No application level certificate exists, generate new assigned certificate with the randomized id based names only, then request same names also for instance level cert - String randomId = generateRandomId(); - EndpointCertificate applicationLevelEndpointCert = requestRandomNames(tenantAndApplicationId, Optional.empty(), randomId, Optional.empty()); - AssignedCertificate applicationLevelCert = new AssignedCertificate(tenantAndApplicationId, Optional.empty(), applicationLevelEndpointCert); - - EndpointCertificate instanceLevelEndpointCert = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), randomId, Optional.of(instanceLevelAssignedCertificate.certificate())); - instanceLevelAssignedCertificate = instanceLevelAssignedCertificate.with(instanceLevelEndpointCert); - - // Save both in transaction - try (NestedTransaction transaction = new NestedTransaction()) { - curator.writeAssignedCertificate(instanceLevelAssignedCertificate, transaction); - curator.writeAssignedCertificate(applicationLevelCert, transaction); - transaction.commit(); - } - } - } - - private void assignInstanceRandomId(AssignedCertificate assignedCertificate) { - String randomId = generateRandomId(); - EndpointCertificate withRandomNames = requestRandomNames(assignedCertificate.application(), assignedCertificate.instance(), randomId, Optional.of(assignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = assignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } - - private EndpointCertificate requestRandomNames(TenantAndApplicationId tenantAndApplicationId, Optional<InstanceName> instanceName, String randomId, Optional<EndpointCertificate> previousRequest) { - String dnsSuffix = Endpoint.dnsSuffix(controller().system()); - List<String> newSanDnsEntries = List.of( - "*.%s.z%s".formatted(randomId, dnsSuffix), - "*.%s.g%s".formatted(randomId, dnsSuffix), - "*.%s.a%s".formatted(randomId, dnsSuffix)); - List<String> existingSanDnsEntries = previousRequest.map(EndpointCertificate::requestedDnsSans).orElse(List.of()); - List<String> requestNames = Stream.concat(existingSanDnsEntries.stream(), newSanDnsEntries.stream()).toList(); - String key = instanceName.map(tenantAndApplicationId::instance).map(ApplicationId::toFullString).orElseGet(tenantAndApplicationId::toString); - return endpointCertificateProvider.requestCaSignedCertificate( - key, - requestNames, - previousRequest, - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withRandomizedId(randomId); - } - - private String generateRandomId() { - List<String> unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList(); - List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::randomizedId).filter(Optional::isPresent).map(Optional::get).toList(); - Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); - String randomId; - do { - randomId = GeneratedEndpoint.createPart(controller().random(true)); - } while (allIds.contains(randomId)); - return randomId; - } - private static String asString(TenantAndApplicationId application, Optional<InstanceName> instanceName) { return application.toString() + instanceName.map(name -> "." + name.value()).orElse(""); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 31236f4fcda..5d6e60ee0bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index b1ea4584497..97bb709d423 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 94c3201edbe..0f482b1a015 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -1,7 +1,9 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import ai.vespa.metrics.ControllerMetrics; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; @@ -11,11 +13,14 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import java.time.Duration; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,22 +37,29 @@ public class JobRunner extends ControllerMaintainer { private final JobController jobs; private final ExecutorService executors; private final StepRunner runner; + private final Metrics metrics; public JobRunner(Controller controller, Duration duration) { - this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), new InternalStepRunner(controller)); + this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), + new InternalStepRunner(controller)); } public JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner) { + this(controller, duration, executors, runner, new Metrics(controller.metric(), Duration.ofMillis(100))); + } + + JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner, Metrics metrics) { super(controller, duration); this.jobs = controller.jobController(); this.jobs.setRunner(this::advance); this.executors = executors; this.runner = runner; + this.metrics = metrics; } @Override protected double maintain() { - executors.execute(() -> jobs.active().forEach(this::advance)); + execute(() -> jobs.active().forEach(this::advance)); jobs.collectGarbage(); return 1.0; } @@ -55,6 +67,7 @@ public class JobRunner extends ControllerMaintainer { @Override public void shutdown() { super.shutdown(); + metrics.shutdown(); executors.shutdown(); } @@ -62,9 +75,9 @@ public class JobRunner extends ControllerMaintainer { public void awaitShutdown() { super.awaitShutdown(); try { - if ( ! executors.awaitTermination(10, TimeUnit.SECONDS)) { + if ( ! executors.awaitTermination(40, TimeUnit.SECONDS)) { executors.shutdownNow(); - if ( ! executors.awaitTermination(40, TimeUnit.SECONDS)) + if ( ! executors.awaitTermination(10, TimeUnit.SECONDS)) throw new IllegalStateException("Failed shutting down " + JobRunner.class.getName()); } } @@ -83,14 +96,14 @@ public class JobRunner extends ControllerMaintainer { jobs.locked(id, run -> { if ( ! run.hasFailed() && controller().clock().instant().isAfter(run.sleepUntil().orElse(run.start()).plus(jobTimeout))) - executors.execute(() -> { + execute(() -> { jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached", false); advance(run.id()); }); else if (run.readySteps().isEmpty()) - executors.execute(() -> finish(run.id())); + execute(() -> finish(run.id())); else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) - run.readySteps().forEach(step -> executors.execute(() -> advance(run.id(), step))); + run.readySteps().forEach(step -> execute(() -> advance(run.id(), step))); return null; }); @@ -145,4 +158,39 @@ public class JobRunner extends ControllerMaintainer { } } + private void execute(Runnable task) { + metrics.queued.incrementAndGet(); + executors.execute(() -> { + metrics.queued.decrementAndGet(); + metrics.active.incrementAndGet(); + try { task.run(); } + finally { metrics.active.decrementAndGet(); } + }); + } + + static class Metrics { + + private final AtomicInteger queued = new AtomicInteger(); + private final AtomicInteger active = new AtomicInteger(); + private final ScheduledExecutorService reporter = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("job-runner-metrics-")); + private final Metric metric; + private final Metric.Context context; + + Metrics(Metric metric, Duration interval) { + this.metric = metric; + this.context = metric.createContext(Map.of()); + reporter.scheduleAtFixedRate(this::report, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); + } + + void report() { + metric.set(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), queued.get(), context); + metric.set(ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), active.get(), context); + } + + void shutdown() { + reporter.shutdown(); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java index 519b1001be4..396ec1ec6f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 6a280e71e98..6f070cbba84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index e4841618852..3ee9650d4ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index dbca6681281..a712c4f35d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 44f0bcecf5f..25a0abbce90 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index a3a866d2036..2fd92970bc9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 37b06fea066..6c414e44a96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 974345330aa..e35bb139142 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 945b6d32a30..0668f8c481c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 6ee1a8b56d7..5cadd13309b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,9 +1,10 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; @@ -18,11 +19,9 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -199,14 +198,9 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) .filter(this::isClusterTypeMeterable) - // Grouping by ApplicationId -> Architecture -> ResourceSnapshot - .collect(Collectors.groupingBy(node -> - node.owner().get(), - groupSnapshotsByArchitectureAndMajorVersion(zoneId))) + .collect(groupSnapshots(zoneId)) .values() .stream() - .flatMap(byArch -> byArch.values().stream()) - .flatMap(byMajor -> byMajor.values().stream()) .toList(); } @@ -281,17 +275,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { )); } - private Collector<Node, ?, Map<NodeResources.Architecture, Map<Integer, ResourceSnapshot>>> groupSnapshotsByArchitectureAndMajorVersion(ZoneId zoneId) { - return Collectors.groupingBy( - (Node node) -> node.resources().architecture(), - Collectors.collectingAndThen( - Collectors.groupingBy( - (Node node) -> node.wantedVersion().getMajor(), - Collectors.toList()), - convertNodeListToResourceSnapshot(zoneId))); + private Collector<Node, ?, Map<ResourceKey, ResourceSnapshot>> groupSnapshots(ZoneId zoneId) { + return Collectors.collectingAndThen( + Collectors.groupingBy( + (Node node) -> new ResourceKey(node.owner().get(), node.resources().architecture(), node.wantedVersion().getMajor(), node.cloudAccount()), + Collectors.toList()), + convertNodeListToResourceSnapshot(zoneId)); } - private Function<Map<Integer, List<Node>>, Map<Integer, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { + private Function<Map<ResourceKey, List<Node>>, Map<ResourceKey, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { return nodesByMajor -> { return nodesByMajor.entrySet().stream() .collect(Collectors.toMap( @@ -299,4 +291,10 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { entry -> ResourceSnapshot.from(entry.getValue(), clock.instant(), zoneId))); }; } + + private record ResourceKey( + ApplicationId applicationId, + NodeResources.Architecture architecture, + int majorVersion, + CloudAccount account) {} } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 59871f716e0..a0c94c0b9a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index aaf730cc158..3cbd7b3e0e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index effcc4dd4df..c31f81497e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java index e3a3415e170..5539c62be98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index 5f7dae5a352..f76b7634c62 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index a929a1d7af8..dceb3921061 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -42,10 +43,16 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; + private final Random random; public Upgrader(Controller controller, Duration interval) { + this(controller, interval, controller.random(false)); + } + + Upgrader(Controller controller, Duration interval, Random random) { super(controller, interval); this.curator = controller.curator(); + this.random = random; } /** @@ -75,7 +82,7 @@ public class Upgrader extends ControllerMaintainer { private InstanceList instances(DeploymentStatusList deploymentStatuses) { return InstanceList.from(deploymentStatuses) .withDeclaredJobs() - .shuffle(controller().random(false)) + .shuffle(random) .byIncreasingDeployedVersion() .unpinned(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 7c4645a6e48..0f39ef7d0f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index df6e89fb5ef..b0d7a0c47e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 1c4d13aa16d..721819522f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java index 6ff3e2d1c52..f8eed5804bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.controller.maintenance; |