Merge branch 'master' into bjormel/aws-main-controller

7 files changed, 119 insertions, 9 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java
index d10e38fd990..e7ec6675a82 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java
@@ -2,23 +2,76 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.LockedTenant;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry;
+import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.time.Duration;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class BillingReportMaintainer extends ControllerMaintainer {
 
     private final BillingReporter reporter;
+    private final BillingController billing;
+    private final PlanRegistry plans;
 
     public BillingReportMaintainer(Controller controller, Duration interval) {
         super(controller, interval, null, Set.of(SystemName.PublicCd));
         this.reporter = controller.serviceRegistry().billingReporter();
+        this.billing = controller.serviceRegistry().billingController();
+        this.plans = controller.serviceRegistry().planRegistry();
     }
 
     @Override
     protected double maintain() {
-        return this.reporter.maintain();
+        maintainTenants();
+        return 0.0;
+    }
+
+    private void maintainTenants() {
+        var tenants = cloudTenants();
+        var tenantNames = List.copyOf(tenants.keySet());
+        var billableTenants = billableTenants(tenantNames);
+
+        billableTenants.forEach(tenant -> {
+            controller().tenants().lockIfPresent(tenant, LockedTenant.Cloud.class, locked -> {
+                var ref = reporter.maintainTenant(locked.get());
+                if (locked.get().billingReference().isEmpty() || ! locked.get().billingReference().get().equals(ref)) {
+                    controller().tenants().store(locked.with(ref));
+                }
+            });
+        });
+    }
+
+    private Map<TenantName, CloudTenant> cloudTenants() {
+        return controller().tenants().asList()
+                .stream()
+                .filter(CloudTenant.class::isInstance)
+                .map(CloudTenant.class::cast)
+                .collect(Collectors.toMap(
+                        Tenant::name,
+                        Function.identity()));
+    }
+
+    private List<Plan> billablePlans() {
+        return plans.all().stream()
+                .filter(Plan::isBilled)
+                .toList();
+    }
+
+    private List<TenantName> billableTenants(List<TenantName> tenants) {
+        return billablePlans().stream()
+                .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream())
+                .toList();
     }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
index 70eeb2b9f6c..ed383175cc3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
@@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer {
             // Create metric for available certificates in the pool as a fraction of configured size
             int poolSize = certPoolSize.value();
             long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count();
-            metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of()));
+            metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of()));
 
             if (certificatePool.size() < poolSize) {
                 provisionRandomizedCertificate();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java
new file mode 100644
index 00000000000..f0fc8985bdf
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java
@@ -0,0 +1,55 @@
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static java.util.logging.Level.WARNING;
+
+/**
+ * Verifies the cloud accounts that may be used by a given user have applied the enclave template
+ * and extracts the version of the applied template.
+ *
+ * All maintainers that operate on external cloud accounts should use the list on the Tenant instance
+ * maintained by this class rather than the cloud-accounts feature flag.
+ *
+ * The template version can be used to determine if new features can be enabled for the cloud account.
+ *
+ * @author freva
+ */
+public class CloudAccountVerifier extends ControllerMaintainer {
+
+    private static final Logger logger = Logger.getLogger(CloudAccountVerifier.class.getName());
+
+    CloudAccountVerifier(Controller controller, Duration interval) {
+        super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public));
+    }
+
+    @Override
+    protected double maintain() {
+        int attempts = 0, failures = 0;
+        for (Tenant tenant : controller().tenants().asList()) {
+            try {
+                attempts++;
+                List<CloudAccountInfo> cloudAccountInfos = controller().applications().accountsOf(tenant.name()).stream()
+                        .flatMap(account -> controller().serviceRegistry()
+                                .archiveService()
+                                .getEnclaveTemplateVersion(account)
+                                .map(version -> new CloudAccountInfo(account, version))
+                                .stream())
+                        .toList();
+                controller().tenants().updateCloudAccounts(tenant.name(), cloudAccountInfos);
+            } catch (RuntimeException e) {
+                logger.log(WARNING, "Failed to verify cloud accounts for tenant " + tenant.name(), e);
+                failures++;
+            }
+        }
+        return asSuccessFactorDeviation(attempts, failures);
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
index f9c93a87c44..f6da3609fbb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
@@ -29,8 +29,8 @@ public class ContactInformationMaintainer extends ControllerMaintainer {
 
     private final ContactRetriever contactRetriever;
 
-    public ContactInformationMaintainer(Controller controller, Duration interval) {
-        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
+    public ContactInformationMaintainer(Controller controller, Duration interval, Double successFactorBaseline) {
+        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)), successFactorBaseline);
         this.contactRetriever = controller.serviceRegistry().contactRetriever();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 6fae732df0a..7afa10ab8d5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -59,7 +59,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader));
         maintainers.add(new JobRunner(controller, intervals.jobRunner));
         maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater));
-        maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer));
+        maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer, successFactorBaseline.contactInformationMaintainerBaseline));
         maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher));
         maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer()));
         maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase()));
@@ -85,6 +85,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval));
         maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer));
         maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer));
+        maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -147,6 +148,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration meteringMonitorMaintainer;
         private final Duration certificatePoolMaintainer;
         private final Duration billingReportMaintainer;
+        private final Duration cloudAccountVerifier;
 
         public Intervals(SystemName system) {
             this.system = Objects.requireNonNull(system);
@@ -184,6 +186,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.meteringMonitorMaintainer = duration(30, MINUTES);
             this.certificatePoolMaintainer = duration(15, MINUTES);
             this.billingReportMaintainer = duration(60, MINUTES);
+            this.cloudAccountVerifier = duration(10, MINUTES);
         }
 
         private Duration duration(long amount, TemporalUnit unit) {
@@ -201,12 +204,14 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Double deploymentMetricsMaintainerBaseline;
         private final Double trafficFractionUpdater;
         private final Double deploymentInfoMaintainerBaseline;
+        private final Double contactInformationMaintainerBaseline;
 
         public SuccessFactorBaseline(SystemName system) {
             Objects.requireNonNull(system);
             this.deploymentMetricsMaintainerBaseline = 0.90;
             this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65;
             this.deploymentInfoMaintainerBaseline = system.isCd() ? 0.5 : 0.95;
+            this.contactInformationMaintainerBaseline = 0.95;
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
index 5218da91c46..6c1c4daa1bb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
@@ -33,7 +33,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer {
     private Set<CloudAccount> externalAccounts() {
         Set<CloudAccount> accounts = new HashSet<>();
         for (Tenant tenant : controller().tenants().asList())
-            accounts.addAll(controller().applications().accountsOf(tenant.name()));
+            tenant.cloudAccounts().forEach(accountInfo -> accounts.add(accountInfo.cloudAccount()));
 
         return accounts;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index c90fcb81c71..805bf3d7ada 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -67,7 +67,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
     private final EndpointSecretManager endpointSecretManager;
     private final EndpointCertificateProvider endpointCertificateProvider;
     final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at());
-    final BooleanFlag assignRandomizedId;
     private final StringFlag endpointCertificateAlgo;
     private final BooleanFlag useAlternateCertProvider;
     private final IntFlag assignRandomizedIdRate;
@@ -81,7 +80,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
         this.endpointSecretManager = controller.serviceRegistry().secretManager();
         this.curator = controller().curator();
         this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
-        this.assignRandomizedId = Flags.ASSIGN_RANDOMIZED_ID.bindTo(controller.flagSource());
         this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource());
         this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource());
         this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource());
@@ -283,7 +281,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
         assignedCertificates.stream()
                 .filter(c -> c.instance().isPresent())
                 .filter(c -> c.certificate().randomizedId().isEmpty())
-                .filter(c -> assignRandomizedId.with(FetchVector.Dimension.INSTANCE_ID, c.application().instance(c.instance().get()).serializedForm()).value())
                 .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion
                 .limit(assignRandomizedIdRate.value())
                 .forEach(c -> assignRandomizedId(c.application(), c.instance().get()));