Maintain roles 5 tenants at a time

3 files changed, 56 insertions, 1 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
index d2adf3d9b73..b6a645d96d2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
@@ -158,6 +158,13 @@ public class TenantController {
         }
     }
 
+    public void updateLastTenantRolesMaintained(TenantName tenantName, Instant lastMaintained) {
+        try (Mutex lock = lock(tenantName)) {
+            var tenant = require(tenantName);
+            curator.writeTenant(LockedTenant.of(tenant, lock).with(lastMaintained).get());
+        }
+    }
+
     /** Deletes the given tenant. */
     public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) {
         try (Mutex lock = lock(tenant)) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
index 19ea3232667..bd121871c7c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
@@ -6,6 +6,8 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.Comparator;
 
 public class TenantRoleMaintainer extends ControllerMaintainer {
 
@@ -16,7 +18,10 @@ public class TenantRoleMaintainer extends ControllerMaintainer {
     @Override
     protected double maintain() {
         var roleService = controller().serviceRegistry().roleService();
-        var tenants = controller().tenants().asList();
+        var tenants = controller().tenants().asList().stream()
+                .sorted(Comparator.comparing(Tenant::tenantRolesLastMaintained))
+                .limit(5)
+                .toList();
 
         // Create separate athenz service for all tenants
         tenants.forEach(roleService::createTenantRole);
@@ -26,6 +31,12 @@ public class TenantRoleMaintainer extends ControllerMaintainer {
                 .toList();
         roleService.maintainRoles(tenantsWithRoles);
 
+        // Update last maintained timestamp
+        var updated = Instant.now(controller().clock());
+        tenants.forEach(t -> {
+            controller().tenants().updateLastTenantRolesMaintained(t.name(), updated);
+        });
+
         return 1.0;
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
index 97656583d04..2f36287df45 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
@@ -9,12 +9,16 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import org.junit.jupiter.api.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
@@ -56,6 +60,39 @@ public class TenantRoleMaintainerTest {
         assertTrue(tenantNames.containsAll(List.of(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant())));
     }
 
+    @Test
+    void maintain_batch() {
+        var tenants = List.of(
+                tester.newDeploymentContext("tenant1", "app1", "default"),
+                tester.newDeploymentContext("tenant2", "app1", "default"),
+                tester.newDeploymentContext("tenant3", "app1", "default"),
+                tester.newDeploymentContext("tenant4", "app1", "default"),
+                tester.newDeploymentContext("tenant5", "app1", "default"),
+                tester.newDeploymentContext("tenant6", "app1", "default"));
+
+        var maintainer = new TenantRoleMaintainer(tester.controller(), Duration.ofDays(1));
+        maintainer.maintain();
+
+        var maintainedTenants = tester.controller().tenants().asList().stream()
+                .filter(t -> t.tenantRolesLastMaintained() != Instant.EPOCH)
+                .toList();
+
+        var unmaintainedTenants = tester.controller().tenants().asList().stream()
+                .filter(t -> t.tenantRolesLastMaintained() == Instant.EPOCH)
+                .toList();
+
+        assertEquals(5, maintainedTenants.size());
+        assertEquals(1, unmaintainedTenants.size());
+
+        tester.clock().advance(Duration.ofDays(1));
+
+        maintainer.maintain();
+        var result = tester.controller().tenants().asList().stream()
+                .collect(Collectors.groupingBy(Tenant::tenantRolesLastMaintained));
+
+        assertFalse(result.containsKey(Instant.EPOCH));
+    }
+
     private long permanentDeployments(Instance instance) {
         return tester.controller().applications().requireInstance(instance.id()).deployments().values().stream()
                 .filter(deployment -> !deployment.zone().environment().isTest())