Merge pull request #25246 from vespa-engine/revert-25244-revert-25241-ogronnesby/tenant-roles-batch-maintained

Revert "Revert "Maintain roles for 5 tenants at a time""

14 files changed, 148 insertions, 43 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
index 79ab8654aa4..53a3f431de7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java
@@ -27,8 +27,8 @@ public class AthenzTenant extends Tenant {
      * Use {@link #create(TenantName, AthenzDomain, Property, Optional, Instant)}.
      * */
     public AthenzTenant(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId,
-                        Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo) {
-        super(name, createdAt, lastLoginInfo, contact);
+                        Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) {
+        super(name, createdAt, lastLoginInfo, contact, tenantRolesLastMaintained);
         this.domain = Objects.requireNonNull(domain, "domain must be non-null");
         this.property = Objects.requireNonNull(property, "property must be non-null");
         this.propertyId = Objects.requireNonNull(propertyId, "propertyId must be non-null");
@@ -62,7 +62,7 @@ public class AthenzTenant extends Tenant {
     /** Create a new Athenz tenant */
     public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property,
                                       Optional<PropertyId> propertyId, Instant createdAt) {
-        return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY);
+        return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY, Instant.EPOCH);
     }
 
     @Override
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
index ae0467fcc86..2275f3a9bf8 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java
@@ -31,8 +31,8 @@ public class CloudTenant extends Tenant {
     /** Public for the serialization layer — do not use! */
     public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator,
                        BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info,
-                       List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore) {
-        super(name, createdAt, lastLoginInfo, Optional.empty());
+                       List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRoleLastMaintained) {
+        super(name, createdAt, lastLoginInfo, Optional.empty(), tenantRoleLastMaintained);
         this.creator = creator;
         this.developerKeys = developerKeys;
         this.info = Objects.requireNonNull(info);
@@ -47,7 +47,7 @@ public class CloudTenant extends Tenant {
                                createdAt,
                                LastLoginInfo.EMPTY,
                                Optional.ofNullable(creator).map(SimplePrincipal::of),
-                               ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty());
+                               ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty(), Instant.EPOCH);
     }
 
     /** The user that created the tenant */
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java
index cf6d73cb8f8..b58fdf81278 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java
@@ -17,7 +17,7 @@ public class DeletedTenant extends Tenant {
     private final Instant deletedAt;
 
     public DeletedTenant(TenantName name, Instant createdAt, Instant deletedAt) {
-        super(name, createdAt, LastLoginInfo.EMPTY, Optional.empty());
+        super(name, createdAt, LastLoginInfo.EMPTY, Optional.empty(), Instant.EPOCH);
         this.deletedAt = Objects.requireNonNull(deletedAt, "deletedAt must be non-null");
     }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
index ed7e6827516..a4500991bf2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
@@ -19,12 +19,14 @@ public abstract class Tenant {
     private final Instant createdAt;
     private final LastLoginInfo lastLoginInfo;
     private final Optional<Contact> contact;
+    private final Instant tenantRolesLastMaintained;
 
-    Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact) {
+    Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact, Instant tenantRolesLastMaintained) {
         this.name = name;
         this.createdAt = createdAt;
         this.lastLoginInfo = lastLoginInfo;
         this.contact = contact;
+        this.tenantRolesLastMaintained = tenantRolesLastMaintained;
     }
 
     /** Name of this tenant */
@@ -47,6 +49,10 @@ public abstract class Tenant {
         return contact;
     }
 
+    public Instant tenantRolesLastMaintained() {
+        return tenantRolesLastMaintained;
+    }
+
     public abstract Type type();
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
index da40f63d543..f8135e27d74 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
@@ -41,11 +41,13 @@ public abstract class LockedTenant {
     final TenantName name;
     final Instant createdAt;
     final LastLoginInfo lastLoginInfo;
+    final Instant tenantRolesLastMaintained;
 
-    private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo) {
+    private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) {
         this.name = requireNonNull(name);
         this.createdAt = requireNonNull(createdAt);
         this.lastLoginInfo = requireNonNull(lastLoginInfo);
+        this.tenantRolesLastMaintained = requireNonNull(tenantRolesLastMaintained);
     }
 
     static LockedTenant of(Tenant tenant, Mutex lock) {
@@ -62,6 +64,8 @@ public abstract class LockedTenant {
 
     public abstract LockedTenant with(LastLoginInfo lastLoginInfo);
 
+    public abstract LockedTenant with(Instant tenantRolesLastMaintained);
+
     public Deleted deleted(Instant deletedAt) {
         return new Deleted(new DeletedTenant(name, createdAt, deletedAt));
     }
@@ -81,8 +85,8 @@ public abstract class LockedTenant {
         private final Optional<Contact> contact;
 
         private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId,
-                       Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo) {
-            super(name, createdAt, lastLoginInfo);
+                       Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) {
+            super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained);
             this.domain = domain;
             this.property = property;
             this.propertyId = propertyId;
@@ -90,33 +94,38 @@ public abstract class LockedTenant {
         }
 
         private Athenz(AthenzTenant tenant) {
-            this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo());
+            this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained());
         }
 
         @Override
         public AthenzTenant get() {
-            return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo);
+            return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
         public Athenz with(AthenzDomain domain) {
-            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo);
+            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
         public Athenz with(Property property) {
-            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo);
+            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
         public Athenz with(PropertyId propertyId) {
-            return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo);
+            return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
         public Athenz with(Contact contact) {
-            return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo);
+            return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
         @Override
         public LockedTenant with(LastLoginInfo lastLoginInfo) {
-            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo);
+            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
+        }
+
+        @Override
+        public LockedTenant with(Instant tenantRolesLastMaintained) {
+            return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
         }
 
     }
@@ -134,8 +143,8 @@ public abstract class LockedTenant {
 
         private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator,
                       BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info,
-                      List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore) {
-            super(name, createdAt, lastLoginInfo);
+                      List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained) {
+            super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained);
             this.developerKeys = ImmutableBiMap.copyOf(developerKeys);
             this.creator = creator;
             this.info = info;
@@ -145,12 +154,12 @@ public abstract class LockedTenant {
         }
 
         private Cloud(CloudTenant tenant) {
-            this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore());
+            this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained());
         }
 
         @Override
         public CloudTenant get() {
-            return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withDeveloperKey(PublicKey key, Principal principal) {
@@ -161,42 +170,47 @@ public abstract class LockedTenant {
             if (keys.inverse().containsKey(simplePrincipal))
                 throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal)));
             keys.put(key, simplePrincipal);
-            return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withoutDeveloperKey(PublicKey key) {
             BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys);
             keys.remove(key);
-            return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withInfo(TenantInfo newInfo) {
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         @Override
         public LockedTenant with(LastLoginInfo lastLoginInfo) {
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withSecretStore(TenantSecretStore tenantSecretStore) {
             ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores);
             secretStores.add(tenantSecretStore);
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) {
             ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores);
             secretStores.remove(tenantSecretStore);
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
 
         public Cloud withArchiveAccess(ArchiveAccess archiveAccess) {
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained);
         }
 
         public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) {
-            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore));
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained);
+        }
+
+        @Override
+        public LockedTenant with(Instant tenantRolesLastMaintained) {
+            return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
         }
     }
 
@@ -207,7 +221,7 @@ public abstract class LockedTenant {
         private final Instant deletedAt;
 
         private Deleted(DeletedTenant tenant) {
-            super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo());
+            super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH);
             this.deletedAt = tenant.deletedAt();
         }
 
@@ -220,6 +234,11 @@ public abstract class LockedTenant {
         public LockedTenant with(LastLoginInfo lastLoginInfo) {
             return this;
         }
+
+        @Override
+        public LockedTenant with(Instant tenantRolesLastMaintained) {
+            return this;
+        }
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
index d2adf3d9b73..b6a645d96d2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
@@ -158,6 +158,13 @@ public class TenantController {
         }
     }
 
+    public void updateLastTenantRolesMaintained(TenantName tenantName, Instant lastMaintained) {
+        try (Mutex lock = lock(tenantName)) {
+            var tenant = require(tenantName);
+            curator.writeTenant(LockedTenant.of(tenant, lock).with(lastMaintained).get());
+        }
+    }
+
     /** Deletes the given tenant. */
     public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) {
         try (Mutex lock = lock(tenant)) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
index 19ea3232667..bd121871c7c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
@@ -6,6 +6,8 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.Comparator;
 
 public class TenantRoleMaintainer extends ControllerMaintainer {
 
@@ -16,7 +18,10 @@ public class TenantRoleMaintainer extends ControllerMaintainer {
     @Override
     protected double maintain() {
         var roleService = controller().serviceRegistry().roleService();
-        var tenants = controller().tenants().asList();
+        var tenants = controller().tenants().asList().stream()
+                .sorted(Comparator.comparing(Tenant::tenantRolesLastMaintained))
+                .limit(5)
+                .toList();
 
         // Create separate athenz service for all tenants
         tenants.forEach(roleService::createTenantRole);
@@ -26,6 +31,12 @@ public class TenantRoleMaintainer extends ControllerMaintainer {
                 .toList();
         roleService.maintainRoles(tenantsWithRoles);
 
+        // Update last maintained timestamp
+        var updated = Instant.now(controller().clock());
+        tenants.forEach(t -> {
+            controller().tenants().updateLastTenantRolesMaintained(t.name(), updated);
+        });
+
         return 1.0;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index b6d0155b6ab..d4536da91fe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -83,6 +83,7 @@ public class TenantSerializer {
     private static final String awsArchiveAccessRoleField = "awsArchiveAccessRole";
     private static final String gcpArchiveAccessMemberField = "gcpArchiveAccessMember";
     private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore";
+    private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained";
 
     private static final String awsIdField = "awsId";
     private static final String roleField = "role";
@@ -94,6 +95,7 @@ public class TenantSerializer {
         tenantObject.setString(typeField, valueOf(tenant.type()));
         tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli());
         toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField));
+        tenantObject.setLong(tenantRolesLastMaintainedField, tenant.tenantRolesLastMaintained().toEpochMilli());
 
         switch (tenant.type()) {
             case athenz:  toSlime((AthenzTenant) tenant, tenantObject); break;
@@ -178,7 +180,8 @@ public class TenantSerializer {
         Optional<Contact> contact = contactFrom(tenantObject.field(contactField));
         Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField));
         LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField));
-        return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo);
+        Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField));
+        return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained);
     }
 
     private CloudTenant cloudTenantFrom(Inspector tenantObject) {
@@ -191,7 +194,8 @@ public class TenantSerializer {
         List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField));
         ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject);
         Optional<Instant> invalidateUserSessionsBefore = SlimeUtils.optionalInstant(tenantObject.field(invalidateUserSessionsBeforeField));
-        return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore);
+        Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField));
+        return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained);
     }
 
     private DeletedTenant deletedTenantFrom(Inspector tenantObject) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
index 97656583d04..2f36287df45 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java
@@ -9,12 +9,16 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import org.junit.jupiter.api.Test;
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
@@ -56,6 +60,39 @@ public class TenantRoleMaintainerTest {
         assertTrue(tenantNames.containsAll(List.of(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant())));
     }
 
+    @Test
+    void maintain_batch() {
+        var tenants = List.of(
+                tester.newDeploymentContext("tenant1", "app1", "default"),
+                tester.newDeploymentContext("tenant2", "app1", "default"),
+                tester.newDeploymentContext("tenant3", "app1", "default"),
+                tester.newDeploymentContext("tenant4", "app1", "default"),
+                tester.newDeploymentContext("tenant5", "app1", "default"),
+                tester.newDeploymentContext("tenant6", "app1", "default"));
+
+        var maintainer = new TenantRoleMaintainer(tester.controller(), Duration.ofDays(1));
+        maintainer.maintain();
+
+        var maintainedTenants = tester.controller().tenants().asList().stream()
+                .filter(t -> t.tenantRolesLastMaintained() != Instant.EPOCH)
+                .toList();
+
+        var unmaintainedTenants = tester.controller().tenants().asList().stream()
+                .filter(t -> t.tenantRolesLastMaintained() == Instant.EPOCH)
+                .toList();
+
+        assertEquals(5, maintainedTenants.size());
+        assertEquals(1, unmaintainedTenants.size());
+
+        tester.clock().advance(Duration.ofDays(1));
+
+        maintainer.maintain();
+        var result = tester.controller().tenants().asList().stream()
+                .collect(Collectors.groupingBy(Tenant::tenantRolesLastMaintained));
+
+        assertFalse(result.containsKey(Instant.EPOCH));
+    }
+
     private long permanentDeployments(Instance instance) {
         return tester.controller().applications().requireInstance(instance.id()).deployments().values().stream()
                 .filter(deployment -> !deployment.zone().environment().isTest())
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
index 852d4847b7e..cd1debc71fd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java
@@ -63,7 +63,8 @@ public class NotificationsDbTest {
                             email)))),
             List.of(),
             new ArchiveAccess(),
-            Optional.empty());
+            Optional.empty(),
+            Instant.EPOCH);
     private static final List<Notification> notifications = List.of(
             notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"),
             notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
index ecf42979b40..0c031a13e6f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java
@@ -44,7 +44,8 @@ public class NotifierTest {
                                     email)))),
             List.of(),
             new ArchiveAccess(),
-            Optional.empty());
+            Optional.empty(),
+            Instant.EPOCH);
 
 
     MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index 5144c5cb7b4..8da0d665282 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -89,7 +89,8 @@ public class TenantSerializerTest {
                 Optional.of(new PropertyId("1")),
                 Optional.of(contact()),
                 Instant.EPOCH,
-                lastLoginInfo(321L, 654L, 987L));
+                lastLoginInfo(321L, 654L, 987L),
+                Instant.EPOCH);
         AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.contact(), serialized.contact());
     }
@@ -105,7 +106,8 @@ public class TenantSerializerTest {
                 TenantInfo.empty(),
                 List.of(),
                 new ArchiveAccess(),
-                Optional.empty());
+                Optional.empty(),
+                Instant.EPOCH);
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.name(), serialized.name());
         assertEquals(tenant.creator(), serialized.creator());
@@ -127,7 +129,8 @@ public class TenantSerializerTest {
                         new TenantSecretStore("ss2", "124", "role2")
                 ),
                 new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role"),
-                Optional.of(Instant.ofEpochMilli(1234567)));
+                Optional.of(Instant.ofEpochMilli(1234567)),
+                Instant.EPOCH);
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(tenant.info(), serialized.info());
         assertEquals(tenant.tenantSecretStores(), serialized.tenantSecretStores());
@@ -177,7 +180,8 @@ public class TenantSerializerTest {
                 TenantInfo.empty(),
                 List.of(),
                 new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com"),
-                Optional.empty());
+                Optional.empty(),
+                Instant.EPOCH);
         CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant));
         assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role");
         assertEquals(serialized.archiveAccess().gcpMember().get(), "user:foo@example.com");
@@ -246,6 +250,19 @@ public class TenantSerializerTest {
         assertEquals(tenant.deletedAt(), serialized.deletedAt());
     }
 
+    @Test
+    void tenant_with_roles_maintained() {
+        AthenzTenant tenant = new AthenzTenant(TenantName.from("athenz-tenant"),
+                new AthenzDomain("domain1"),
+                new Property("property1"),
+                Optional.of(new PropertyId("1")),
+                Optional.of(contact()),
+                Instant.EPOCH,
+                lastLoginInfo(321L, 654L, 987L),
+                Instant.ofEpochMilli(1_000_000));
+        assertEquals(tenant, serializer.tenantFrom(serializer.toSlime(tenant)));
+    }
+
     private static Contact contact() {
         return new Contact(
                 URI.create("http://contact1.test"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index f1381ead221..d40485ff5c0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -1287,7 +1287,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // Create legacy tenant name containing underscores
         tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN,
-                new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY));
+                new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY, Instant.EPOCH));
 
         // POST (add) a Athenz tenant with dashes duplicates existing one with underscores
         tester.assertResponse(request("/application/v4/tenant/my-tenant", POST)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
index fcbecfa2e68..410bb801e85 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
@@ -78,7 +78,8 @@ public class SignatureFilterTest {
                                                      TenantInfo.empty(),
                                                      List.of(),
                                                      new ArchiveAccess(),
-                                                     Optional.empty()));
+                                                     Optional.empty(),
+                                                     Instant.EPOCH));
         tester.curator().writeApplication(new Application(appId, tester.clock().instant()));
     }
 
@@ -125,7 +126,8 @@ public class SignatureFilterTest {
                 TenantInfo.empty(),
                 List.of(),
                 new ArchiveAccess(),
-                Optional.empty()));
+                Optional.empty(),
+                Instant.EPOCH));
         verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
                 new SecurityContext(new SimplePrincipal("user"),
                         Set.of(Role.reader(id.tenant()),