Aggregate token version status, and show in app-v4 handler

3 files changed, 173 insertions, 9 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 16d862a66ef..2708fa6af58 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -112,6 +112,7 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses;
 import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService;
+import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService.State;
 import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
 import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
@@ -164,6 +165,7 @@ import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Scanner;
 import java.util.StringJoiner;
+import java.util.TreeMap;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.logging.Level;
@@ -175,6 +177,7 @@ import static com.yahoo.jdisc.Response.Status.CONFLICT;
 import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_TEST_ZIP;
 import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_ZIP;
 import static com.yahoo.yolean.Exceptions.uncheck;
+import static java.util.Comparator.comparing;
 import static java.util.Comparator.comparingInt;
 import static java.util.Map.Entry.comparingByKey;
 import static java.util.stream.Collectors.collectingAndThen;
@@ -964,27 +967,38 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
     }
 
     private HttpResponse listTokens(String tenant, HttpRequest request) {
-        var tokens = controller.dataplaneTokenService().listTokens(TenantName.from(tenant))
-                .stream().sorted(Comparator.comparing(DataplaneTokenVersions::tokenId)).toList();
         Slime slime = new Slime();
         Cursor tokensArray = slime.setObject().setArray("tokens");
-        for (DataplaneTokenVersions token : tokens) {
+        controller.dataplaneTokenService().listTokensWithState(TenantName.from(tenant)).forEach((token, states) -> {
             Cursor tokenObject = tokensArray.addObject();
             tokenObject.setString("id", token.tokenId().value());
             Cursor fingerprintsArray = tokenObject.setArray("versions");
-            var versions = token.tokenVersions().stream()
-                    .sorted(Comparator.comparing(DataplaneTokenVersions.Version::creationTime)).toList();
-            for (var tokenVersion : versions) {
+            for (var tokenVersion : token.tokenVersions()) {
                 Cursor fingerprintObject = fingerprintsArray.addObject();
                 fingerprintObject.setString("fingerprint", tokenVersion.fingerPrint().value());
                 fingerprintObject.setString("created", tokenVersion.creationTime().toString());
                 fingerprintObject.setString("author", tokenVersion.author());
                 fingerprintObject.setString("expiration", tokenVersion.expiration().map(Instant::toString).orElse("none"));
+                fingerprintObject.setString("state", valueOf(states.get(tokenVersion.fingerPrint())));
             }
-        }
+            states.forEach((print, state) -> {
+                if (state != State.DEACTIVATING) return;
+                Cursor fingerprintObject = fingerprintsArray.addObject();
+                fingerprintObject.setString("fingerprint", print.value());
+                fingerprintObject.setString("state", valueOf(state));
+            });
+        });
         return new SlimeJsonResponse(slime);
     }
 
+    private static String valueOf(DataplaneTokenService.State state) {
+        return switch (state) {
+            case DEPLOYING: yield "deploying";
+            case ACTIVE: yield "active";
+            case DEACTIVATING: yield "deactivating";
+        };
+    }
+
 
     private HttpResponse generateToken(String tenant, String tokenid, HttpRequest request) {
         var expiration = resolveExpiration(request).orElse(null);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java
index 385200a1624..40416ed3c64 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenService.java
@@ -1,15 +1,22 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.dataplanetoken;
 
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.token.Token;
 import com.yahoo.security.token.TokenCheckHash;
 import com.yahoo.security.token.TokenDomain;
 import com.yahoo.security.token.TokenGenerator;
 import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneToken;
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions;
+import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions.Version;
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint;
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -17,11 +24,21 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import java.security.Principal;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Phaser;
 import java.util.stream.Stream;
 
+import static java.util.Comparator.comparing;
+import static java.util.stream.Collectors.toMap;
+
 /**
  * Service to list, generate and delete data plane tokens
  *
@@ -34,7 +51,7 @@ public class DataplaneTokenService {
     private static final int CHECK_HASH_BYTES = 32;
     public static final Duration DEFAULT_TTL = Duration.ofDays(30);
 
-
+    private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("dataplane-token-service-"));
     private final Controller controller;
 
     public DataplaneTokenService(Controller controller) {
@@ -48,6 +65,74 @@ public class DataplaneTokenService {
         return controller.curator().readDataplaneTokens(tenantName);
     }
 
+    public enum State { DEPLOYING, ACTIVE, DEACTIVATING }
+
+    /** List all known tokens for a tenant, with the state of each token version (both current and deactivating). */
+    public Map<DataplaneTokenVersions, Map<FingerPrint, State>> listTokensWithState(TenantName tenantName) {
+        List<DataplaneTokenVersions> currentTokens = listTokens(tenantName);
+        Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokens = listActiveTokens(tenantName);
+        Map<TokenId, Map<FingerPrint, Boolean>> activeFingerprints = computeStates(activeTokens);
+        Map<DataplaneTokenVersions, Map<FingerPrint, State>> tokens = new TreeMap<>(comparing(DataplaneTokenVersions::tokenId));
+        for (DataplaneTokenVersions token : currentTokens) {
+            Map<FingerPrint, State> states = new TreeMap<>();
+            // Current tokens are active iff. they are active everywhere.
+            for (Version version : token.tokenVersions()) {
+                states.put(version.fingerPrint(),
+                           activeFingerprints.getOrDefault(token.tokenId(), Map.of())
+                                             .getOrDefault(version.fingerPrint(), false) ? State.ACTIVE : State.DEPLOYING);
+            }
+            // Active, non-current token versions are deactivating.
+            for (FingerPrint print : activeFingerprints.getOrDefault(token.tokenId(), Map.of()).keySet()) {
+                states.putIfAbsent(print, State.DEACTIVATING);
+            }
+            tokens.put(token, states);
+        }
+        // Active, non-current tokens are also deactivating.
+        activeFingerprints.forEach((id, prints) -> {
+            if (currentTokens.stream().noneMatch(token -> token.tokenId().equals(id))) {
+                Map<FingerPrint, State> states = new TreeMap<>();
+                for (FingerPrint print : prints.keySet()) states.put(print, State.DEACTIVATING);
+                tokens.put(new DataplaneTokenVersions(id, List.of()), states);
+            }
+        });
+        return tokens;
+    }
+
+    private Map<HostName, Map<TokenId, List<FingerPrint>>> listActiveTokens(TenantName tenantName) {
+        Map<HostName, Map<TokenId, List<FingerPrint>>> tokens = new ConcurrentHashMap<>();
+        Phaser phaser = new Phaser(1);
+        for (Application application : controller.applications().asList(tenantName)) {
+            for (Instance instance : application.instances().values()) {
+                for (ZoneId zone : instance.deployments().keySet()) {
+                    DeploymentId deployment = new DeploymentId(instance.id(), zone);
+                    phaser.register();
+                    executor.execute(() -> {
+                        try { tokens.putAll(controller.serviceRegistry().configServer().activeTokenFingerprints(deployment)); }
+                        finally { phaser.arrive(); }
+                    });
+                }
+            }
+        }
+        phaser.arriveAndAwaitAdvance();
+        return tokens;
+    }
+
+    /** Computes whether each print is active on all hosts where its token is present. */
+    private Map<TokenId, Map<FingerPrint, Boolean>> computeStates(Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokens) {
+        Map<TokenId, Map<FingerPrint, Boolean>> states = new HashMap<>();
+        for (Map<TokenId, List<FingerPrint>> token : activeTokens.values()) {
+            token.forEach((id, prints) -> {
+                states.merge(id,
+                             prints.stream().collect(toMap(print -> print, __ -> true)),
+                             (a, b) -> new HashMap<>() {{
+                                 a.forEach((p, s) -> put(p, s && b.getOrDefault(p, false)));
+                                 b.forEach((p, s) -> putIfAbsent(p, false));
+                             }});
+            });
+        }
+        return states;
+    }
+
     /**
      * Generates a token using tenant name as the check access context.
      * Persists the token fingerprint and check access hash, but not the token value
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenServiceTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenServiceTest.java
index acfba03a700..157e33a7e3e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenServiceTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/dataplanetoken/DataplaneTokenServiceTest.java
@@ -1,6 +1,9 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.dataplanetoken;
 
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
@@ -9,25 +12,86 @@ import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.Dataplan
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint;
 import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService.State;
 import org.junit.jupiter.api.Test;
 
 import java.security.Principal;
 import java.time.Duration;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 
+import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toSet;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class DataplaneTokenServiceTest {
+
     private final ControllerTester tester = new ControllerTester(SystemName.Public);
     private final DataplaneTokenService dataplaneTokenService = new DataplaneTokenService(tester.controller());
     private final TenantName tenantName = TenantName.from("tenant");
-    Principal principal = new SimplePrincipal("user");
+    private final Principal principal = new SimplePrincipal("user");
     private final TokenId tokenId = TokenId.of("myTokenId");
+    private final Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokens = tester.configServer().activeTokenFingerprints(null);
+
+    @Test
+    void computes_aggregate_state() {
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        DeploymentContext app = deploymentTester.newDeploymentContext(tenantName.value(), "app", "default");
+        app.submit().deploy();
+
+        TokenId[] id = new TokenId[4];
+        FingerPrint[][] print = new FingerPrint[4][3];
+        for (int i = 0; i < id.length; i++) {
+            id[i] = TokenId.of("id" + i);
+            for (int j = 0; j < 3; j++) {
+                print[i][j] = dataplaneTokenService.generateToken(tenantName, id[i], null, principal).fingerPrint();
+            }
+        }
+        dataplaneTokenService.deleteToken(tenantName, id[2], print[2][0]);
+        dataplaneTokenService.deleteToken(tenantName, id[2], print[2][1]);
+        for (int j = 0; j < 3; j++) {
+            dataplaneTokenService.deleteToken(tenantName, id[3], print[3][j]);
+        }
+        // "host1" has all versions of all current tokens, except the first versions of tokens 1 and 2.
+        activeTokens.put(HostName.of("host1"),
+                         Map.of(id[0], List.of(print[0]),
+                                id[1], List.of(print[1][1], print[1][2]),
+                                id[2], List.of(print[2][1], print[2][2])));
+        // "host2" has all versions of all current tokens, except the last version of token 1.
+        activeTokens.put(HostName.of("host2"),
+                         Map.of(id[0], List.of(print[0]),
+                                id[1], List.of(print[1][0], print[1][1]),
+                                id[2], List.of(print[2])));
+        // "host3" has no current tokens at all, but has the last version of token 3
+        activeTokens.put(HostName.of("host3"),
+                         Map.of(id[3], List.of(print[3][2])));
+
+        // All fingerprints of token 0 are active on all hosts where token 0 is found, so they are all active.
+        // The first and last fingerprints of token 1 are missing from one host each, so these are activating.
+        // The first fingerprints of token 2 are no longer current, but the second is found on a host; both deactivating.
+        // The whole of token 3 is forgotten, but the last fingerprint is found on a host; deactivating.
+        assertEquals(new TreeMap<>(Map.of(id[0], new TreeMap<>(Map.of(print[0][0], State.ACTIVE,
+                                                                      print[0][1], State.ACTIVE,
+                                                                      print[0][2], State.ACTIVE)),
+                                          id[1], new TreeMap<>(Map.of(print[1][0], State.DEPLOYING,
+                                                                      print[1][1], State.ACTIVE,
+                                                                      print[1][2], State.DEPLOYING)),
+                                          id[2], new TreeMap<>(Map.of(print[2][0], State.DEACTIVATING,
+                                                                      print[2][1], State.DEACTIVATING,
+                                                                      print[2][2], State.ACTIVE)),
+                                          id[3], new TreeMap<>(Map.of(print[3][2], State.DEACTIVATING)))),
+                     new TreeMap<>(dataplaneTokenService.listTokensWithState(tenantName).entrySet().stream()
+                                                        .collect(toMap(tokens -> tokens.getKey().tokenId(),
+                                                                       tokens -> new TreeMap<>(tokens.getValue())))));
+    }
 
     @Test
     void generates_and_persists_token() {
@@ -82,4 +146,5 @@ public class DataplaneTokenServiceTest {
         IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> dataplaneTokenService.deleteToken(tenantName, tokenId, dataplaneToken.fingerPrint()));
         assertEquals("Token does not exist: " + tokenId, exception.getMessage());
     }
+
 }