Merge pull request #16989 from vespa-engine/jonmv/faster-tenant-listing

Jonmv/faster tenant listing

3 files changed, 29 insertions, 63 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index ccb9dccdb3b..159e0bb1f0f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -185,16 +185,14 @@ public class Application {
     /** Returns the total quota usage for this application, excluding temporary deployments */
     public QuotaUsage quotaUsage() {
         return instances().values().stream()
-                .filter(instance -> !instance.id().instance().isTester()) // Exclude temporary deployments
-                .map(Instance::quotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none);
+                          .map(Instance::quotaUsage).reduce(QuotaUsage::add).orElse(QuotaUsage.none);
     }
 
     /** Returns the total quota usage for this application, excluding one specific deployment (and temporary deployments) */
     public QuotaUsage quotaUsage(ApplicationId application, ZoneId zone) {
         return instances().values().stream()
-                .filter(instance -> !instance.id().instance().isTester()) // Exclude temporary deployments
-                .map(instance -> instance.quotaUsageExcluding(application, zone))
-                .reduce(QuotaUsage::add).orElse(QuotaUsage.none);
+                          .map(instance -> instance.quotaUsageExcluding(application, zone))
+                          .reduce(QuotaUsage::add).orElse(QuotaUsage.none);
     }
 
     /** Returns the set of deploy keys for this application. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index f252e532ea8..96fcb1d7f56 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -36,6 +36,7 @@ import com.yahoo.slime.Inspector;
 import com.yahoo.slime.JsonParseException;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
@@ -51,7 +52,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
@@ -140,6 +140,7 @@ import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
 import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
 import static java.util.Map.Entry.comparingByKey;
 import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toUnmodifiableList;
 
 /**
@@ -345,8 +346,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse recursiveRoot(HttpRequest request) {
         Slime slime = new Slime();
         Cursor tenantArray = slime.setArray();
+        List<Application> applications = controller.applications().asList();
         for (Tenant tenant : controller.tenants().asList())
-            toSlime(tenantArray.addObject(), tenant, request);
+            toSlime(tenantArray.addObject(),
+                    tenant,
+                    applications.stream().filter(app -> app.id().tenant().equals(tenant.name())).collect(toList()),
+                    request);
         return new SlimeJsonResponse(slime);
     }
 
@@ -372,7 +377,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse tenant(Tenant tenant, HttpRequest request) {
         Slime slime = new Slime();
-        toSlime(slime.setObject(), tenant, request);
+        toSlime(slime.setObject(), tenant, controller.applications().asList(tenant.name()), request);
         return new SlimeJsonResponse(slime);
     }
 
@@ -482,7 +487,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Slime slime = new Slime();
         Cursor applicationArray = slime.setArray();
-        for (com.yahoo.vespa.hosted.controller.Application application : controller.applications().asList(tenant)) {
+        for (Application application : controller.applications().asList(tenant)) {
             if (applicationName.map(application.id().application().value()::equals).orElse(true)) {
                 Cursor applicationObject = applicationArray.addObject();
                 applicationObject.setString("tenant", application.id().tenant().value());
@@ -753,7 +758,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new MessageResponse(messageBuilder.toString());
     }
 
-    private com.yahoo.vespa.hosted.controller.Application getApplication(String tenantName, String applicationName) {
+    private Application getApplication(String tenantName, String applicationName) {
         TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenantName, applicationName);
         return controller.applications().getApplication(applicationId)
                          .orElseThrow(() -> new NotExistsException(applicationId + " not found"));
@@ -791,7 +796,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse clusters(String tenantName, String applicationName, String instanceName, String environment, String region) {
         ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
         ZoneId zone = requireZone(environment, region);
-        Application application = controller.serviceRegistry().configServer().nodeRepository().getApplication(zone, id);
+        com.yahoo.vespa.hosted.controller.api.integration.configserver.Application application = controller.serviceRegistry().configServer().nodeRepository().getApplication(zone, id);
 
         Slime slime = new Slime();
         Cursor clustersObject = slime.setObject().setObject("clusters");
@@ -932,7 +937,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new MessageResponse(type.jobName() + " for " + id + " resumed");
     }
 
-    private void toSlime(Cursor object, com.yahoo.vespa.hosted.controller.Application application, HttpRequest request) {
+    private void toSlime(Cursor object, Application application, HttpRequest request) {
         object.setString("tenant", application.id().tenant().value());
         object.setString("application", application.id().application().value());
         object.setString("deployments", withPath("/application/v4" +
@@ -1070,7 +1075,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private void toSlime(Cursor object, Instance instance, DeploymentStatus status, HttpRequest request) {
-        com.yahoo.vespa.hosted.controller.Application application = status.application();
+        Application application = status.application();
         object.setString("tenant", instance.id().tenant().value());
         object.setString("application", instance.id().application().value());
         object.setString("instance", instance.id().instance().value());
@@ -1572,7 +1577,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         Inspector requestObject = toSlime(request.getData()).get();
         TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
         Credentials credentials = accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest());
-        com.yahoo.vespa.hosted.controller.Application application = controller.applications().createApplication(id, credentials);
+        Application application = controller.applications().createApplication(id, credentials);
         Slime slime = new Slime();
         toSlime(id, slime.setObject(), request);
         return new SlimeJsonResponse(slime);
@@ -1837,7 +1842,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Optional<ApplicationPackage> applicationPackage = Optional.ofNullable(dataParts.get("applicationZip"))
                                                                   .map(ApplicationPackage::new);
-        Optional<com.yahoo.vespa.hosted.controller.Application> application = controller.applications().getApplication(TenantAndApplicationId.from(applicationId));
+        Optional<Application> application = controller.applications().getApplication(TenantAndApplicationId.from(applicationId));
 
         Inspector sourceRevision = deployOptions.field("sourceRevision");
         Inspector buildNumber = deployOptions.field("buildNumber");
@@ -1977,10 +1982,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                          .orElseThrow(() -> new NotExistsException(new TenantId(tenantName)));
     }
 
-    private void toSlime(Cursor object, Tenant tenant, HttpRequest request) {
+    private void toSlime(Cursor object, Tenant tenant, List<Application> applications, HttpRequest request) {
         object.setString("tenant", tenant.name().value());
         object.setString("type", tenantType(tenant));
-        List<com.yahoo.vespa.hosted.controller.Application> applications = controller.applications().asList(tenant.name());
         switch (tenant.type()) {
             case athenz:
                 AthenzTenant athenzTenant = (AthenzTenant) tenant;
@@ -2013,7 +2017,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
                 var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name());
                 var usedQuota = applications.stream()
-                        .map(com.yahoo.vespa.hosted.controller.Application::quotaUsage)
+                        .map(Application::quotaUsage)
                         .reduce(QuotaUsage.none, QuotaUsage::add);
 
                 toSlime(tenantQuota, usedQuota, object.setObject("quota"));
@@ -2024,16 +2028,19 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
         // TODO jonmv: This should list applications, not instances.
         Cursor applicationArray = object.setArray("applications");
-        for (com.yahoo.vespa.hosted.controller.Application application : applications) {
-            DeploymentStatus status = controller.jobController().deploymentStatus(application);
+        for (Application application : applications) {
+            DeploymentStatus status = null;
             for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values()
                                                                           : application.instances().values())
-                if (recurseOverApplications(request))
+                if (recurseOverApplications(request)) {
+                    if (status == null) status = controller.jobController().deploymentStatus(application);
                     toSlime(applicationArray.addObject(), instance, status, request);
-                else
+                }
+                else {
                     toSlime(instance.id(), applicationArray.addObject(), request);
+                }
         }
-        tenantMetaDataToSlime(tenant, object.setObject("metaData"));
+        tenantMetaDataToSlime(tenant, applications, object.setObject("metaData"));
     }
 
     private void toSlime(Quota quota, QuotaUsage usage, Cursor object) {
@@ -2101,8 +2108,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         object.setString("url", withPath("/application/v4/tenant/" + tenant.name().value(), requestURI).toString());
     }
 
-    private void tenantMetaDataToSlime(Tenant tenant, Cursor object) {
-        List<com.yahoo.vespa.hosted.controller.Application> applications = controller.applications().asList(tenant.name());
+    private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) {
         Optional<Instant> lastDev = applications.stream()
                 .flatMap(application -> application.instances().values().stream())
                 .flatMap(instance -> controller.jobController().jobs(instance.id()).stream()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
index d0de33f30e3..28bbeb63caa 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java
@@ -82,42 +82,4 @@ public class DeploymentQuotaCalculatorTest {
         assertEquals(tenantQuota, calculated);
     }
 
-    @Test
-    public void temporary_deployments_are_excluded() {
-        var tenantQuota = Quota.unlimited().withBudget(42);
-
-        var instanceInTestEnv = new Instance(ApplicationId.from("default", "default", "foo")).withNewDeployment(
-                ZoneId.from("test", "apac1"),
-                ApplicationVersion.unknown,
-                Version.emptyVersion,
-                Instant.EPOCH,
-                Map.of(),
-                QuotaUsage.create(1));
-
-        var testerInstance = new Instance(ApplicationId.from("default", "default", "bar-t")).withNewDeployment(
-                ZoneId.from("prod", "apac1"),
-                ApplicationVersion.unknown,
-                Version.emptyVersion,
-                Instant.EPOCH,
-                Map.of(),
-                QuotaUsage.create(1));
-
-        var app = new Application(
-                TenantAndApplicationId.from(ApplicationId.defaultId()),
-                Instant.EPOCH,
-                DeploymentSpec.empty,
-                ValidationOverrides.empty,
-                Optional.empty(),
-                Optional.empty(),
-                Optional.empty(),
-                OptionalInt.empty(),
-                new ApplicationMetrics(100, 100),
-                Set.of(),
-                OptionalLong.empty(),
-                Optional.empty(),
-                List.of(instanceInTestEnv, testerInstance));
-
-        Quota calculated = DeploymentQuotaCalculator.calculate(tenantQuota, List.of(app), ApplicationId.defaultId(), ZoneId.from("dev", "apac1"), DeploymentSpec.empty);
-        assertEquals(tenantQuota, calculated);
-    }
 }