Merge pull request #18533 from vespa-engine/freva/app-v4

Application v4 improvements

5 files changed, 177 insertions, 107 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index d022588c757..054c5f21b0c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -245,7 +245,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
@@ -546,28 +546,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         if (controller.tenants().get(tenantName).isEmpty())
             return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
 
+        List<Application> applications = applicationName.isEmpty() ?
+                controller.applications().asList(tenant) :
+                controller.applications().getApplication(TenantAndApplicationId.from(tenantName, applicationName.get()))
+                    .map(List::of)
+                    .orElseThrow(() -> new NotExistsException("Application '" + applicationName.get() + "' does not exist"));
+
         Slime slime = new Slime();
         Cursor applicationArray = slime.setArray();
-        for (Application application : controller.applications().asList(tenant)) {
-            if (applicationName.map(application.id().application().value()::equals).orElse(true)) {
-                Cursor applicationObject = applicationArray.addObject();
-                applicationObject.setString("tenant", application.id().tenant().value());
-                applicationObject.setString("application", application.id().application().value());
-                applicationObject.setString("url", withPath("/application/v4" +
-                                                            "/tenant/" + application.id().tenant().value() +
-                                                            "/application/" + application.id().application().value(),
-                                                            request.getUri()).toString());
-                Cursor instanceArray = applicationObject.setArray("instances");
-                for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet()
-                                                                                  : application.instances().keySet()) {
-                    Cursor instanceObject = instanceArray.addObject();
-                    instanceObject.setString("instance", instance.value());
-                    instanceObject.setString("url", withPath("/application/v4" +
-                                                             "/tenant/" + application.id().tenant().value() +
-                                                             "/application/" + application.id().application().value() +
-                                                             "/instance/" + instance.value(),
-                                                             request.getUri()).toString());
-                }
+        for (Application application : applications) {
+            Cursor applicationObject = applicationArray.addObject();
+            applicationObject.setString("tenant", application.id().tenant().value());
+            applicationObject.setString("application", application.id().application().value());
+            applicationObject.setString("url", withPath("/application/v4" +
+                                                        "/tenant/" + application.id().tenant().value() +
+                                                        "/application/" + application.id().application().value(),
+                                                        request.getUri()).toString());
+            Cursor instanceArray = applicationObject.setArray("instances");
+            for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet()
+                                                                              : application.instances().keySet()) {
+                Cursor instanceObject = instanceArray.addObject();
+                instanceObject.setString("instance", instance.value());
+                instanceObject.setString("url", withPath("/application/v4" +
+                                                         "/tenant/" + application.id().tenant().value() +
+                                                         "/application/" + application.id().application().value() +
+                                                         "/instance/" + instance.value(),
+                                                         request.getUri()).toString());
             }
         }
         return new SlimeJsonResponse(slime);
@@ -1387,7 +1391,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId())
                 .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli()));
 
-        DeploymentStatus status = controller.jobController().deploymentStatus(application);
         application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i)));
         sourceRevisionToSlime(deployment.applicationVersion().source(), response);
 
@@ -1396,18 +1399,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
             if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod)
                 toSlime(instance.rotations(), instance.rotationStatus(), deployment, response);
 
-            JobType.from(controller.system(), deployment.zone())
-                   .map(type -> new JobId(instance.id(), type))
-                   .map(status.jobSteps()::get)
-                   .ifPresent(stepStatus -> {
-                       JobControllerApiHandlerHelper.applicationVersionToSlime(
-                               response.setObject("applicationVersion"), deployment.applicationVersion());
-                       if (!status.jobsToRun().containsKey(stepStatus.job().get()))
-                           response.setString("status", "complete");
-                       else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true))
-                           response.setString("status", "pending");
-                       else response.setString("status", "running");
-                   });
+            if (!deployment.zone().environment().isManuallyDeployed()) {
+                DeploymentStatus status = controller.jobController().deploymentStatus(application);
+                JobType.from(controller.system(), deployment.zone())
+                        .map(type -> new JobId(instance.id(), type))
+                        .map(status.jobSteps()::get)
+                        .ifPresent(stepStatus -> {
+                            JobControllerApiHandlerHelper.applicationVersionToSlime(
+                                    response.setObject("applicationVersion"), deployment.applicationVersion());
+                            if (!status.jobsToRun().containsKey(stepStatus.job().get()))
+                                response.setString("status", "complete");
+                            else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true))
+                                response.setString("status", "pending");
+                            else response.setString("status", "running");
+                        });
+            }
         }
 
         response.setDouble("quota", deployment.quota().rate());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index d246b20f2fe..eb9b0e03f3d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -40,10 +40,12 @@ import java.time.Instant;
 import java.time.format.TextStyle;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary;
@@ -84,7 +86,7 @@ class JobControllerApiHandlerHelper {
 
                   Cursor jobObject = jobsArray.addObject();
                   jobObject.setString("jobName", job.type().jobName());
-                  toSlime(jobObject.setArray("runs"), runs, baseUriForJobs);
+                  toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs);
               });
 
         return new SlimeJsonResponse(slime);
@@ -117,11 +119,19 @@ class JobControllerApiHandlerHelper {
     }
 
     /** Returns a response with the runs for the given job type. */
-    static HttpResponse runResponse(Map<RunId, Run> runs, URI baseUriForJobType) {
+    static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) {
         Slime slime = new Slime();
         Cursor cursor = slime.setObject();
 
-        runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType));
+        // TODO (freva): Remove after console migrated to use new format
+        if (limitStr.isEmpty())
+            runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType));
+        else {
+            int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE);
+            toSlime(cursor.setArray("runs"), runs.values().stream()
+                    .sorted(Comparator.comparing((Run run) -> run.id().number()).reversed())
+                    .collect(Collectors.toUnmodifiableList()), limit, baseUriForJobType);
+        }
 
         return new SlimeJsonResponse(slime);
     }
@@ -377,7 +387,7 @@ class JobControllerApiHandlerHelper {
                     toSlime(runObject.setObject("versions"), versions);
                 }
 
-                toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob);
+                toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob);
             });
         }
 
@@ -428,8 +438,8 @@ class JobControllerApiHandlerHelper {
         return Optional.of(versions.get(i));
     }
 
-    private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) {
-        runs.stream().limit(10).forEach(run -> {
+    private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) {
+        runs.stream().limit(limit).forEach(run -> {
             Cursor runObject = runsArray.addObject();
             runObject.setLong("id", run.id().number());
             runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 3fa593fcf64..a01097cfcb6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -225,6 +225,10 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // GET tenant applications
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID),
                               new File("application-list.json"));
+        // GET tenant application instances for application that does not exist
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/fake-app/instance/", GET).userIdentity(USER_ID),
+                "{\"error-code\":\"NOT_FOUND\",\"message\":\"Application 'fake-app' does not exist\"}", 404);
+
         // GET tenant applications (instances of "application1" only)
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID),
                               new File("application-list.json"));
@@ -801,7 +805,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // GET system test job overview.
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET)
-                                      .userIdentity(USER_ID),
+                                      .userIdentity(USER_ID).properties(Map.of("limit", "100")),
                               new File("system-test-job.json"));
 
         // GET system test run 1 details.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 73fda6ffcc0..72295497c03 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -131,14 +131,14 @@ public class JobControllerApiHandlerHelperTest {
         // Only us-east-3 is verified, on revision1.
         // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2,
         // one success from revision2 to revision3 and two failures from revision1 to revision3.
-        assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json");
+        assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json");
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json");
 
         var userApp = tester.newDeploymentContext(app.instanceId().tenant().value(), app.instanceId().application().value(), "user");
         userApp.runJob(devAwsUsEast2a, applicationPackage);
-        assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
+        assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json");
         assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json");
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
index 77998d91f20..c0988e8c301 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
@@ -1,69 +1,119 @@
 {
-  "1": {
-    "id": 1,
-    "status": "success",
-    "start": "(ignore)",
-    "end": "(ignore)",
-    "wantedPlatform": "6.1",
-    "wantedApplication": {
-      "hash": "1.0.1-commit1",
-      "build": 1,
-      "source": {
-        "gitRepository": "repository1",
-        "gitBranch": "master",
-        "gitCommit": "commit1"
+  "runs": [
+    {
+      "id": 2,
+      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2",
+      "start": "(ignore)",
+      "status": "running",
+      "versions": {
+        "targetPlatform": "6.1.0",
+        "targetApplication": {
+          "build": 4,
+          "compileVersion": "6.1.0",
+          "sourceUrl": "repository1/tree/commit1",
+          "commit": "commit1"
+        }
       },
-      "sourceUrl": "repository1/tree/commit1",
-      "commit": "commit1"
+      "steps": [
+        {
+          "name": "deployTester",
+          "status": "unfinished"
+        },
+        {
+          "name": "installTester",
+          "status": "unfinished"
+        },
+        {
+          "name": "deployReal",
+          "status": "unfinished"
+        },
+        {
+          "name": "installReal",
+          "status": "unfinished"
+        },
+        {
+          "name": "startTests",
+          "status": "unfinished"
+        },
+        {
+          "name": "endTests",
+          "status": "unfinished"
+        },
+        {
+          "name": "copyVespaLogs",
+          "status": "unfinished"
+        },
+        {
+          "name": "deactivateReal",
+          "status": "unfinished"
+        },
+        {
+          "name": "deactivateTester",
+          "status": "unfinished"
+        },
+        {
+          "name": "report",
+          "status": "unfinished"
+        }
+      ]
     },
-    "steps": {
-      "deployTester": "succeeded",
-      "installTester": "succeeded",
-      "deployReal": "succeeded",
-      "installReal": "succeeded",
-      "startTests": "succeeded",
-      "endTests": "succeeded",
-      "copyVespaLogs": "succeeded",
-      "deactivateReal": "succeeded",
-      "deactivateTester": "succeeded",
-      "report": "succeeded"
-    },
-    "tasks": {
-      "deploy": "succeeded",
-      "install": "succeeded",
-      "test": "succeeded"
-    },
-    "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1"
-  },
-  "2": {
-    "id": 2,
-    "status": "running",
-    "start": "(ignore)",
-    "wantedPlatform": "6.1",
-    "wantedApplication": {
-      "hash": "1.0.4-commit1",
-      "build": 4,
-      "source": {
-        "gitRepository": "repository1",
-        "gitBranch": "master",
-        "gitCommit": "commit1"
+    {
+      "id": 1,
+      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1",
+      "start": "(ignore)",
+      "end": "(ignore)",
+      "status": "success",
+      "versions": {
+        "targetPlatform": "6.1.0",
+        "targetApplication": {
+          "build": 1,
+          "compileVersion": "6.1.0",
+          "sourceUrl": "repository1/tree/commit1",
+          "commit": "commit1"
+        }
       },
-      "sourceUrl": "repository1/tree/commit1",
-      "commit": "commit1"
-    },
-    "steps": {
-      "deployTester": "unfinished",
-      "installTester": "unfinished",
-      "deployReal": "unfinished",
-      "installReal": "unfinished",
-      "startTests": "unfinished",
-      "endTests": "unfinished",
-      "copyVespaLogs": "unfinished",
-      "deactivateReal": "unfinished",
-      "deactivateTester": "unfinished",
-      "report": "unfinished"
-    },
-    "tasks": {"deploy": "running"},
-    "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2"
-  }
+      "steps": [
+        {
+          "name": "deployTester",
+          "status": "succeeded"
+        },
+        {
+          "name": "installTester",
+          "status": "succeeded"
+        },
+        {
+          "name": "deployReal",
+          "status": "succeeded"
+        },
+        {
+          "name": "installReal",
+          "status": "succeeded"
+        },
+        {
+          "name": "startTests",
+          "status": "succeeded"
+        },
+        {
+          "name": "endTests",
+          "status": "succeeded"
+        },
+        {
+          "name": "copyVespaLogs",
+          "status": "succeeded"
+        },
+        {
+          "name": "deactivateReal",
+          "status": "succeeded"
+        },
+        {
+          "name": "deactivateTester",
+          "status": "succeeded"
+        },
+        {
+          "name": "report",
+          "status": "succeeded"
+        }
+      ]
+    }
+  ]
 }