More new JobList and co

4 files changed, 96 insertions, 99 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
index 779beab60d1..93dc5db206a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
@@ -27,6 +27,11 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus
         return ApplicationList.from(mapToList(DeploymentStatus::application));
     }
 
+    public DeploymentStatusList withProductionDeployment() {
+        return matching(status -> status.application().productionDeployments().values().stream()
+                                        .anyMatch(deployments -> ! deployments.isEmpty()));
+    }
+
     public DeploymentStatusList failing() {
         return matching(DeploymentStatus::hasFailures);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
index f771e9314eb..92bf1119020 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 
@@ -126,6 +127,11 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
             return present().mapToList(which.andThen(Optional::get).andThen(mapper));
         }
 
+        /** Returns the runs of the given kind. */
+        public List<Run> asList() {
+            return mapToList(Function.identity());
+        }
+
         /** Returns the subset of jobs where the run of the given type occurred before the given instant */
         public JobList endedBefore(Instant threshold) {
             return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 46f4b2816c1..e33f6f0963b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -11,24 +11,26 @@ import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.restapi.Path;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.application.JobList;
-import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.restapi.ErrorResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.restapi.Uri;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.deployment.JobList;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Instant;
 import java.util.Comparator;
+import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
 import static java.util.Comparator.comparing;
 
 /**
@@ -98,53 +100,55 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
                 configServerObject.setString("hostname", hostname.value());
             }
 
+            Map<ApplicationId, JobList> jobs = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()))
+                                                         .asList().stream()
+                                                         .flatMap(status -> status.instanceJobs().entrySet().stream())
+                                                         .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
             Cursor failingArray = versionObject.setArray("failingApplications");
             for (ApplicationId id : version.statistics().failing()) {
-                controller.applications().getInstance(id).ifPresent(application -> {
-                    firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> {
+                if (jobs.containsKey(id))
+                    firstFailingOn(version.versionNumber(), jobs.get(id)).ifPresent(firstFailing -> {
                         Cursor applicationObject = failingArray.addObject();
-                        toSlime(applicationObject, application, request);
-                        applicationObject.setString("failing", firstFailing.type().jobName());
+                        toSlime(applicationObject, id, request);
+                        applicationObject.setString("failing", firstFailing.id().type().jobName());
                     });
-                });
             }
 
             Cursor productionArray = versionObject.setArray("productionApplications");
             for (ApplicationId id : version.statistics().production()) {
-                controller.applications().getInstance(id).ifPresent(application -> {
-                    int successes = productionSuccessesFor(version.versionNumber(), application);
-                    if (successes == 0) return; // Just upgraded to a newer version.
+                if (jobs.containsKey(id)) {
+                    int successes = productionSuccessesFor(version.versionNumber(), jobs.get(id));
+                    if (successes == 0) continue; // Just upgraded to a newer version.
                     Cursor applicationObject = productionArray.addObject();
-                    toSlime(applicationObject, application, request);
-                    applicationObject.setLong("productionJobs", productionJobsFor(application));
-                    applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application));
-                });
+                    toSlime(applicationObject, id, request);
+                    applicationObject.setLong("productionJobs", jobs.get(id).production().size());
+                    applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), jobs.get(id)));
+                }
             }
 
             Cursor runningArray = versionObject.setArray("deployingApplications");
             for (ApplicationId id : version.statistics().deploying()) {
-                controller.applications().getInstance(id).ifPresent(application -> {
-                    lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> {
+                if (jobs.containsKey(id))
+                    lastDeployingTo(version.versionNumber(), jobs.get(id)).ifPresent(lastDeploying -> {
                         Cursor applicationObject = runningArray.addObject();
-                        toSlime(applicationObject, application, request);
-                        applicationObject.setString("running", lastDeploying.type().jobName());
+                        toSlime(applicationObject, id, request);
+                        applicationObject.setString("running", lastDeploying.id().type().jobName());
                     });
-                });
             }
         }
         return new SlimeJsonResponse(slime);
     }
 
-    private void toSlime(Cursor object, Instance instance, HttpRequest request) {
-        object.setString("tenant", instance.id().tenant().value());
-        object.setString("application", instance.id().application().value());
-        object.setString("instance", instance.id().instance().value());
+    private void toSlime(Cursor object, ApplicationId id, HttpRequest request) {
+        object.setString("tenant", id.tenant().value());
+        object.setString("application", id.application().value());
+        object.setString("instance", id.instance().value());
         object.setString("url", new Uri(request.getUri()).withPath("/application/v4/tenant/" +
-                                                                   instance.id().tenant().value() +
+                                                                   id.tenant().value() +
                                                                    "/application/" +
-                                                                   instance.id().application().value()).toString());
-        object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(instance.id()))
-                                                             .deploymentSpec().requireInstance(instance.name()).upgradePolicy()));
+                                                                   id.application().value()).toString());
+        object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(id))
+                                                             .deploymentSpec().requireInstance(id.instance()).upgradePolicy()));
     }
 
     private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) {
@@ -157,40 +161,30 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
     // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus
 
     /** The first upgrade job to fail on this version, for this application */
-    private Optional<JobStatus> firstFailingOn(Version version, Instance instance) {
-        return JobList.from(instance)
-                      .failing()
-                      .not().failingApplicationChange()
-                      .not().failingBecause(outOfCapacity)
-                      .lastCompleted().on(version)
-                      .asList().stream()
-                      .min(Comparator.<JobStatus, Instant>comparing(job -> job.lastCompleted().get().at())
-                                   .thenComparing(job -> job.type()));
-    }
-
-    /** The number of production jobs for this application */
-    private int productionJobsFor(Instance instance) {
-        return JobList.from(instance)
-                      .production()
-                      .size();
+    private Optional<Run> firstFailingOn(Version version, JobList jobs) {
+        return jobs.failing()
+                   .not().failingApplicationChange()
+                   .not().withStatus(RunStatus.outOfCapacity)
+                   .lastCompleted().on(version)
+                   .lastCompleted().asList().stream()
+                   .min(Comparator.<Run, Instant>comparing(run -> run.start())
+                                .thenComparing(run -> run.id().type()));
     }
 
     /** The number of production jobs with last success on the given version, for this application */
-    private int productionSuccessesFor(Version version, Instance instance) {
-        return JobList.from(instance)
-                      .production()
-                      .lastSuccess().on(version)
-                      .size();
+    private int productionSuccessesFor(Version version, JobList jobs) {
+        return jobs.production()
+                   .lastSuccess().on(version)
+                   .size();
     }
 
     /** The last triggered upgrade to this version, for this application */
-    private Optional<JobStatus> lastDeployingTo(Version version, Instance instance) {
-        return JobList.from(instance)
-                      .upgrading()
-                      .lastTriggered().on(version)
-                      .asList().stream()
-                      .max(Comparator.<JobStatus, Instant>comparing(job -> job.lastCompleted().get().at())
-                                   .thenComparing(job -> job.type()));
+    private Optional<Run> lastDeployingTo(Version version, JobList jobs) {
+        return jobs.upgrading()
+                   .lastTriggered().on(version)
+                   .lastTriggered().asList().stream()
+                   .max(Comparator.<Run, Instant>comparing(run -> run.start())
+                                .thenComparing(run -> run.id().type()));
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 97506d3f6ea..ee6cd35b90a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -7,13 +7,16 @@ import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.JobList;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
+import com.yahoo.vespa.hosted.controller.deployment.JobList;
+import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader;
 
 import java.util.ArrayList;
@@ -29,8 +32,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
-
 /**
  * Information about the current platform versions in use.
  * The versions in use are the set of all versions running in current applications, versions
@@ -118,8 +119,9 @@ public class VersionStatus {
             systemVersion = newSystemVersion;
         }
 
-        Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions.keySet(),
-                                                                                            controller.applications().asList());
+
+        var deploymentStatistics = computeDeploymentStatistics(infrastructureVersions.keySet(),
+                                                               controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())));
         List<VespaVersion> versions = new ArrayList<>();
         List<Version> releasedVersions = controller.mavenRepository().metadata().versions();
 
@@ -187,50 +189,40 @@ public class VersionStatus {
     }
 
     private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions,
-                                                                                List<Application> instances) {
+                                                                                DeploymentStatusList statuses) {
         Map<Version, DeploymentStatistics> versionMap = new HashMap<>();
 
         for (Version infrastructureVersion : infrastructureVersions) {
             versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion));
         }
 
-        for (Application application : ApplicationList.from(instances).withProductionDeployment().asList())
-            for (Instance instance : application.instances().values()) {
-                // Note that each version deployed on this application in production exists
-                // (ignore non-production versions)
-                for (Deployment deployment : instance.productionDeployments().values()) {
+        for (DeploymentStatus status : statuses.withProductionDeployment().asList()) {
+            for (Instance instance : status.application().instances().values())
+                for (Deployment deployment : instance.productionDeployments().values())
                     versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty);
-                }
 
-                // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs
-
-                // Failing versions
-                JobList.from(instance)
-                       .failing()
-                       .not().failingApplicationChange()
-                       .not().failingBecause(outOfCapacity)
-                       .mapToList(job -> job.lastCompleted().get().platform())
-                       .forEach(version -> versionMap
-                               .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                       .withFailing(instance.id())));
-
-                // Succeeding versions
-                JobList.from(instance)
-                       .lastSuccess().present()
-                       .production()
-                       .mapToList(job -> job.lastSuccess().get().platform())
-                       .forEach(version -> versionMap
-                               .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                       .withProduction(instance.id())));
-
-                // Deploying versions
-                JobList.from(instance)
-                       .upgrading()
-                       .mapToList(job -> job.lastTriggered().get().platform())
-                       .forEach(version -> versionMap
-                               .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                       .withDeploying(instance.id())));
-            }
+            status.instanceJobs().forEach((id, jobs) -> {
+                jobs.failing()
+                    .not().failingApplicationChange()
+                    .not().withStatus(RunStatus.outOfCapacity)
+                    .lastCompleted().mapToList(run -> run.versions().targetPlatform())
+                    .forEach(version -> versionMap.put(version,
+                                                       versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
+                                                                 .withFailing(id)));
+
+                jobs.production()
+                    .lastSuccess().mapToList(run -> run.versions().targetPlatform())
+                    .forEach(version -> versionMap.put(version,
+                                                       versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
+                                                                 .withProduction(id)));
+
+                jobs.upgrading()
+                    .lastTriggered().mapToList(run -> run.versions().targetPlatform())
+                    .forEach(version -> versionMap.put(version,
+                                                       versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
+                                                                 .withDeploying(id)));
+            });
+        }
         return versionMap.values();
     }