Merge pull request #4017 from vespa-engine/jvenstad/deployment-api-expansion-continued

Jvenstad/deployment api expansion continued

14 files changed, 444 insertions, 205 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index 364e91f1828..9ad03522179 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -20,6 +20,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Collector;
 import java.util.stream.Collectors;
 
 /**
@@ -98,7 +99,8 @@ public class Application {
     public Map<Zone, Deployment> productionDeployments() {
         return deployments.values().stream()
                                    .filter(deployment -> deployment.zone().environment() == Environment.prod)
-                                   .collect(Collectors.toMap(Deployment::zone, Function.identity()));
+                                   .collect(Collectors.collectingAndThen(Collectors.toMap(Deployment::zone, Function.identity()),
+                                                                         ImmutableMap::copyOf));
     }
 
     public DeploymentJobs deploymentJobs() { return deploymentJobs; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 26debf3083f..66bb076e099 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -303,7 +303,7 @@ public class ApplicationController {
             else
                 version = application.currentDeployVersion(controller, zone);
 
-            DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone);
+            Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone);
             ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob);
 
             if ( ! options.deployCurrentVersion) {
@@ -314,11 +314,11 @@ public class ApplicationController {
                     application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value());
                 if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange)
                     application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision)));
-                if ( ! canDeployDirectlyTo(zone, options) && jobType != null) {
+                if ( ! canDeployDirectlyTo(zone, options) && jobType.isPresent()) {
                     // Update with (potentially) missing information about what we triggered
-                    JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType);
+                    JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get());
                     application = application.with(application.deploymentJobs()
-                                                           .withTriggering(jobType,
+                                                           .withTriggering(jobType.get(),
                                                                            application.deploying(),
                                                                            triggering.id(),
                                                                            version,
@@ -423,10 +423,9 @@ public class ApplicationController {
      * This is needed (only) in the case where some external entity triggers a job.
      */
     private JobStatus.JobRun getOrCreateTriggering(Application application, Version version, DeploymentJobs.JobType jobType) {
-        if (jobType == null) return incompleteTriggeringEvent(version);
         JobStatus status = application.deploymentJobs().jobStatus().get(jobType);
-        if (status == null) return  incompleteTriggeringEvent(version);
-        if ( ! status.lastTriggered().isPresent()) return  incompleteTriggeringEvent(version);
+        if (status == null) return incompleteTriggeringEvent(version);
+        if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version);
         return status.lastTriggered().get();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index 710d2ad6492..ae015c36aac 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 
 import java.time.Instant;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
@@ -23,17 +24,17 @@ public class ApplicationList {
 
     private final ImmutableList<Application> list;
 
-    private ApplicationList(List<Application> applications) {
+    private ApplicationList(Iterable<Application> applications) {
         this.list = ImmutableList.copyOf(applications);
     }
     
     // ----------------------------------- Factories
     
-    public static ApplicationList from(List<Application> applications) {
+    public static ApplicationList from(Iterable<Application> applications) {
         return new ApplicationList(applications);
     }
 
-    public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) {
+    public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) {
         return listOf(ids.stream().map(applications::require));
     }
 
@@ -166,7 +167,7 @@ public class ApplicationList {
 
     /** Returns the subset of applications that are not currently upgrading */
     public ApplicationList notCurrentlyUpgrading(Change.VersionChange change, Instant jobTimeoutLimit) {
-        return listOf(list.stream().filter(a -> !currentlyUpgrading(change, a, jobTimeoutLimit)));
+        return listOf(list.stream().filter(a -> ! currentlyUpgrading(change, a, jobTimeoutLimit)));
     }
 
     // ----------------------------------- Internal helpers
@@ -193,56 +194,39 @@ public class ApplicationList {
         if ( ! application.deploying().isPresent()) return false;
         return application.deploying().get() instanceof Change.ApplicationChange;
     }
-    
+
     private static boolean failingOn(Version version, Application application) {
-        for (JobStatus jobStatus : application.deploymentJobs().jobStatus().values())
-            if ( ! jobStatus.isSuccess() && jobStatus.lastCompleted().get().version().equals(version)) return true;
-        return false;
+        return JobList.from(application)
+                .failing()
+                .lastCompleted().on(version)
+                .anyMatch();
     }
 
     private static boolean currentlyUpgrading(Change.VersionChange change, Application application, Instant jobTimeoutLimit) {
-        return application.deploymentJobs().jobStatus().values().stream()
-                .filter(status -> status.isRunning(jobTimeoutLimit))
-                .filter(status -> status.lastTriggered().isPresent())
-                .map(status -> status.lastTriggered().get())
-                .anyMatch(jobRun -> jobRun.version().equals(change.version()));
+        return JobList.from(application)
+                .running(jobTimeoutLimit)
+                .lastTriggered().on(change.version())
+                .anyMatch();
     }
 
     private static boolean failingUpgradeToVersionSince(Application application, Version version, Instant threshold) {
-        return application.deploymentJobs().jobStatus().values().stream()
-                .filter(job -> isUpgradeFailure(job))
-                .filter(job -> job.firstFailing().get().at().isBefore(threshold))
-                .anyMatch(job -> job.lastCompleted().get().version().equals(version));
+        return JobList.from(application)
+                .not().failingApplicationChange()
+                .firstFailing().before(threshold)
+                .lastCompleted().on(version)
+                .anyMatch();
     }
 
     private static boolean failingApplicationChangeSince(Application application, Instant threshold) {
-        return application.deploymentJobs().jobStatus().values().stream()
-                .filter(job -> isApplicationChangeFailure(job))
-                .anyMatch(job -> job.firstFailing().get().at().isBefore(threshold));
-    }
-
-    private static boolean isUpgradeFailure(JobStatus job) {
-        if (   job.isSuccess()) return false;
-        if ( ! job.lastSuccess().isPresent()) return false; // An application which never succeeded is surely bad.
-        if ( ! job.lastSuccess().get().revision().isPresent()) return false; // Indicates the component job, which is not an upgrade.
-        if ( ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision())) return false; // Application change may be to blame.
-        return ! job.firstFailing().get().version().equals(job.lastSuccess().get().version()); // Return whether there is a version change.
+        return JobList.from(application)
+                .failingApplicationChange()
+                .firstFailing().before(threshold)
+                .anyMatch();
     }
 
-    private static boolean isApplicationChangeFailure(JobStatus job) {
-        if (   job.isSuccess()) return false;
-        if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad.
-        if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change.
-        if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame.
-        return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change.
-    }
-
-
     /** Convenience converter from a stream to an ApplicationList */
     private static ApplicationList listOf(Stream<Application> applications) {
-        ImmutableList.Builder<Application> b = new ImmutableList.Builder<>();
-        applications.forEach(b::add);
-        return new ApplicationList(b.build());
+        return from(applications::iterator);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index 8f654c66871..f94b0f7284b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -13,12 +13,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 
 import java.time.Instant;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Information about which deployment jobs an application should run and their current status.
@@ -148,13 +150,11 @@ public class DeploymentJobs {
     
     /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */
     public Instant failingSince() {
-        Instant failingSince = null;
-        for (JobStatus jobStatus : jobStatus().values()) {
-            if (jobStatus.isSuccess()) continue;
-            if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at()))
-                failingSince = jobStatus.firstFailing().get().at();
-        }
-        return failingSince;
+        return JobList.from(jobStatus().values())
+                .failing()
+                .mapToList(job -> job.firstFailing().get().at())
+                .stream()
+                .min(Comparator.naturalOrder()).orElse(null);
     }
 
     /**
@@ -184,18 +184,12 @@ public class DeploymentJobs {
         productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2"));
 
         private final String id;
-        private final Map<SystemName, Zone> zones;
+        private final ImmutableMap<SystemName, Zone> zones;
 
-        JobType(String id, Zone... zone) {
+        JobType(String id, Zone... zones) {
             this.id = id;
-            Map<SystemName, Zone> zones = new HashMap<>();
-            for (Zone z : zone) {
-                if (zones.containsKey(z.system())) {
-                    throw new IllegalArgumentException("A job can only map to a single zone per system");
-                }
-                zones.put(z.system(), z);
-            }
-            this.zones = Collections.unmodifiableMap(zones);
+            this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(),
+                                                                                       zone -> zone)));
         }
 
         public String id() { return id; }
@@ -242,23 +236,23 @@ public class DeploymentJobs {
             }
         }
         
-        /** Returns the job type for the given zone, or null if none */
-        public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) {
+        /** Returns the job type for the given zone */
+        public static Optional<JobType> from(SystemName system, Zone zone) {
            for (JobType job : values()) {
-               Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system);
+               Optional<Zone> jobZone = job.zone(system);
                if (jobZone.isPresent() && jobZone.get().equals(zone))
-                   return job;
+                   return Optional.of(job);
            }
-           return null;
+           return Optional.empty();
         }
 
         /** Returns the job job type for the given environment and region or null if none */
-        public static JobType from(SystemName system, Environment environment, RegionName region) {
+        public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) {
             switch (environment) {
-                case test: return systemTest;
-                case staging: return stagingTest;
+                case test: return Optional.of(systemTest);
+                case staging: return Optional.of(stagingTest);
             }
-            return from(system, new com.yahoo.config.provision.Zone(environment, region));
+            return from(system, new Zone(environment, region));
         }
 
         private static Zone zone(SystemName system, String environment, String region) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java
new file mode 100644
index 00000000000..19c7bbe98b4
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java
@@ -0,0 +1,179 @@
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
+
+/**
+ * A list of deployment jobs that can be filtered in various ways.
+ *
+ * @author jvenstad
+ */
+public class JobList {
+
+    private final ImmutableList<JobStatus> list;
+    private final boolean negate;
+
+    private JobList(Iterable<JobStatus> jobs, boolean negate) {
+        this.list = ImmutableList.copyOf(jobs);
+        this.negate = negate;
+    }
+
+    private JobList(Iterable<JobStatus> jobs) {
+        this(jobs, false);
+    }
+
+    // ----------------------------------- Factories
+
+    public static JobList from(Iterable<JobStatus> jobs) {
+        return new JobList(jobs);
+    }
+
+    public static JobList from(Application application) {
+        return from(application.deploymentJobs().jobStatus().values());
+    }
+
+    // ----------------------------------- Accessors
+
+    /** Returns the jobstatuses in this as an immutable list */
+    public List<JobStatus> asList() { return list; }
+
+    /** Returns the jobstatuses in this as an immutable list after mapping with the given function */
+    public <Type> List<Type> mapToList(Function<JobStatus, Type> mapper) {
+        return ImmutableList.copyOf(list.stream().map(mapper)::iterator);
+    }
+
+    public boolean isEmpty() { return list.isEmpty(); }
+
+    public boolean anyMatch() { return ! isEmpty(); }
+
+    public int size() { return list.size(); }
+
+    // ----------------------------------- Basic filters
+
+    /** Negates the next filter operation */
+    public JobList not() {
+        return new JobList(list, ! negate);
+    }
+
+    /** Returns the subset of jobs which are currently running, according to the given timeout */
+    public JobList running(Instant timeoutLimit) {
+        return filter(job -> job.isRunning(timeoutLimit));
+    }
+
+    /** Returns the subset of jobs which are currently failing */
+    public JobList failing() {
+        return filter(job -> ! job.isSuccess());
+    }
+
+    /** Returns the subset of jobs which must be failing due to an application change */
+    public JobList failingApplicationChange() {
+        return filter(job -> failingApplicationChange(job));
+    }
+
+    /** Returns the subset of jobs which are failing with the given job error */
+    public JobList failingBecause(DeploymentJobs.JobError error) {
+        return filter(job -> job.jobError().filter(error::equals).isPresent());
+    }
+
+    /** Returns the subset of jobs of the given type -- most useful when negated */
+    public JobList type(JobType type) {
+        return filter(job -> job.type() == type);
+    }
+
+    /** Returns the subset of jobs of which are production jobs */
+    public JobList production() {
+        return filter(job -> job.type().isProduction());
+    }
+
+    // ----------------------------------- JobRun filtering
+
+    /** Returns the list in a state where the next filter is for the lastTriggered run type */
+    public JobRunFilter lastTriggered() {
+        return new JobRunFilter(job -> job.lastTriggered());
+    }
+
+    /** Returns the list in a state where the next filter is for the lastCompleted run type */
+    public JobRunFilter lastCompleted() {
+        return new JobRunFilter(job -> job.lastCompleted());
+    }
+
+    /** Returns the list in a state where the next filter is for the lastSuccess run type */
+    public JobRunFilter lastSuccess() {
+        return new JobRunFilter(job -> job.lastSuccess());
+    }
+
+    /** Returns the list in a state where the next filter is for the firstFailing run type */
+    public JobRunFilter firstFailing() {
+        return new JobRunFilter(job -> job.firstFailing());
+    }
+
+
+    /** Allows sub-filters for runs of the given kind */
+    public class JobRunFilter {
+
+        private final Function<JobStatus, Optional<JobRun>> which;
+
+        private JobRunFilter(Function<JobStatus, Optional<JobRun>> which) {
+            this.which = which;
+        }
+
+        /** Returns the subset of jobs where the run of the given type exists */
+        public JobList present() {
+            return filter(run -> true);
+        }
+
+        /** Returns the subset of jobs where the run of the given type occurred before the given instant */
+        public JobList before(Instant threshold) {
+            return filter(run -> run.at().isBefore(threshold));
+        }
+
+        /** Returns the subset of jobs where the run of the given type occurred after the given instant */
+        public JobList after(Instant threshold) {
+            return filter(run -> run.at().isAfter(threshold));
+        }
+
+        /** Returns the subset of jobs where the run of the given type was on the given version */
+        public JobList on(Version version) {
+            return filter(run -> run.version().equals(version));
+        }
+
+        public JobList upgrade() {
+            return filter(run -> run.upgrade());
+        }
+
+        /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */
+        private JobList filter(Predicate<JobRun> condition) {
+            return JobList.this.filter(job -> which.apply(job).filter(condition).isPresent());
+        }
+
+    }
+
+
+    // ----------------------------------- Internal helpers
+
+    private static boolean failingApplicationChange(JobStatus job) {
+        if (   job.isSuccess()) return false;
+        if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad.
+        if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change.
+        if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame.
+        return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change.
+    }
+
+    /** Returns a new JobList which is the result of filtering with the -- possibly negated -- condition */
+    private JobList filter(Predicate<JobStatus> condition) {
+        return from(list.stream().filter(negate ? condition.negate() : condition)::iterator);
+    }
+
+}
+
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
index bb84c9e17d4..8fb01577e2c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
@@ -162,7 +162,8 @@ public class DeploymentOrder {
 
     /** Resolve job from deployment step */
     private JobType toJob(DeploymentSpec.DeclaredZone zone) {
-        return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null));
+        return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null))
+                .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone));
     }
 
     /** Returns whether deployment should be postponed according to delay */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
index 81c3bb963db..6b60b49e1ef 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -40,6 +41,7 @@ public class VersionStatusSerializer {
     private static final String versionField = "version";
     private static final String failingField = "failing";
     private static final String productionField = "production";
+    private static final String deployingField = "deploying";
 
     public Slime toSlime(VersionStatus status) {
         Slime slime = new Slime();
@@ -74,9 +76,10 @@ public class VersionStatusSerializer {
         object.setString(versionField, statistics.version().toString());
         applicationsToSlime(statistics.failing(), object.setArray(failingField));
         applicationsToSlime(statistics.production(), object.setArray(productionField));
+        applicationsToSlime(statistics.deploying(), object.setArray(deployingField));
     }
 
-    private void applicationsToSlime(List<ApplicationId> applications, Cursor array) {
+    private void applicationsToSlime(Collection<ApplicationId> applications, Cursor array) {
         applications.forEach(application -> array.addString(application.serializedForm()));
     }
 
@@ -105,7 +108,8 @@ public class VersionStatusSerializer {
     private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) {
         return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()),
                                         applicationsFromSlime(object.field(failingField)),
-                                        applicationsFromSlime(object.field(productionField)));
+                                        applicationsFromSlime(object.field(productionField)),
+                                        applicationsFromSlime(object.field(deployingField)));
     }
 
     private List<ApplicationId> applicationsFromSlime(Inspector array) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 8a5f1e4639a..bcc72245f0b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.container.jdisc.HttpRequest;
@@ -11,6 +12,8 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.JobList;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
@@ -19,15 +22,17 @@ import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse;
 import com.yahoo.vespa.hosted.controller.restapi.Path;
 import com.yahoo.yolean.Exceptions;
 
-import java.time.Instant;
 import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.logging.Level;
 
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
+import static java.util.Comparator.comparing;
+
 /**
  * This implements the deployment/v1 API which provides information about the status of Vespa platform and
  * application deployments.
- * 
+ *
  * @author bratseth
  */
 public class DeploymentApiHandler extends LoggingRequestHandler {
@@ -56,7 +61,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
             return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
         }
     }
-    
+
     private HttpResponse handleGET(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/deployment/v1/")) return root(request);
@@ -70,7 +75,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
         response.headers().put("Allow", "GET,OPTIONS");
         return response;
     }
-    
+
     private HttpResponse root(HttpRequest request) {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
@@ -83,7 +88,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
             versionObject.setLong("date", version.releasedAt().toEpochMilli());
             versionObject.setBool("controllerVersion", version.isSelfVersion());
             versionObject.setBool("systemVersion", version.isCurrentSystemVersion());
-            
+
             Cursor configServerArray = versionObject.setArray("configServers");
             for (String configServerHostnames : version.configServerHostnames()) {
                 Cursor configServerObject = configServerArray.addObject();
@@ -92,29 +97,42 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
 
             Cursor failingArray = versionObject.setArray("failingApplications");
             for (ApplicationId id : version.statistics().failing()) {
-                Optional<Application> application = controller.applications().get(id);
-                if ( ! application.isPresent()) continue; // deleted just now
-
-                Instant failingSince = application.get().deploymentJobs().failingSince();
-                if (failingSince == null) continue; // started working just now
-
-                Cursor applicationObject = failingArray.addObject();
-                toSlime(application.get(), applicationObject, request);
-                applicationObject.setLong("failingSince", failingSince.toEpochMilli());
-
+                controller.applications().get(id).ifPresent(application -> {
+                    firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> {
+                        Cursor applicationObject = failingArray.addObject();
+                        toSlime(applicationObject, application, request);
+                        applicationObject.setString("failing", firstFailing.type().id());
+                    });
+                });
             }
 
             Cursor productionArray = versionObject.setArray("productionApplications");
             for (ApplicationId id : version.statistics().production()) {
-                Optional<Application> application = controller.applications().get(id);
-                if ( ! application.isPresent()) continue; // deleted just now
-                toSlime(application.get(), productionArray.addObject(), request);
+                controller.applications().get(id).ifPresent(application -> {
+                    int successes = productionSuccessesFor(version.versionNumber(), application);
+                    if (successes == 0) return; // Just upgraded to a newer version.
+                    Cursor applicationObject = productionArray.addObject();
+                    toSlime(applicationObject, application, request);
+                    applicationObject.setLong("productionJobs", productionJobsFor(application));
+                    applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application));
+                });
+            }
+
+            Cursor runningArray = versionObject.setArray("deployingApplications");
+            for (ApplicationId id : version.statistics().deploying()) {
+                controller.applications().get(id).ifPresent(application -> {
+                    lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> {
+                        Cursor applicationObject = runningArray.addObject();
+                        toSlime(applicationObject, application, request);
+                        applicationObject.setString("running", lastDeploying.type().id());
+                    });
+                });
             }
         }
         return new SlimeJsonResponse(slime);
     }
 
-    private void toSlime(Application application, Cursor object, HttpRequest request) {
+    private void toSlime(Cursor object, Application application, HttpRequest request) {
         object.setString("tenant", application.id().tenant().value());
         object.setString("application", application.id().application().value());
         object.setString("instance", application.id().instance().value());
@@ -132,4 +150,41 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
         return upgradePolicy.name();
     }
 
+    // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus
+
+    /** The first upgrade job to fail on this version, for this application */
+    private Optional<JobStatus> firstFailingOn(Version version, Application application) {
+        return JobList.from(application)
+                .failing()
+                .not().failingApplicationChange()
+                .not().failingBecause(outOfCapacity)
+                .lastCompleted().on(version)
+                .asList().stream()
+                .min(comparing(job -> job.lastCompleted().get().at()));
+    }
+
+    /** The number of production jobs for this application */
+    private int productionJobsFor(Application application) {
+        return JobList.from(application)
+                .production()
+                .size();
+    }
+
+    /** The number of production jobs with last success on the given version, for this application */
+    private int productionSuccessesFor(Version version, Application application) {
+        return JobList.from(application)
+                .production()
+                .lastSuccess().on(version)
+                .size();
+    }
+
+    /** The last triggered upgrade to this version, for this application */
+    private Optional<JobStatus> lastDeployingTo(Version version, Application application) {
+        return JobList.from(application)
+                .running(controller.applications().deploymentTrigger().jobTimeoutLimit())
+                .lastTriggered().upgrade()
+                .asList().stream()
+                .max(comparing(job -> job.lastTriggered().get().at()));
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
index 6174a017a54..ae7223489c2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
@@ -1,12 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 
-import java.util.List;
+import java.util.Collection;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * Statistics about deployments on a platform version. This is immutable.
@@ -16,20 +17,22 @@ import java.util.Objects;
 public class DeploymentStatistics {
     
     private final Version version;
-    private final ImmutableList<ApplicationId> failing;
-    private final ImmutableList<ApplicationId> production;
+    private final ImmutableSet<ApplicationId> failing;
+    private final ImmutableSet<ApplicationId> production;
+    private final ImmutableSet<ApplicationId> deploying;
 
     /** DO NOT USE. Public for serialization purposes */
-    public DeploymentStatistics(Version version, List<ApplicationId> failingApplications,
-                                List<ApplicationId> production) {
+    public DeploymentStatistics(Version version, Collection<ApplicationId> failingApplications,
+                                Collection<ApplicationId> production, Collection<ApplicationId> deploying) {
         this.version = version;
-        this.failing = ImmutableList.copyOf(failingApplications);
-        this.production = ImmutableList.copyOf(production);
+        this.failing = ImmutableSet.copyOf(failingApplications);
+        this.production = ImmutableSet.copyOf(production);
+        this.deploying = ImmutableSet.copyOf(deploying);
     }
 
     /** Returns a statistics instance with the values as 0 */
     public static DeploymentStatistics empty(Version version) { 
-        return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of()); 
+        return new DeploymentStatistics(version, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of());
     }
 
     /** Returns the version these statistics are for */
@@ -39,23 +42,31 @@ public class DeploymentStatistics {
      * Returns the applications which have at least one job (of any type) which fails on this version, 
      * excluding errors known to not be caused by this version
      */
-    public List<ApplicationId> failing() { return failing; }
+    public Set<ApplicationId> failing() { return failing; }
     
     /** Returns the applications which have this version in production in at least one zone */
-    public List<ApplicationId> production() { return production; }
-    
+    public Set<ApplicationId> production() { return production; }
+
+    /** Returns the applications which are currently upgrading to this version */
+    public Set<ApplicationId> deploying() { return deploying; }
+
     /** Returns a version of this with the given failing application added */
     public DeploymentStatistics withFailing(ApplicationId application) { 
-        return new DeploymentStatistics(version, add(application, failing), production);
+        return new DeploymentStatistics(version, add(application, failing), production, deploying);
     }
 
     /** Returns a version of this with the given production application added */
     public DeploymentStatistics withProduction(ApplicationId application) {
-        return new DeploymentStatistics(version, failing, add(application, production));
+        return new DeploymentStatistics(version, failing, add(application, production), deploying);
+    }
+
+    /** Returns a version of this with the given deploying application added */
+    public DeploymentStatistics withDeploying(ApplicationId application) {
+        return new DeploymentStatistics(version, failing, production, add(application, deploying));
     }
     
-    private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) {
-        ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>();
+    private ImmutableSet<ApplicationId> add(ApplicationId application, ImmutableSet<ApplicationId> list) {
+        ImmutableSet.Builder<ApplicationId> b = new ImmutableSet.Builder<>();
         b.addAll(list);
         b.add(application);
         return b.build();
@@ -75,4 +86,5 @@ public class DeploymentStatistics {
     public int hashCode() {
         return Objects.hash(version, failing, production);
     }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 73e4eb4d527..3de4a7539f8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
-import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.JobList;
 
 import java.net.URI;
 import java.time.Instant;
@@ -28,6 +28,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
+
 /**
  * Information about the current platform versions in use.
  * The versions in use are the set of all versions running in current applications, versions
@@ -92,7 +94,8 @@ public class VersionStatus {
         Version systemVersion = infrastructureVersions.stream().sorted().findFirst().get();
 
         Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions,
-                                                                                            controller.applications().asList());
+                                                                                            controller.applications().asList(),
+                                                                                            controller.applications().deploymentTrigger().jobTimeoutLimit());
         List<VespaVersion> versions = new ArrayList<>();
 
         for (DeploymentStatistics statistics : deploymentStatistics) {
@@ -126,7 +129,8 @@ public class VersionStatus {
     }
     
     private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions,
-                                                                                List<Application> applications) {
+                                                                                List<Application> applications,
+                                                                                Instant jobTimeoutLimit) {
         Map<Version, DeploymentStatistics> versionMap = new HashMap<>();
 
         for (Version infrastructureVersion : infrastructureVersions) {
@@ -142,41 +146,39 @@ public class VersionStatus {
                 versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty);
             }
 
-            // List versions which have failing jobs, and versions which are in production
+            // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs
 
             // Failing versions
-            Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream()
-                    .filter(jobStatus -> jobStatus.lastCompleted().isPresent())
-                    .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade())
-                    .filter(jobStatus -> jobStatus.jobError().isPresent())
-                    .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity)
-                    .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version()));
-            for (Version v : failingJobsByVersion.keySet()) {
-                versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id()));
-            }
+            JobList.from(application)
+                    .failing()
+                    .not().failingApplicationChange()
+                    .not().failingBecause(outOfCapacity)
+                    .mapToList(job -> job.lastCompleted().get().version())
+                    .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withFailing(application.id())));
 
             // Succeeding versions
-            Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream()
-                    .filter(jobStatus -> jobStatus.lastSuccess().isPresent())
-                    .filter(jobStatus -> jobStatus.type().isProduction())
-                    .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version()));
-            for (Version v : succeedingJobsByVersions.keySet()) {
-                versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id()));
-            }
+            JobList.from(application)
+                    .lastSuccess().present()
+                    .production()
+                    .mapToList(job -> job.lastSuccess().get().version())
+                    .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withProduction(application.id())));
+
+            // Deploying versions
+            JobList.from(application)
+                    .running(jobTimeoutLimit)
+                    .lastTriggered().upgrade()
+                    .mapToList(job -> job.lastTriggered().get().version())
+                    .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withDeploying(application.id())));
         }
         return versionMap.values();
     }
     
-    private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) {
-        return statistics == null ? DeploymentStatistics.empty(version) : statistics;
-    }
-
-    private static VespaVersion createVersion(DeploymentStatistics statistics, 
+    private static VespaVersion createVersion(DeploymentStatistics statistics,
                                               boolean isSystemVersion, 
                                               Collection<String> configServerHostnames,
                                               Controller controller) {
         GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString());
-        Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime());
+        Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); // commitedAt ...
         VespaVersion.Confidence confidence;
         // Always compute confidence for system version
         if (isSystemVersion) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 79fd717a24f..9e228e0becb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -225,21 +226,20 @@ public class DeploymentTester {
     /** Assert that the sceduled jobs of this application are exactly those given, and take them */
     private void consumeJobs(Application application, boolean expectOnlyTheseJobs, JobType... jobs) {
         for (JobType job : jobs) {
-            Optional<BuildService.BuildJob> buildJob = findJob(application, job);
-            assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent());
-            assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId());
-            assertEquals(job.id(), buildJob.get().jobName());
+            BuildService.BuildJob buildJob = findJob(application, job);
+            assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId());
+            assertEquals(job.id(), buildJob.jobName());
         }
         if (expectOnlyTheseJobs)
             assertEquals(jobs.length, countJobsOf(application));
         buildSystem().removeJobs(application.id());
     }
 
-    private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) {
+    private BuildService.BuildJob findJob(Application application, JobType jobType) {
         for (BuildService.BuildJob job : buildSystem().jobs())
             if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id()))
-                return Optional.of(job);
-        return Optional.empty();
+                return job;
+        throw new NoSuchElementException(jobType + " is not scheduled for " + application);
     }
 
     private int countJobsOf(Application application) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
index f5f43265cb8..189b3a97a80 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -27,6 +27,8 @@ public class VersionStatusSerializerTest {
                 Version.fromString("5.0"),
                 Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")),
                 Arrays.asList(ApplicationId.from("tenant2", "success1", "default"),
+                              ApplicationId.from("tenant2", "success2", "default")),
+                Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"),
                               ApplicationId.from("tenant2", "success2", "default"))
         );
         vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index 7fd000b82c5..5f7fedfd75f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -1,27 +1,26 @@
 {
   "versions":[
     {
-      "version":"(ignore)",
-      "confidence":"high",
-      "commit":"(ignore)",
-      "date":0,
-      "controllerVersion":false,
-      "systemVersion":false,
-      "configServers":[
-
-      ],
-      "failingApplications":[
-
-      ],
-      "productionApplications":[
+      "version": "(ignore)",
+      "confidence": "high",
+      "commit": "(ignore)",
+      "date": 0,
+      "controllerVersion": false,
+      "systemVersion": false,
+      "configServers": [ ],
+      "failingApplications": [ ],
+      "productionApplications": [
         {
-          "tenant":"tenant1",
-          "application":"application1",
-          "instance":"default",
-          "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1",
-          "upgradePolicy":"default"
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+          "upgradePolicy": "default",
+          "productionJobs": 1,
+          "productionSuccesses": 1
         }
-      ]
+      ],
+      "deployingApplications": [ ]
     },
     {
       "version":"(ignore)",
@@ -40,40 +39,47 @@
       ],
       "failingApplications":[
         {
-          "tenant":"tenant1",
-          "application":"application1",
-          "instance":"default",
-          "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1",
-          "upgradePolicy":"default",
-          "failingSince":"(ignore)"
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+          "upgradePolicy": "default",
+          "failing": "staging-test"
         }
       ],
       "productionApplications":[
         {
-          "tenant":"tenant2",
-          "application":"application2",
-          "instance":"default",
-          "url":"http://localhost:8080/application/v4/tenant/tenant2/application/application2",
-          "upgradePolicy":"default"
+          "tenant": "tenant2",
+          "application": "application2",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2",
+          "upgradePolicy": "default",
+          "productionJobs": 1,
+          "productionSuccesses": 1
+        }
+      ],
+      "deployingApplications": [
+        {
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+          "upgradePolicy": "default",
+          "running": "staging-test"
         }
       ]
     },
     {
-      "version":"(ignore)",
-      "confidence":"normal",
-      "commit":"(ignore)",
-      "date":0,
-      "controllerVersion":true,
-      "systemVersion":false,
-      "configServers":[
-
-      ],
-      "failingApplications":[
-
-      ],
-      "productionApplications":[
-
-      ]
+      "version": "(ignore)",
+      "confidence": "normal",
+      "commit": "(ignore)",
+      "date": 0,
+      "controllerVersion": true,
+      "systemVersion": false,
+      "configServers": [ ],
+      "failingApplications": [ ],
+      "productionApplications": [ ],
+      "deployingApplications": [ ]
     }
   ]
 }
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 519c457e73b..1157863f009 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
+import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.Environment;
@@ -19,6 +20,7 @@ import org.junit.Test;
 
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Collections;
 import java.util.List;
 
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
@@ -28,6 +30,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -95,23 +98,20 @@ public class VersionStatusTest {
         List<VespaVersion> versions = tester.controller().versionStatus().versions();
         assertEquals("The two versions above exist", 2, versions.size());
 
+        System.err.println(tester.controller().applications().deploymentTrigger().jobTimeoutLimit());
+
         VespaVersion v1 = versions.get(0);
         assertEquals(version1, v1.versionNumber());
-        assertEquals(0, v1.statistics().failing().size());
-        // All applications are on v1 in at least one zone
-        assertEquals(3, v1.statistics().production().size());
-        assertTrue(v1.statistics().production().contains(app2.id()));
-        assertTrue(v1.statistics().production().contains(app1.id()));
+        assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing());
+        assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v1.statistics().production());
+        assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying());
 
         VespaVersion v2 = versions.get(1);
         assertEquals(version2, v2.versionNumber());
-        // All applications have failed on v2 in at least one zone
-        assertEquals(3, v2.statistics().failing().size());
-        assertTrue(v2.statistics().failing().contains(app1.id()));
-        assertTrue(v2.statistics().failing().contains(app3.id()));
-        // Only one application is on v2 in at least one zone
-        assertEquals(1, v2.statistics().production().size());
-        assertTrue(v2.statistics().production().contains(app2.id()));
+        assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().failing());
+        assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(app2.id()), v2.statistics().production());
+        // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest.
+        // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying());
     }
     
     @Test
@@ -241,7 +241,7 @@ public class VersionStatusTest {
     }
 
     @Test
-    public void testIgnoreConfigdeince() {
+    public void testIgnoreConfidence() {
         DeploymentTester tester = new DeploymentTester();
 
         Version version0 = new Version("5.0");
@@ -270,7 +270,6 @@ public class VersionStatusTest {
         tester.completeUpgradeWithError(default3, version1, "default", stagingTest);
         tester.completeUpgradeWithError(default4, version1, "default", stagingTest);
         tester.updateVersionStatus();
-
         assertEquals("Canaries have upgraded, 1 of 4 default apps failing: Broken",
                      Confidence.broken, confidence(tester.controller(), version1));