Add AbstractFilteringList as a super for such lists

4 files changed, 188 insertions, 81 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index 33d03801efc..8978af78574 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
-import com.google.common.collect.ImmutableList;
+import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
@@ -16,7 +16,6 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -24,55 +23,48 @@ import java.util.stream.Collectors;
  *
  * @author jonmv
  */
-public class ApplicationList {
+public class ApplicationList extends AbstractFilteringList<Application, ApplicationList> {
 
-    private final List<Application> list;
-
-    private ApplicationList(List<Application> applications) {
-        this.list = applications;
+    private ApplicationList(Collection<? extends Application> applications, boolean negate) {
+        super(applications, negate, ApplicationList::new);
     }
 
     // ----------------------------------- Factories
 
-    public static ApplicationList from(Collection<Application> applications) {
-        return new ApplicationList(List.copyOf(applications));
+    public static ApplicationList from(Collection<? extends Application> applications) {
+        return new ApplicationList(applications, false);
     }
 
     public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) {
-        return new ApplicationList(ids.stream()
-                                      .map(TenantAndApplicationId::from)
-                                      .distinct()
-                                      .map(applications::requireApplication)
-                                      .collect(Collectors.toUnmodifiableList()));
+        return from(ids.stream()
+                       .map(TenantAndApplicationId::from)
+                       .distinct()
+                       .map(applications::requireApplication)
+                       .collect(Collectors.toUnmodifiableList()));
     }
 
     // ----------------------------------- Accessors
 
-    /** Returns the applications in this as an immutable list */
-    public List<Application> asList() { return list; }
-
     /** Returns the ids of the applications in this as an immutable list */
-    public List<TenantAndApplicationId> idList() { return list.stream().map(Application::id).collect(Collectors.toUnmodifiableList()); }
-
-    public boolean isEmpty() { return list.isEmpty(); }
-
-    public int size() { return list.size(); }
+    public List<TenantAndApplicationId> idList() {
+        return mapToList(Application::id);
+    }
 
     // ----------------------------------- Filters
 
     /** Returns the subset of applications which are upgrading (to any version), not considering block windows. */
     public ApplicationList upgrading() {
-        return filteredOn(application -> application.change().platform().isPresent());
+        return matching(application -> application.change().platform().isPresent());
     }
 
     /** Returns the subset of applications which are currently upgrading to the given version */
     public ApplicationList upgradingTo(Version version) {
-        return filteredOn(application -> isUpgradingTo(version, application));
+        return matching(application -> isUpgradingTo(version, application));
     }
 
     /** Returns the subset of applications which are not pinned to a certain Vespa version. */
     public ApplicationList unpinned() {
-        return filteredOn(application -> ! application.change().isPinned());
+        return matching(application -> ! application.change().isPinned());
     }
 
     /** Returns the subset of applications which are currently not upgrading to the given version */
@@ -81,7 +73,7 @@ public class ApplicationList {
     }
 
     public ApplicationList notFailingUpgrade() {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .allMatch(instance -> JobList.from(instance)
                                                                                  .failing()
                                                                                  .not().failingApplicationChange()
@@ -90,7 +82,7 @@ public class ApplicationList {
 
     /** Returns the subset of applications which are currently not upgrading to any of the given versions */
     public ApplicationList notUpgradingTo(Collection<Version> versions) {
-        return filteredOn(application -> versions.stream().noneMatch(version -> isUpgradingTo(version, application)));
+        return matching(application -> versions.stream().noneMatch(version -> isUpgradingTo(version, application)));
     }
 
     /**
@@ -104,91 +96,91 @@ public class ApplicationList {
 
     /** Returns the subset of applications which have changes left to deploy; blocked, or deploying */
     public ApplicationList withChanges() {
-        return filteredOn(application -> application.change().hasTargets() || application.outstandingChange().hasTargets());
+        return matching(application -> application.change().hasTargets() || application.outstandingChange().hasTargets());
     }
 
     /** Returns the subset of applications which are currently not deploying a change */
     public ApplicationList notDeploying() {
-        return filteredOn(application -> ! application.change().hasTargets());
+        return matching(application -> ! application.change().hasTargets());
     }
 
     /** Returns the subset of applications which currently does not have any failing jobs */
     public ApplicationList notFailing() {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .noneMatch(instance -> instance.deploymentJobs().hasFailures()));
     }
 
     /** Returns the subset of applications which currently have failing jobs */
     public ApplicationList failing() {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .anyMatch(instance -> instance.deploymentJobs().hasFailures()));
     }
 
     /** Returns the subset of applications which have been failing an upgrade to the given version since the given instant */
     public ApplicationList failingUpgradeToVersionSince(Version version, Instant threshold) {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .anyMatch(instance -> failingUpgradeToVersionSince(instance, version, threshold)));
     }
 
     /** Returns the subset of applications which have been failing an application change since the given instant */
     public ApplicationList failingApplicationChangeSince(Instant threshold) {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                           .anyMatch(instance -> failingApplicationChangeSince(instance, threshold)));
     }
 
     /** Returns the subset of applications which currently does not have any failing jobs on the given version */
     public ApplicationList notFailingOn(Version version) {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                           .noneMatch(instance -> failingOn(version, instance)));
     }
 
     /** Returns the subset of applications which have at least one production deployment */
     public ApplicationList withProductionDeployment() {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .anyMatch(instance -> instance.productionDeployments().size() > 0));
     }
 
     /** Returns the subset of applications which started failing on the given version */
     public ApplicationList startedFailingOn(Version version) {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .anyMatch(instance ->  ! JobList.from(instance).firstFailing().on(version).isEmpty()));
     }
 
     /** Returns the subset of applications which has the given upgrade policy */
     // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance.
     public ApplicationList with(UpgradePolicy policy) {
-        return filteredOn(application ->  application.deploymentSpec().instances().stream()
+        return matching(application ->  application.deploymentSpec().instances().stream()
                                                      .anyMatch(instance -> instance.upgradePolicy() == policy));
     }
 
     /** Returns the subset of applications which does not have the given upgrade policy */
     // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance.
     public ApplicationList without(UpgradePolicy policy) {
-        return filteredOn(application ->  application.deploymentSpec().instances().stream()
+        return matching(application ->  application.deploymentSpec().instances().stream()
                                                      .allMatch(instance -> instance.upgradePolicy() != policy));
     }
 
     /** Returns the subset of applications which have at least one deployment on a lower version than the given one */
     public ApplicationList onLowerVersionThan(Version version) {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .flatMap(instance -> instance.productionDeployments().values().stream())
                                                     .anyMatch(deployment -> deployment.version().isBefore(version)));
     }
 
     /** Returns the subset of applications which have a project ID */
     public ApplicationList withProjectId() {
-        return filteredOn(application -> application.projectId().isPresent());
+        return matching(application -> application.projectId().isPresent());
     }
 
     /** Returns the subset of applications that are allowed to upgrade at the given time */
     public ApplicationList canUpgradeAt(Instant instant) {
-        return filteredOn(application -> application.deploymentSpec().instances().stream()
+        return matching(application -> application.deploymentSpec().instances().stream()
                                                     .allMatch(instance -> instance.canUpgradeAt(instant)));
     }
 
     /** Returns the subset of applications that have at least one assigned rotation */
     public ApplicationList hasRotation() {
-        return filteredOn(application -> application.instances().values().stream()
+        return matching(application -> application.instances().values().stream()
                                                     .anyMatch(instance -> ! instance.rotations().isEmpty()));
     }
 
@@ -199,20 +191,14 @@ public class ApplicationList {
      * @param defaultMajorVersion the default major version to assume for applications not specifying one
      */
     public ApplicationList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) {
-        return filteredOn(application -> targetMajorVersion <= application.deploymentSpec().majorVersion()
+        return matching(application -> targetMajorVersion <= application.deploymentSpec().majorVersion()
                                                                           .orElse(application.majorVersion()
                                                                                              .orElse(defaultMajorVersion)));
     }
 
     /** Returns the subset of application which have submitted a non-empty deployment spec. */
     public ApplicationList withDeploymentSpec() {
-        return filteredOn(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec()));
-    }
-
-    /** Returns the first n application in this (or all, if there are less than n). */
-    public ApplicationList first(int n) {
-        if (list.size() < n) return this;
-        return new ApplicationList(list.subList(0, n));
+        return matching(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec()));
     }
 
      // ----------------------------------- Sorting
@@ -223,10 +209,8 @@ public class ApplicationList {
      * Applications without any deployments are ordered first.
      */
     public ApplicationList byIncreasingDeployedVersion() {
-        return new ApplicationList(list.stream()
-                                       .sorted(Comparator.comparing(application -> application.oldestDeployedPlatform()
-                                                                                              .orElse(Version.emptyVersion)))
-                                       .collect(Collectors.toUnmodifiableList()));
+        return sortedBy(Comparator.comparing(application -> application.oldestDeployedPlatform()
+                                                                       .orElse(Version.emptyVersion)));
     }
 
     // ----------------------------------- Internal helpers
@@ -257,8 +241,4 @@ public class ApplicationList {
                         .isEmpty();
     }
 
-    private ApplicationList filteredOn(Predicate<Application> condition) {
-        return new ApplicationList(list.stream().filter(condition).collect(Collectors.toUnmodifiableList()));
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
index c4613db27d1..e12bb5cda7f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
@@ -1,10 +1,12 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
+import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Predicate;
@@ -17,41 +19,34 @@ import java.util.stream.Stream;
  *
  * @author mpolden
  */
-public class EndpointList {
+public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> {
 
     public static final EndpointList EMPTY = new EndpointList(List.of());
 
-    private final List<Endpoint> endpoints;
-
-    private EndpointList(List<Endpoint> endpoints) {
+    private EndpointList(Collection<? extends Endpoint> endpoints, boolean negate) {
+        super(endpoints, negate, EndpointList::new);
         if (endpoints.stream().distinct().count() != endpoints.size()) {
             throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints);
         }
-        this.endpoints = List.copyOf(endpoints);
     }
 
-    public List<Endpoint> asList() {
-        return endpoints;
+    private EndpointList(Collection<? extends Endpoint> endpoints) {
+        this(endpoints, false);
     }
 
     /** Returns the main endpoint, if any */
     public Optional<Endpoint> main() {
-        return endpoints.stream().filter(Predicate.not(Endpoint::legacy)).findFirst();
+        return asList().stream().filter(Predicate.not(Endpoint::legacy)).findFirst();
     }
 
     /** Returns the subset of endpoints are either legacy or not */
     public EndpointList legacy(boolean legacy) {
-        return of(endpoints.stream().filter(endpoint -> endpoint.legacy() == legacy));
+        return matching(endpoint -> endpoint.legacy() == legacy);
     }
 
     /** Returns the subset of endpoints with given scope */
     public EndpointList scope(Endpoint.Scope scope) {
-        return of(endpoints.stream().filter(endpoint -> endpoint.scope() == scope));
-    }
-
-    /** Returns the union of this and given endpoints */
-    public EndpointList and(EndpointList endpoints) {
-        return of(Stream.concat(asList().stream(), endpoints.asList().stream()));
+        return matching(endpoint -> endpoint.scope() == scope);
     }
 
     public static EndpointList of(Stream<Endpoint> endpoints) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index d0f34857b06..ae7ff81c001 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -63,14 +63,6 @@ import static java.util.stream.Collectors.toList;
  */
 public class DeploymentTrigger {
 
-    /*
-     * Instance orchestration TODO jonmv.
-     * Store new production application packages under non-instance path
-     * Read production packages from non-instance path, with fallback
-     * Deprecate and redirect some instance qualified paths in application/v4
-     * Orchestrate deployment across instances.
-     */
-
     public static final Duration maxPause = Duration.ofDays(3);
     private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName());
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
new file mode 100644
index 00000000000..973b718a4c6
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -0,0 +1,140 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.collections.AbstractFilteringList;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+/**
+ * A list of deployment jobs that can be filtered in various ways.
+ *
+ * @author jonmv
+ */
+public class JobList extends AbstractFilteringList<JobStatus, JobList> {
+
+    private JobList(Collection<? extends JobStatus> jobs, boolean negate) {
+        super(jobs, negate, JobList::new);
+    }
+
+    // ----------------------------------- Factories
+
+    public static JobList from(Collection<? extends JobStatus> jobs) {
+        return new JobList(jobs, false);
+    }
+
+    // ----------------------------------- Basic filters
+
+    /** Returns the subset of jobs which are currently upgrading */
+    public JobList upgrading() {
+        return matching(job -> job.isRunning()
+                               && job.lastSuccess().isPresent()
+                               && job.lastSuccess().get().versions().targetPlatform().isBefore(job.lastTriggered().get().versions().targetPlatform()));
+    }
+
+    /** Returns the subset of jobs which are currently failing */
+    public JobList failing() {
+        return matching(job -> ! job.isSuccess());
+    }
+
+    /** Returns the subset of jobs which must be failing due to an application change */
+    public JobList failingApplicationChange() {
+        return matching(JobList::failingApplicationChange);
+    }
+
+    /** Returns the subset of jobs which are failing with the given run status */
+    public JobList withStatus(RunStatus status) {
+        return matching(job -> job.lastStatus().map(status::equals).orElse(false));
+    }
+
+    /** Returns the subset of jobs of the given type -- most useful when negated */
+    public JobList type(JobType type) {
+        return matching(job -> job.id().type() == type);
+    }
+
+    /** Returns the subset of jobs of which are production jobs */
+    public JobList production() {
+        return matching(job -> job.id().type().isProduction());
+    }
+
+    // ----------------------------------- JobRun filtering
+
+    /** Returns the list in a state where the next filter is for the lastTriggered run type */
+    public RunFilter lastTriggered() {
+        return new RunFilter(JobStatus::lastTriggered);
+    }
+
+    /** Returns the list in a state where the next filter is for the lastCompleted run type */
+    public RunFilter lastCompleted() {
+        return new RunFilter(JobStatus::lastCompleted);
+    }
+
+    /** Returns the list in a state where the next filter is for the lastSuccess run type */
+    public RunFilter lastSuccess() {
+        return new RunFilter(JobStatus::lastSuccess);
+    }
+
+    /** Returns the list in a state where the next filter is for the firstFailing run type */
+    public RunFilter firstFailing() {
+        return new RunFilter(JobStatus::firstFailing);
+    }
+
+
+    /** Allows sub-filters for runs of the given kind */
+    public class RunFilter {
+
+        private final Function<JobStatus, Optional<Run>> which;
+
+        private RunFilter(Function<JobStatus, Optional<Run>> which) {
+            this.which = which;
+        }
+
+        /** Returns the subset of jobs where the run of the given type exists */
+        public JobList present() {
+            return matching(run -> true);
+        }
+
+        /** Returns the subset of jobs where the run of the given type occurred before the given instant */
+        public JobList startedBefore(Instant threshold) {
+            return matching(run -> run.start().isBefore(threshold));
+        }
+
+        /** Returns the subset of jobs where the run of the given type occurred after the given instant */
+        public JobList startedAfter(Instant threshold) {
+            return matching(run -> run.start().isAfter(threshold));
+        }
+
+        /** Returns the subset of jobs where the run of the given type was on the given version */
+        public JobList on(ApplicationVersion version) {
+            return matching(run -> run.versions().targetApplication().equals(version));
+        }
+
+        /** Returns the subset of jobs where the run of the given type was on the given version */
+        public JobList on(Version version) {
+            return matching(run -> run.versions().targetPlatform().equals(version));
+        }
+
+        /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */
+        private JobList matching(Predicate<Run> condition) {
+            return JobList.this.matching(job -> which.apply(job).filter(condition).isPresent());
+        }
+
+    }
+
+    // ----------------------------------- Internal helpers
+
+    private static boolean failingApplicationChange(JobStatus job) {
+        if (job.isSuccess()) return false;
+        if (job.lastSuccess().isEmpty()) return true; // An application which never succeeded is surely bad.
+        if ( ! job.firstFailing().get().versions().targetPlatform().equals(job.lastSuccess().get().versions().targetPlatform())) return false; // Version change may be to blame.
+        return ! job.firstFailing().get().versions().targetApplication().equals(job.lastSuccess().get().versions().targetApplication()); // Return whether there is an application change.
+    }
+
+}
+