From 24a1e9481ae732a67c8fe385293d07b8cf9b1096 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 8 Nov 2019 08:43:17 +0100 Subject: Add AbstractFilteringList as a super for such lists --- .../controller/application/ApplicationList.java | 96 ++++++-------- .../controller/application/EndpointList.java | 25 ++-- .../controller/deployment/DeploymentTrigger.java | 8 -- .../hosted/controller/deployment/JobList.java | 140 +++++++++++++++++++++ 4 files changed, 188 insertions(+), 81 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 33d03801efc..8978af78574 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; -import com.google.common.collect.ImmutableList; +import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -24,55 +23,48 @@ import java.util.stream.Collectors; * * @author jonmv */ -public class ApplicationList { +public class ApplicationList extends AbstractFilteringList { - private final List list; - - private ApplicationList(List applications) { - this.list = applications; + private ApplicationList(Collection applications, boolean negate) { + super(applications, negate, ApplicationList::new); } // ----------------------------------- Factories - public static ApplicationList from(Collection applications) { - return new ApplicationList(List.copyOf(applications)); + public static ApplicationList from(Collection applications) { + return new ApplicationList(applications, false); } public static ApplicationList from(Collection ids, ApplicationController applications) { - return new ApplicationList(ids.stream() - .map(TenantAndApplicationId::from) - .distinct() - .map(applications::requireApplication) - .collect(Collectors.toUnmodifiableList())); + return from(ids.stream() + .map(TenantAndApplicationId::from) + .distinct() + .map(applications::requireApplication) + .collect(Collectors.toUnmodifiableList())); } // ----------------------------------- Accessors - /** Returns the applications in this as an immutable list */ - public List asList() { return list; } - /** Returns the ids of the applications in this as an immutable list */ - public List idList() { return list.stream().map(Application::id).collect(Collectors.toUnmodifiableList()); } - - public boolean isEmpty() { return list.isEmpty(); } - - public int size() { return list.size(); } + public List idList() { + return mapToList(Application::id); + } // ----------------------------------- Filters /** Returns the subset of applications which are upgrading (to any version), not considering block windows. */ public ApplicationList upgrading() { - return filteredOn(application -> application.change().platform().isPresent()); + return matching(application -> application.change().platform().isPresent()); } /** Returns the subset of applications which are currently upgrading to the given version */ public ApplicationList upgradingTo(Version version) { - return filteredOn(application -> isUpgradingTo(version, application)); + return matching(application -> isUpgradingTo(version, application)); } /** Returns the subset of applications which are not pinned to a certain Vespa version. */ public ApplicationList unpinned() { - return filteredOn(application -> ! application.change().isPinned()); + return matching(application -> ! application.change().isPinned()); } /** Returns the subset of applications which are currently not upgrading to the given version */ @@ -81,7 +73,7 @@ public class ApplicationList { } public ApplicationList notFailingUpgrade() { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .allMatch(instance -> JobList.from(instance) .failing() .not().failingApplicationChange() @@ -90,7 +82,7 @@ public class ApplicationList { /** Returns the subset of applications which are currently not upgrading to any of the given versions */ public ApplicationList notUpgradingTo(Collection versions) { - return filteredOn(application -> versions.stream().noneMatch(version -> isUpgradingTo(version, application))); + return matching(application -> versions.stream().noneMatch(version -> isUpgradingTo(version, application))); } /** @@ -104,91 +96,91 @@ public class ApplicationList { /** Returns the subset of applications which have changes left to deploy; blocked, or deploying */ public ApplicationList withChanges() { - return filteredOn(application -> application.change().hasTargets() || application.outstandingChange().hasTargets()); + return matching(application -> application.change().hasTargets() || application.outstandingChange().hasTargets()); } /** Returns the subset of applications which are currently not deploying a change */ public ApplicationList notDeploying() { - return filteredOn(application -> ! application.change().hasTargets()); + return matching(application -> ! application.change().hasTargets()); } /** Returns the subset of applications which currently does not have any failing jobs */ public ApplicationList notFailing() { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .noneMatch(instance -> instance.deploymentJobs().hasFailures())); } /** Returns the subset of applications which currently have failing jobs */ public ApplicationList failing() { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> instance.deploymentJobs().hasFailures())); } /** Returns the subset of applications which have been failing an upgrade to the given version since the given instant */ public ApplicationList failingUpgradeToVersionSince(Version version, Instant threshold) { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> failingUpgradeToVersionSince(instance, version, threshold))); } /** Returns the subset of applications which have been failing an application change since the given instant */ public ApplicationList failingApplicationChangeSince(Instant threshold) { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> failingApplicationChangeSince(instance, threshold))); } /** Returns the subset of applications which currently does not have any failing jobs on the given version */ public ApplicationList notFailingOn(Version version) { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .noneMatch(instance -> failingOn(version, instance))); } /** Returns the subset of applications which have at least one production deployment */ public ApplicationList withProductionDeployment() { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> instance.productionDeployments().size() > 0)); } /** Returns the subset of applications which started failing on the given version */ public ApplicationList startedFailingOn(Version version) { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> ! JobList.from(instance).firstFailing().on(version).isEmpty())); } /** Returns the subset of applications which has the given upgrade policy */ // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance. public ApplicationList with(UpgradePolicy policy) { - return filteredOn(application -> application.deploymentSpec().instances().stream() + return matching(application -> application.deploymentSpec().instances().stream() .anyMatch(instance -> instance.upgradePolicy() == policy)); } /** Returns the subset of applications which does not have the given upgrade policy */ // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance. public ApplicationList without(UpgradePolicy policy) { - return filteredOn(application -> application.deploymentSpec().instances().stream() + return matching(application -> application.deploymentSpec().instances().stream() .allMatch(instance -> instance.upgradePolicy() != policy)); } /** Returns the subset of applications which have at least one deployment on a lower version than the given one */ public ApplicationList onLowerVersionThan(Version version) { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .flatMap(instance -> instance.productionDeployments().values().stream()) .anyMatch(deployment -> deployment.version().isBefore(version))); } /** Returns the subset of applications which have a project ID */ public ApplicationList withProjectId() { - return filteredOn(application -> application.projectId().isPresent()); + return matching(application -> application.projectId().isPresent()); } /** Returns the subset of applications that are allowed to upgrade at the given time */ public ApplicationList canUpgradeAt(Instant instant) { - return filteredOn(application -> application.deploymentSpec().instances().stream() + return matching(application -> application.deploymentSpec().instances().stream() .allMatch(instance -> instance.canUpgradeAt(instant))); } /** Returns the subset of applications that have at least one assigned rotation */ public ApplicationList hasRotation() { - return filteredOn(application -> application.instances().values().stream() + return matching(application -> application.instances().values().stream() .anyMatch(instance -> ! instance.rotations().isEmpty())); } @@ -199,20 +191,14 @@ public class ApplicationList { * @param defaultMajorVersion the default major version to assume for applications not specifying one */ public ApplicationList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) { - return filteredOn(application -> targetMajorVersion <= application.deploymentSpec().majorVersion() + return matching(application -> targetMajorVersion <= application.deploymentSpec().majorVersion() .orElse(application.majorVersion() .orElse(defaultMajorVersion))); } /** Returns the subset of application which have submitted a non-empty deployment spec. */ public ApplicationList withDeploymentSpec() { - return filteredOn(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec())); - } - - /** Returns the first n application in this (or all, if there are less than n). */ - public ApplicationList first(int n) { - if (list.size() < n) return this; - return new ApplicationList(list.subList(0, n)); + return matching(application -> ! DeploymentSpec.empty.equals(application.deploymentSpec())); } // ----------------------------------- Sorting @@ -223,10 +209,8 @@ public class ApplicationList { * Applications without any deployments are ordered first. */ public ApplicationList byIncreasingDeployedVersion() { - return new ApplicationList(list.stream() - .sorted(Comparator.comparing(application -> application.oldestDeployedPlatform() - .orElse(Version.emptyVersion))) - .collect(Collectors.toUnmodifiableList())); + return sortedBy(Comparator.comparing(application -> application.oldestDeployedPlatform() + .orElse(Version.emptyVersion))); } // ----------------------------------- Internal helpers @@ -257,8 +241,4 @@ public class ApplicationList { .isEmpty(); } - private ApplicationList filteredOn(Predicate condition) { - return new ApplicationList(list.stream().filter(condition).collect(Collectors.toUnmodifiableList())); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index c4613db27d1..e12bb5cda7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -1,10 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.yahoo.collections.AbstractFilteringList; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.function.Predicate; @@ -17,41 +19,34 @@ import java.util.stream.Stream; * * @author mpolden */ -public class EndpointList { +public class EndpointList extends AbstractFilteringList { public static final EndpointList EMPTY = new EndpointList(List.of()); - private final List endpoints; - - private EndpointList(List endpoints) { + private EndpointList(Collection endpoints, boolean negate) { + super(endpoints, negate, EndpointList::new); if (endpoints.stream().distinct().count() != endpoints.size()) { throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); } - this.endpoints = List.copyOf(endpoints); } - public List asList() { - return endpoints; + private EndpointList(Collection endpoints) { + this(endpoints, false); } /** Returns the main endpoint, if any */ public Optional main() { - return endpoints.stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); + return asList().stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); } /** Returns the subset of endpoints are either legacy or not */ public EndpointList legacy(boolean legacy) { - return of(endpoints.stream().filter(endpoint -> endpoint.legacy() == legacy)); + return matching(endpoint -> endpoint.legacy() == legacy); } /** Returns the subset of endpoints with given scope */ public EndpointList scope(Endpoint.Scope scope) { - return of(endpoints.stream().filter(endpoint -> endpoint.scope() == scope)); - } - - /** Returns the union of this and given endpoints */ - public EndpointList and(EndpointList endpoints) { - return of(Stream.concat(asList().stream(), endpoints.asList().stream())); + return matching(endpoint -> endpoint.scope() == scope); } public static EndpointList of(Stream endpoints) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d0f34857b06..ae7ff81c001 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -63,14 +63,6 @@ import static java.util.stream.Collectors.toList; */ public class DeploymentTrigger { - /* - * Instance orchestration TODO jonmv. - * Store new production application packages under non-instance path - * Read production packages from non-instance path, with fallback - * Deprecate and redirect some instance qualified paths in application/v4 - * Orchestrate deployment across instances. - */ - public static final Duration maxPause = Duration.ofDays(3); private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java new file mode 100644 index 00000000000..973b718a4c6 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -0,0 +1,140 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.collections.AbstractFilteringList; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.time.Instant; +import java.util.Collection; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; + +/** + * A list of deployment jobs that can be filtered in various ways. + * + * @author jonmv + */ +public class JobList extends AbstractFilteringList { + + private JobList(Collection jobs, boolean negate) { + super(jobs, negate, JobList::new); + } + + // ----------------------------------- Factories + + public static JobList from(Collection jobs) { + return new JobList(jobs, false); + } + + // ----------------------------------- Basic filters + + /** Returns the subset of jobs which are currently upgrading */ + public JobList upgrading() { + return matching(job -> job.isRunning() + && job.lastSuccess().isPresent() + && job.lastSuccess().get().versions().targetPlatform().isBefore(job.lastTriggered().get().versions().targetPlatform())); + } + + /** Returns the subset of jobs which are currently failing */ + public JobList failing() { + return matching(job -> ! job.isSuccess()); + } + + /** Returns the subset of jobs which must be failing due to an application change */ + public JobList failingApplicationChange() { + return matching(JobList::failingApplicationChange); + } + + /** Returns the subset of jobs which are failing with the given run status */ + public JobList withStatus(RunStatus status) { + return matching(job -> job.lastStatus().map(status::equals).orElse(false)); + } + + /** Returns the subset of jobs of the given type -- most useful when negated */ + public JobList type(JobType type) { + return matching(job -> job.id().type() == type); + } + + /** Returns the subset of jobs of which are production jobs */ + public JobList production() { + return matching(job -> job.id().type().isProduction()); + } + + // ----------------------------------- JobRun filtering + + /** Returns the list in a state where the next filter is for the lastTriggered run type */ + public RunFilter lastTriggered() { + return new RunFilter(JobStatus::lastTriggered); + } + + /** Returns the list in a state where the next filter is for the lastCompleted run type */ + public RunFilter lastCompleted() { + return new RunFilter(JobStatus::lastCompleted); + } + + /** Returns the list in a state where the next filter is for the lastSuccess run type */ + public RunFilter lastSuccess() { + return new RunFilter(JobStatus::lastSuccess); + } + + /** Returns the list in a state where the next filter is for the firstFailing run type */ + public RunFilter firstFailing() { + return new RunFilter(JobStatus::firstFailing); + } + + + /** Allows sub-filters for runs of the given kind */ + public class RunFilter { + + private final Function> which; + + private RunFilter(Function> which) { + this.which = which; + } + + /** Returns the subset of jobs where the run of the given type exists */ + public JobList present() { + return matching(run -> true); + } + + /** Returns the subset of jobs where the run of the given type occurred before the given instant */ + public JobList startedBefore(Instant threshold) { + return matching(run -> run.start().isBefore(threshold)); + } + + /** Returns the subset of jobs where the run of the given type occurred after the given instant */ + public JobList startedAfter(Instant threshold) { + return matching(run -> run.start().isAfter(threshold)); + } + + /** Returns the subset of jobs where the run of the given type was on the given version */ + public JobList on(ApplicationVersion version) { + return matching(run -> run.versions().targetApplication().equals(version)); + } + + /** Returns the subset of jobs where the run of the given type was on the given version */ + public JobList on(Version version) { + return matching(run -> run.versions().targetPlatform().equals(version)); + } + + /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */ + private JobList matching(Predicate condition) { + return JobList.this.matching(job -> which.apply(job).filter(condition).isPresent()); + } + + } + + // ----------------------------------- Internal helpers + + private static boolean failingApplicationChange(JobStatus job) { + if (job.isSuccess()) return false; + if (job.lastSuccess().isEmpty()) return true; // An application which never succeeded is surely bad. + if ( ! job.firstFailing().get().versions().targetPlatform().equals(job.lastSuccess().get().versions().targetPlatform())) return false; // Version change may be to blame. + return ! job.firstFailing().get().versions().targetApplication().equals(job.lastSuccess().get().versions().targetApplication()); // Return whether there is an application change. + } + +} + -- cgit v1.2.3