From 902b5d840afaada05e57ab3faf7f619d3aab9455 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 27 Nov 2019 15:28:55 +0100 Subject: More new JobList and co --- .../deployment/DeploymentStatusList.java | 5 + .../hosted/controller/deployment/JobList.java | 6 ++ .../restapi/deployment/DeploymentApiHandler.java | 108 ++++++++++----------- .../hosted/controller/versions/VersionStatus.java | 76 +++++++-------- 4 files changed, 96 insertions(+), 99 deletions(-) (limited to 'controller-server/src') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index 779beab60d1..93dc5db206a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -27,6 +27,11 @@ public class DeploymentStatusList extends AbstractFilteringList status.application().productionDeployments().values().stream() + .anyMatch(deployments -> ! deployments.isEmpty())); + } + public DeploymentStatusList failing() { return matching(DeploymentStatus::hasFailures); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index f771e9314eb..92bf1119020 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -126,6 +127,11 @@ public class JobList extends AbstractFilteringList { return present().mapToList(which.andThen(Optional::get).andThen(mapper)); } + /** Returns the runs of the given kind. */ + public List asList() { + return mapToList(Function.identity()); + } + /** Returns the subset of jobs where the run of the given type occurred before the given instant */ public JobList endedBefore(Instant threshold) { return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 46f4b2816c1..e33f6f0963b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -11,24 +11,26 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.JobList; -import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.restapi.Uri; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.JobList; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Instant; import java.util.Comparator; +import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; import static java.util.Comparator.comparing; /** @@ -98,53 +100,55 @@ public class DeploymentApiHandler extends LoggingRequestHandler { configServerObject.setString("hostname", hostname.value()); } + Map jobs = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())) + .asList().stream() + .flatMap(status -> status.instanceJobs().entrySet().stream()) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); Cursor failingArray = versionObject.setArray("failingApplications"); for (ApplicationId id : version.statistics().failing()) { - controller.applications().getInstance(id).ifPresent(application -> { - firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> { + if (jobs.containsKey(id)) + firstFailingOn(version.versionNumber(), jobs.get(id)).ifPresent(firstFailing -> { Cursor applicationObject = failingArray.addObject(); - toSlime(applicationObject, application, request); - applicationObject.setString("failing", firstFailing.type().jobName()); + toSlime(applicationObject, id, request); + applicationObject.setString("failing", firstFailing.id().type().jobName()); }); - }); } Cursor productionArray = versionObject.setArray("productionApplications"); for (ApplicationId id : version.statistics().production()) { - controller.applications().getInstance(id).ifPresent(application -> { - int successes = productionSuccessesFor(version.versionNumber(), application); - if (successes == 0) return; // Just upgraded to a newer version. + if (jobs.containsKey(id)) { + int successes = productionSuccessesFor(version.versionNumber(), jobs.get(id)); + if (successes == 0) continue; // Just upgraded to a newer version. Cursor applicationObject = productionArray.addObject(); - toSlime(applicationObject, application, request); - applicationObject.setLong("productionJobs", productionJobsFor(application)); - applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application)); - }); + toSlime(applicationObject, id, request); + applicationObject.setLong("productionJobs", jobs.get(id).production().size()); + applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), jobs.get(id))); + } } Cursor runningArray = versionObject.setArray("deployingApplications"); for (ApplicationId id : version.statistics().deploying()) { - controller.applications().getInstance(id).ifPresent(application -> { - lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> { + if (jobs.containsKey(id)) + lastDeployingTo(version.versionNumber(), jobs.get(id)).ifPresent(lastDeploying -> { Cursor applicationObject = runningArray.addObject(); - toSlime(applicationObject, application, request); - applicationObject.setString("running", lastDeploying.type().jobName()); + toSlime(applicationObject, id, request); + applicationObject.setString("running", lastDeploying.id().type().jobName()); }); - }); } } return new SlimeJsonResponse(slime); } - private void toSlime(Cursor object, Instance instance, HttpRequest request) { - object.setString("tenant", instance.id().tenant().value()); - object.setString("application", instance.id().application().value()); - object.setString("instance", instance.id().instance().value()); + private void toSlime(Cursor object, ApplicationId id, HttpRequest request) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("instance", id.instance().value()); object.setString("url", new Uri(request.getUri()).withPath("/application/v4/tenant/" + - instance.id().tenant().value() + + id.tenant().value() + "/application/" + - instance.id().application().value()).toString()); - object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(instance.id())) - .deploymentSpec().requireInstance(instance.name()).upgradePolicy())); + id.application().value()).toString()); + object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(id)) + .deploymentSpec().requireInstance(id.instance()).upgradePolicy())); } private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) { @@ -157,40 +161,30 @@ public class DeploymentApiHandler extends LoggingRequestHandler { // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus /** The first upgrade job to fail on this version, for this application */ - private Optional firstFailingOn(Version version, Instance instance) { - return JobList.from(instance) - .failing() - .not().failingApplicationChange() - .not().failingBecause(outOfCapacity) - .lastCompleted().on(version) - .asList().stream() - .min(Comparator.comparing(job -> job.lastCompleted().get().at()) - .thenComparing(job -> job.type())); - } - - /** The number of production jobs for this application */ - private int productionJobsFor(Instance instance) { - return JobList.from(instance) - .production() - .size(); + private Optional firstFailingOn(Version version, JobList jobs) { + return jobs.failing() + .not().failingApplicationChange() + .not().withStatus(RunStatus.outOfCapacity) + .lastCompleted().on(version) + .lastCompleted().asList().stream() + .min(Comparator.comparing(run -> run.start()) + .thenComparing(run -> run.id().type())); } /** The number of production jobs with last success on the given version, for this application */ - private int productionSuccessesFor(Version version, Instance instance) { - return JobList.from(instance) - .production() - .lastSuccess().on(version) - .size(); + private int productionSuccessesFor(Version version, JobList jobs) { + return jobs.production() + .lastSuccess().on(version) + .size(); } /** The last triggered upgrade to this version, for this application */ - private Optional lastDeployingTo(Version version, Instance instance) { - return JobList.from(instance) - .upgrading() - .lastTriggered().on(version) - .asList().stream() - .max(Comparator.comparing(job -> job.lastCompleted().get().at()) - .thenComparing(job -> job.type())); + private Optional lastDeployingTo(Version version, JobList jobs) { + return jobs.upgrading() + .lastTriggered().on(version) + .lastTriggered().asList().stream() + .max(Comparator.comparing(run -> run.start()) + .thenComparing(run -> run.id().type())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 97506d3f6ea..ee6cd35b90a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -7,13 +7,16 @@ import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; +import com.yahoo.vespa.hosted.controller.deployment.JobList; +import com.yahoo.vespa.hosted.controller.deployment.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; import java.util.ArrayList; @@ -29,8 +32,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; - /** * Information about the current platform versions in use. * The versions in use are the set of all versions running in current applications, versions @@ -118,8 +119,9 @@ public class VersionStatus { systemVersion = newSystemVersion; } - Collection deploymentStatistics = computeDeploymentStatistics(infrastructureVersions.keySet(), - controller.applications().asList()); + + var deploymentStatistics = computeDeploymentStatistics(infrastructureVersions.keySet(), + controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()))); List versions = new ArrayList<>(); List releasedVersions = controller.mavenRepository().metadata().versions(); @@ -187,50 +189,40 @@ public class VersionStatus { } private static Collection computeDeploymentStatistics(Set infrastructureVersions, - List instances) { + DeploymentStatusList statuses) { Map versionMap = new HashMap<>(); for (Version infrastructureVersion : infrastructureVersions) { versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion)); } - for (Application application : ApplicationList.from(instances).withProductionDeployment().asList()) - for (Instance instance : application.instances().values()) { - // Note that each version deployed on this application in production exists - // (ignore non-production versions) - for (Deployment deployment : instance.productionDeployments().values()) { + for (DeploymentStatus status : statuses.withProductionDeployment().asList()) { + for (Instance instance : status.application().instances().values()) + for (Deployment deployment : instance.productionDeployments().values()) versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); - } - // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs - - // Failing versions - JobList.from(instance) - .failing() - .not().failingApplicationChange() - .not().failingBecause(outOfCapacity) - .mapToList(job -> job.lastCompleted().get().platform()) - .forEach(version -> versionMap - .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) - .withFailing(instance.id()))); - - // Succeeding versions - JobList.from(instance) - .lastSuccess().present() - .production() - .mapToList(job -> job.lastSuccess().get().platform()) - .forEach(version -> versionMap - .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) - .withProduction(instance.id()))); - - // Deploying versions - JobList.from(instance) - .upgrading() - .mapToList(job -> job.lastTriggered().get().platform()) - .forEach(version -> versionMap - .put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) - .withDeploying(instance.id()))); - } + status.instanceJobs().forEach((id, jobs) -> { + jobs.failing() + .not().failingApplicationChange() + .not().withStatus(RunStatus.outOfCapacity) + .lastCompleted().mapToList(run -> run.versions().targetPlatform()) + .forEach(version -> versionMap.put(version, + versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) + .withFailing(id))); + + jobs.production() + .lastSuccess().mapToList(run -> run.versions().targetPlatform()) + .forEach(version -> versionMap.put(version, + versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) + .withProduction(id))); + + jobs.upgrading() + .lastTriggered().mapToList(run -> run.versions().targetPlatform()) + .forEach(version -> versionMap.put(version, + versionMap.getOrDefault(version, DeploymentStatistics.empty(version)) + .withDeploying(id))); + }); + } return versionMap.values(); } -- cgit v1.2.3