From 0e7c81fc39363e5091b0a6f8300f578fd5c5bf39 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 6 Nov 2017 15:35:29 +0100 Subject: Updated response to show failing, running and # of # production jobs --- .../controller/application/DeploymentJobs.java | 16 ++--- .../hosted/controller/application/JobList.java | 6 +- .../restapi/deployment/DeploymentApiHandler.java | 72 +++++++++++++--------- .../hosted/controller/versions/VersionStatus.java | 7 +-- .../restapi/deployment/responses/root.json | 10 +-- 5 files changed, 61 insertions(+), 50 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index d84ca64e70a..f94b0f7284b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -13,14 +13,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import java.time.Instant; import java.util.Collection; -import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -152,13 +150,11 @@ public class DeploymentJobs { /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */ public Instant failingSince() { - Instant failingSince = null; - for (JobStatus jobStatus : jobStatus().values()) { - if (jobStatus.isSuccess()) continue; - if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at())) - failingSince = jobStatus.firstFailing().get().at(); - } - return failingSince; + return JobList.from(jobStatus().values()) + .failing() + .mapToList(job -> job.firstFailing().get().at()) + .stream() + .min(Comparator.naturalOrder()).orElse(null); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index 1a8de949ae9..19c7bbe98b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -12,6 +12,8 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; + /** * A list of deployment jobs that can be filtered in various ways. * @@ -47,7 +49,7 @@ public class JobList { public List asList() { return list; } /** Returns the jobstatuses in this as an immutable list after mapping with the given function */ - public List asList(Function mapper) { + public List mapToList(Function mapper) { return ImmutableList.copyOf(list.stream().map(mapper)::iterator); } @@ -57,7 +59,7 @@ public class JobList { public int size() { return list.size(); } - // ----------------------------------- Filters + // ----------------------------------- Basic filters /** Negates the next filter operation */ public JobList not() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index b10bbcd8da0..bcc72245f0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -12,7 +12,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -99,7 +99,9 @@ public class DeploymentApiHandler extends LoggingRequestHandler { for (ApplicationId id : version.statistics().failing()) { controller.applications().get(id).ifPresent(application -> { firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> { - toSlime(failingArray.addObject(), application, firstFailing.type(), request); + Cursor applicationObject = failingArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("failing", firstFailing.type().id()); }); }); } @@ -107,17 +109,22 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor productionArray = versionObject.setArray("productionApplications"); for (ApplicationId id : version.statistics().production()) { controller.applications().get(id).ifPresent(application -> { - lastProductionOn(version.versionNumber(), application).ifPresent(lastProduction -> { - toSlime(productionArray.addObject(), application, lastProduction.type(), request); - }); + int successes = productionSuccessesFor(version.versionNumber(), application); + if (successes == 0) return; // Just upgraded to a newer version. + Cursor applicationObject = productionArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setLong("productionJobs", productionJobsFor(application)); + applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application)); }); } - Cursor deployingArray = versionObject.setArray("deployingApplications"); + Cursor runningArray = versionObject.setArray("deployingApplications"); for (ApplicationId id : version.statistics().deploying()) { controller.applications().get(id).ifPresent(application -> { lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> { - toSlime(deployingArray.addObject(), application, lastDeploying.type(), request); + Cursor applicationObject = runningArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("running", lastDeploying.type().id()); }); }); } @@ -125,7 +132,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Cursor object, Application application, JobType jobType, HttpRequest request) { + private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); @@ -134,7 +141,6 @@ public class DeploymentApiHandler extends LoggingRequestHandler { "/application/" + application.id().application().value()).toString()); object.setString("upgradePolicy", toString(application.deploymentSpec().upgradePolicy())); - object.setString("jobType", jobType.id()); } private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) { @@ -146,33 +152,39 @@ public class DeploymentApiHandler extends LoggingRequestHandler { // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus - /** The first upgrade job to fail for this version x application */ + /** The first upgrade job to fail on this version, for this application */ private Optional firstFailingOn(Version version, Application application) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) - .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade()) - .filter(jobStatus -> jobStatus.jobError().isPresent()) - .filter(jobStatus -> jobStatus.jobError().get() != outOfCapacity) - .filter(jobStatus -> jobStatus.lastCompleted().get().version().equals(version)) - .min(comparing(jobStatus -> jobStatus.lastCompleted().get().at())); + return JobList.from(application) + .failing() + .not().failingApplicationChange() + .not().failingBecause(outOfCapacity) + .lastCompleted().on(version) + .asList().stream() + .min(comparing(job -> job.lastCompleted().get().at())); + } + + /** The number of production jobs for this application */ + private int productionJobsFor(Application application) { + return JobList.from(application) + .production() + .size(); } - /** The last production job to succeed for this version x application */ - private Optional lastProductionOn(Version version, Application application) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastSuccess().isPresent()) - .filter(jobStatus -> jobStatus.type().isProduction()) - .filter(jobStatus -> jobStatus.lastSuccess().get().version().equals(version)) - .max(comparing(jobStatus -> jobStatus.lastSuccess().get().at())); + /** The number of production jobs with last success on the given version, for this application */ + private int productionSuccessesFor(Version version, Application application) { + return JobList.from(application) + .production() + .lastSuccess().on(version) + .size(); } - /** The last deployment triggered for this version x application */ + /** The last triggered upgrade to this version, for this application */ private Optional lastDeployingTo(Version version, Application application) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(jobStatus -> jobStatus.isRunning(controller.applications().deploymentTrigger().jobTimeoutLimit())) - .filter(jobStatus -> jobStatus.lastTriggered().get().upgrade()) - .filter(jobStatus -> jobStatus.lastTriggered().get().version().equals(version)) - .max(comparing(jobStatus -> jobStatus.lastTriggered().get().at())); + return JobList.from(application) + .running(controller.applications().deploymentTrigger().jobTimeoutLimit()) + .lastTriggered().upgrade() + .asList().stream() + .max(comparing(job -> job.lastTriggered().get().at())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index fbe57b4becf..3de4a7539f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobList; import java.net.URI; @@ -154,21 +153,21 @@ public class VersionStatus { .failing() .not().failingApplicationChange() .not().failingBecause(outOfCapacity) - .asList(job -> job.lastCompleted().get().version()) + .mapToList(job -> job.lastCompleted().get().version()) .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withFailing(application.id()))); // Succeeding versions JobList.from(application) .lastSuccess().present() .production() - .asList(job -> job.lastSuccess().get().version()) + .mapToList(job -> job.lastSuccess().get().version()) .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withProduction(application.id()))); // Deploying versions JobList.from(application) .running(jobTimeoutLimit) .lastTriggered().upgrade() - .asList(job -> job.lastTriggered().get().version()) + .mapToList(job -> job.lastTriggered().get().version()) .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withDeploying(application.id()))); } return versionMap.values(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 2c4fc7cb5f1..5f7fedfd75f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -16,7 +16,8 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", - "jobType": "production-corp-us-east-1" + "productionJobs": 1, + "productionSuccesses": 1 } ], "deployingApplications": [ ] @@ -43,7 +44,7 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", - "jobType": "staging-test" + "failing": "staging-test" } ], "productionApplications":[ @@ -53,7 +54,8 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", "upgradePolicy": "default", - "jobType": "production-corp-us-east-1" + "productionJobs": 1, + "productionSuccesses": 1 } ], "deployingApplications": [ @@ -63,7 +65,7 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", - "jobType": "staging-test" + "running": "staging-test" } ] }, -- cgit v1.2.3