diff options
author | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2017-11-02 11:34:11 +0100 |
---|---|---|
committer | Jon Marius Venstad <jvenstad@yahoo-inc.com> | 2017-11-02 11:34:11 +0100 |
commit | b646be2be8b21406ec4a27abf6fdcbdcaa2e6476 (patch) | |
tree | 7e6a01ebf946258cdd3ae4e95f85d88dde2ae968 | |
parent | 07169abe734b00c55f5150e3c96e986c60d9fe89 (diff) |
Expose deploying applications, and some cleanup
12 files changed, 87 insertions, 78 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 364e91f1828..47848a618d8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -98,7 +98,8 @@ public class Application { public Map<Zone, Deployment> productionDeployments() { return deployments.values().stream() .filter(deployment -> deployment.zone().environment() == Environment.prod) - .collect(Collectors.toMap(Deployment::zone, Function.identity())); + .collect(Collectors.collectingAndThen(Collectors.toMap(Deployment::zone, Function.identity()), + ImmutableMap::copyOf)); } public DeploymentJobs deploymentJobs() { return deploymentJobs; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 12ebfa625ac..d80fdb316e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -303,7 +303,7 @@ public class ApplicationController { else version = application.currentDeployVersion(controller, zone); - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); if ( ! options.deployCurrentVersion) { @@ -314,11 +314,11 @@ public class ApplicationController { application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! canDeployDirectlyTo(zone, options) && jobType != null) { + if ( ! canDeployDirectlyTo(zone, options) && jobType.isPresent()) { // Update with (potentially) missing information about what we triggered - JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType); + JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get()); application = application.with(application.deploymentJobs() - .withTriggering(jobType, + .withTriggering(jobType.get(), application.deploying(), triggering.id(), version, @@ -417,10 +417,9 @@ public class ApplicationController { * This is needed (only) in the case where some external entity triggers a job. */ private JobStatus.JobRun getOrCreateTriggering(Application application, Version version, DeploymentJobs.JobType jobType) { - if (jobType == null) return incompleteTriggeringEvent(version); JobStatus status = application.deploymentJobs().jobStatus().get(jobType); - if (status == null) return incompleteTriggeringEvent(version); - if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); + if (status == null) return incompleteTriggeringEvent(version); + if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); return status.lastTriggered().get(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index a0d038c0e8d..8ecb00105ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import java.time.Instant; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.stream.Stream; @@ -32,7 +33,7 @@ public class ApplicationList { return new ApplicationList(applications); } - public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) { + public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) { return listOf(ids.stream().map(applications::require)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 8f654c66871..d84ca64e70a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -17,8 +17,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Information about which deployment jobs an application should run and their current status. @@ -184,18 +188,12 @@ public class DeploymentJobs { productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); private final String id; - private final Map<SystemName, Zone> zones; + private final ImmutableMap<SystemName, Zone> zones; - JobType(String id, Zone... zone) { + JobType(String id, Zone... zones) { this.id = id; - Map<SystemName, Zone> zones = new HashMap<>(); - for (Zone z : zone) { - if (zones.containsKey(z.system())) { - throw new IllegalArgumentException("A job can only map to a single zone per system"); - } - zones.put(z.system(), z); - } - this.zones = Collections.unmodifiableMap(zones); + this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(), + zone -> zone))); } public String id() { return id; } @@ -242,23 +240,23 @@ public class DeploymentJobs { } } - /** Returns the job type for the given zone, or null if none */ - public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) { + /** Returns the job type for the given zone */ + public static Optional<JobType> from(SystemName system, Zone zone) { for (JobType job : values()) { - Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system); + Optional<Zone> jobZone = job.zone(system); if (jobZone.isPresent() && jobZone.get().equals(zone)) - return job; + return Optional.of(job); } - return null; + return Optional.empty(); } /** Returns the job job type for the given environment and region or null if none */ - public static JobType from(SystemName system, Environment environment, RegionName region) { + public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { switch (environment) { - case test: return systemTest; - case staging: return stagingTest; + case test: return Optional.of(systemTest); + case staging: return Optional.of(stagingTest); } - return from(system, new com.yahoo.config.provision.Zone(environment, region)); + return from(system, new Zone(environment, region)); } private static Zone zone(SystemName system, String environment, String region) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index bb84c9e17d4..8fb01577e2c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -162,7 +162,8 @@ public class DeploymentOrder { /** Resolve job from deployment step */ private JobType toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)); + return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)) + .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone)); } /** Returns whether deployment should be postponed according to delay */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 81c3bb963db..6b60b49e1ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -40,6 +41,7 @@ public class VersionStatusSerializer { private static final String versionField = "version"; private static final String failingField = "failing"; private static final String productionField = "production"; + private static final String deployingField = "deploying"; public Slime toSlime(VersionStatus status) { Slime slime = new Slime(); @@ -74,9 +76,10 @@ public class VersionStatusSerializer { object.setString(versionField, statistics.version().toString()); applicationsToSlime(statistics.failing(), object.setArray(failingField)); applicationsToSlime(statistics.production(), object.setArray(productionField)); + applicationsToSlime(statistics.deploying(), object.setArray(deployingField)); } - private void applicationsToSlime(List<ApplicationId> applications, Cursor array) { + private void applicationsToSlime(Collection<ApplicationId> applications, Cursor array) { applications.forEach(application -> array.addString(application.serializedForm())); } @@ -105,7 +108,8 @@ public class VersionStatusSerializer { private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), applicationsFromSlime(object.field(failingField)), - applicationsFromSlime(object.field(productionField))); + applicationsFromSlime(object.field(productionField)), + applicationsFromSlime(object.field(deployingField))); } private List<ApplicationId> applicationsFromSlime(Inspector array) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index b0f2500614e..e4221a6c6c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -112,7 +112,9 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor deployingArray = versionObject.setArray("deployingApplications"); for (ApplicationId id : version.statistics().deploying()) { - + Optional<Application> application = controller.applications().get(id); + if ( ! application.isPresent()) continue; // deleted just now + toSlime(application.get(), deployingArray.addObject(), request); } } return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index b2ec8c5e269..ae7223489c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import java.util.List; +import java.util.Collection; import java.util.Objects; +import java.util.Set; /** * Statistics about deployments on a platform version. This is immutable. @@ -16,22 +17,22 @@ import java.util.Objects; public class DeploymentStatistics { private final Version version; - private final ImmutableList<ApplicationId> failing; - private final ImmutableList<ApplicationId> production; - private final ImmutableList<ApplicationId> deploying; + private final ImmutableSet<ApplicationId> failing; + private final ImmutableSet<ApplicationId> production; + private final ImmutableSet<ApplicationId> deploying; /** DO NOT USE. Public for serialization purposes */ - public DeploymentStatistics(Version version, List<ApplicationId> failingApplications, - List<ApplicationId> production, List<ApplicationId> deploying) { + public DeploymentStatistics(Version version, Collection<ApplicationId> failingApplications, + Collection<ApplicationId> production, Collection<ApplicationId> deploying) { this.version = version; - this.failing = ImmutableList.copyOf(failingApplications); - this.production = ImmutableList.copyOf(production); - this.deploying = ImmutableList.copyOf(deploying); + this.failing = ImmutableSet.copyOf(failingApplications); + this.production = ImmutableSet.copyOf(production); + this.deploying = ImmutableSet.copyOf(deploying); } /** Returns a statistics instance with the values as 0 */ public static DeploymentStatistics empty(Version version) { - return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); + return new DeploymentStatistics(version, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of()); } /** Returns the version these statistics are for */ @@ -41,13 +42,13 @@ public class DeploymentStatistics { * Returns the applications which have at least one job (of any type) which fails on this version, * excluding errors known to not be caused by this version */ - public List<ApplicationId> failing() { return failing; } + public Set<ApplicationId> failing() { return failing; } /** Returns the applications which have this version in production in at least one zone */ - public List<ApplicationId> production() { return production; } + public Set<ApplicationId> production() { return production; } /** Returns the applications which are currently upgrading to this version */ - public List<ApplicationId> deploying() { return deploying; } + public Set<ApplicationId> deploying() { return deploying; } /** Returns a version of this with the given failing application added */ public DeploymentStatistics withFailing(ApplicationId application) { @@ -64,8 +65,8 @@ public class DeploymentStatistics { return new DeploymentStatistics(version, failing, production, add(application, deploying)); } - private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) { - ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>(); + private ImmutableSet<ApplicationId> add(ApplicationId application, ImmutableSet<ApplicationId> list) { + ImmutableSet.Builder<ApplicationId> b = new ImmutableSet.Builder<>(); b.addAll(list); b.add(application); return b.build(); @@ -85,4 +86,5 @@ public class DeploymentStatistics { public int hashCode() { return Objects.hash(version, failing, production); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 07933a3c9e0..99aecab376c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -12,8 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import java.net.URI; import java.time.Instant; @@ -146,51 +144,40 @@ public class VersionStatus { versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); } - // List versions which have failing jobs, versions which are in production, and versions to which applications are deploying + // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs // Failing versions - Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream() + jobs.jobStatus().values().stream() .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade()) .filter(jobStatus -> jobStatus.jobError().isPresent()) .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version())); - for (Version v : failingJobsByVersion.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id())); - } + .map(jobStatus -> jobStatus.lastCompleted().get().version()) + .forEach(version -> versionMap.merge(version, DeploymentStatistics.empty(version), (statistics, __ ) -> statistics.withFailing(application.id()))); // Succeeding versions - Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream() + jobs.jobStatus().values().stream() .filter(jobStatus -> jobStatus.lastSuccess().isPresent()) .filter(jobStatus -> jobStatus.type().isProduction()) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version())); - for (Version v : succeedingJobsByVersions.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id())); - } + .map(jobStatus -> jobStatus.lastSuccess().get().version()) + .forEach(version -> versionMap.merge(version, DeploymentStatistics.empty(version), (statistics, __ ) -> statistics.withProduction(application.id()))); // Deploying versions - Map<Version, List<JobStatus>> deployingJobsByVersion = jobs.jobStatus().values().stream() - .filter(jobStatus -> jobStatus.isRunning(jobTimeoutLimit)) + jobs.jobStatus().values().stream() .filter(jobStatus -> jobStatus.type() != JobType.component) - .filter(jobStatus -> jobStatus.lastTriggered().isPresent()) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastTriggered().get().version())); // Plain wrong for component jobs ... - for (Version v : deployingJobsByVersion.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withDeploying(application.id())); - } + .filter(jobStatus -> jobStatus.isRunning(jobTimeoutLimit)) + .map(jobStatus -> jobStatus.lastTriggered().get().version()) + .forEach(version -> versionMap.merge(version, DeploymentStatistics.empty(version), (statistics, __ ) -> statistics.withDeploying(application.id()))); } return versionMap.values(); } - private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) { - return statistics == null ? DeploymentStatistics.empty(version) : statistics; - } - - private static VespaVersion createVersion(DeploymentStatistics statistics, + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isSystemVersion, Collection<String> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); - Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); + Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); // commitedAt ... VespaVersion.Confidence confidence; // Always compute confidence for system version if (isSystemVersion) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index f5f43265cb8..189b3a97a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -27,6 +27,8 @@ public class VersionStatusSerializerTest { Version.fromString("5.0"), Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 00bd1ed8208..c3968d53f85 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -17,7 +17,8 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default" } - ] + ], + "deployingApplications": [] }, { "version": "(ignore)", @@ -45,6 +46,15 @@ "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", "upgradePolicy": "default" } + ], + "deployingApplications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default" + } ] }, { @@ -63,7 +73,8 @@ } ], "failingApplications": [], - "productionApplications": [] + "productionApplications": [], + "deployingApplications": [] }, { "version": "(ignore)", @@ -74,7 +85,8 @@ "systemVersion": false, "configServers": [], "failingApplications": [], - "productionApplications": [] + "productionApplications": [], + "deployingApplications": [] } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index bc31079cfe0..7d1d507807a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -235,7 +235,7 @@ public class VersionStatusTest { @Test - public void testIgnoreConfigdeince() { + public void testIgnoreConfidence() { DeploymentTester tester = new DeploymentTester(); Version version0 = new Version("5.0"); |