diff options
| author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2018-04-23 20:59:27 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-04-23 20:59:27 +0200 |
| commit | 60b66c70d7fd7c7f17bb88d6e1fef06d63ea0910 (patch) | |
| tree | 998de20039a87d09ab0513894d4854d85211b88d | |
| parent | 25753e098fc7b8ad4dab12d344ac9b4e276f5d2a (diff) | |
| parent | ac66027dc8f884b8dc85676c53c61369db9488dc (diff) | |
Merge pull request #5667 from vespa-engine/jvenstad/DO-unified
Jvenstad/test every version combination before deploying
19 files changed, 450 insertions, 278 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java index d853443bf5d..937df497133 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; -import java.util.Objects; +import com.yahoo.config.provision.ApplicationId; /** * @author jvenstad @@ -21,37 +21,47 @@ public interface BuildService { boolean isRunning(BuildJob buildJob); - // TODO jvenstad: Implement with DeploymentTrigger.Job class BuildJob { + private final ApplicationId applicationId; private final long projectId; private final String jobName; - public BuildJob(long projectId, String jobName) { + protected BuildJob(ApplicationId applicationId, long projectId, String jobName) { + this.applicationId = applicationId; this.projectId = projectId; this.jobName = jobName; } + public static BuildJob of(ApplicationId applicationId, long projectId, String jobName) { + return new BuildJob(applicationId, projectId, jobName); + } + + public ApplicationId applicationId() { return applicationId; } public long projectId() { return projectId; } public String jobName() { return jobName; } @Override - public boolean equals(Object o) { + public final boolean equals(Object o) { if (this == o) return true; if ( ! (o instanceof BuildJob)) return false; + BuildJob buildJob = (BuildJob) o; - return projectId == buildJob.projectId && - Objects.equals(jobName, buildJob.jobName); + + if (projectId != buildJob.projectId) return false; + return jobName.equals(buildJob.jobName); } @Override - public String toString() { - return jobName + "@" + projectId; + public final int hashCode() { + int result = (int) (projectId ^ (projectId >>> 32)); + result = 31 * result + jobName.hashCode(); + return result; } @Override - public int hashCode() { - return Objects.hash(projectId, jobName); + public String toString() { + return jobName + " for " + applicationId + " with project " + projectId; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java index b942a6baa98..9219619dc9e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockBuildService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import java.util.ArrayList; @@ -36,8 +37,8 @@ public class MockBuildService extends AbstractComponent implements BuildService } /** Removes the given job for the given project and returns whether it was found. */ - public boolean removeJob(long projectId, String jobType) { - return jobs.remove(new BuildJob(projectId, jobType)); + public boolean remove(BuildJob buildJob) { + return jobs.remove(buildJob); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0749e49729f..84d821b9f46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -37,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -283,7 +284,7 @@ public class ApplicationController { } else { JobType jobType = JobType.from(controller.system(), zone) .orElseThrow(() -> new IllegalArgumentException("No job found for zone " + zone)); - Optional<JobStatus.JobRun> triggered = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) + Optional<JobRun> triggered = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) .flatMap(JobStatus::lastTriggered); // TODO jvenstad: Verify this response with a test, and see that it sorts itself when triggered. if ( ! triggered.isPresent()) @@ -295,6 +296,7 @@ public class ApplicationController { ? application.oldestDeployedApplication().orElse(triggered.get().applicationVersion()) : triggered.get().applicationVersion(); applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), applicationVersion.id())); + validateRun(application, zone, platformVersion, applicationVersion); } validate(applicationPackage.deploymentSpec()); @@ -315,12 +317,6 @@ public class ApplicationController { store(application); // store missing information even if we fail deployment below } - // TODO jvenstad: Use code from DeploymentTrigger? Also, validate application version. - // Validate the change being deployed - if ( ! canDeployDirectly) { - validateChange(application, zone, platformVersion); - } - // Assign global rotation application = withRotation(application, zone); Set<String> rotationNames = new HashSet<>(); @@ -601,26 +597,20 @@ public class ApplicationController { .forEach(zone -> { if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), zone.region().orElse(null)))) { - throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in " + - "this system!"); + throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); } }); } - /** Verify that what we want to deploy is tested and that we aren't downgrading */ - private void validateChange(Application application, ZoneId zone, Version version) { - if ( ! application.deploymentJobs().isDeployableTo(zone.environment(), application.change())) { - throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as " + application.change() + " is not tested"); - } - // TODO jvenstad: Rewrite to use decided versions. Simplifies the below. - Deployment existingDeployment = application.deployments().get(zone); - if (zone.environment().isProduction() && existingDeployment != null && - existingDeployment.version().isAfter(version)) { - throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as the requested version " + version + " is older than" + - " the current version " + existingDeployment.version()); - } + /** Verify that we don't downgrade an existing production deployment. */ + private void validateRun(Application application, ZoneId zone, Version platformVersion, ApplicationVersion applicationVersion) { + Deployment deployment = application.deployments().get(zone); + if ( zone.environment().isProduction() && deployment != null + && ( platformVersion.compareTo(deployment.version()) < 0 + || applicationVersion.compareTo(deployment.applicationVersion()) < 0)) + throw new IllegalArgumentException(String.format("Rejecting deployment of %s to %s, as the requested versions (platform: %s, application: %s)" + + " are older than the currently deployed (platform: %s, application: %s).", + application, zone, platformVersion, applicationVersion, deployment.version(), deployment.applicationVersion())); } public RotationRepository rotationRepository() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 711853d2f9c..6891477d142 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -2,11 +2,13 @@ package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -97,20 +99,6 @@ public class DeploymentJobs { return ! JobList.from(status.values()).failing().isEmpty(); } - /** Returns whether change can be deployed to the given environment */ - public boolean isDeployableTo(Environment environment, Change change) { - // TODO jvenstad: Rewrite to verify versions when deployment is already decided. - if (environment == null || ! change.isPresent()) { - return true; - } - if (environment == Environment.staging) { - return successAt(change, JobType.systemTest).isPresent(); - } else if (environment == Environment.prod) { - return successAt(change, JobType.stagingTest).isPresent(); - } - return true; // other environments do not have any preconditions - } - /** Returns the JobStatus of the given JobType, or empty. */ public Optional<JobStatus> statusOf(JobType jobType) { return Optional.ofNullable(jobStatus().get(jobType)); @@ -125,14 +113,6 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } - /** Returns the time of success for the given change for the given job type, or empty if unsuccessful. */ - public Optional<Instant> successAt(Change change, JobType jobType) { - return statusOf(jobType) - .flatMap(JobStatus::lastSuccess) - .filter(status -> status.lastCompletedWas(change)) - .map(JobStatus.JobRun::at); - } - private static OptionalLong requireId(OptionalLong id, String message) { Objects.requireNonNull(id, message); if ( ! id.isPresent()) { @@ -262,6 +242,7 @@ public class DeploymentJobs { public boolean success() { return ! jobError.isPresent(); } public Optional<SourceRevision> sourceRevision() { return sourceRevision; } public Optional<JobError> jobError() { return jobError; } + public BuildService.BuildJob buildJob() { return BuildService.BuildJob.of(applicationId, projectId, jobType.jobName()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index bb90370f6c0..27992d43eba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -146,6 +146,11 @@ public class JobList { } /** Returns the subset of jobs where the run of the given type was on the given version */ + public JobList on(ApplicationVersion version) { + return filter(run -> run.applicationVersion().equals(version)); + } + + /** Returns the subset of jobs where the run of the given type was on the given version */ public JobList on(Version version) { return filter(run -> run.version().equals(version)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index 697ef7dbd4e..e3ac4cb49f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -8,6 +8,8 @@ import java.time.Instant; import java.util.Objects; import java.util.Optional; +import static java.util.Objects.requireNonNull; + /** * The last known build status of a particular deployment job for a particular application. * This is immutable. @@ -33,12 +35,12 @@ public class JobStatus { public JobStatus(DeploymentJobs.JobType type, Optional<DeploymentJobs.JobError> jobError, Optional<JobRun> lastTriggered, Optional<JobRun> lastCompleted, Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess) { - Objects.requireNonNull(type, "jobType cannot be null"); - Objects.requireNonNull(jobError, "jobError cannot be null"); - Objects.requireNonNull(lastTriggered, "lastTriggered cannot be null"); - Objects.requireNonNull(lastCompleted, "lastCompleted cannot be null"); - Objects.requireNonNull(firstFailing, "firstFailing cannot be null"); - Objects.requireNonNull(lastSuccess, "lastSuccess cannot be null"); + requireNonNull(type, "jobType cannot be null"); + requireNonNull(jobError, "jobError cannot be null"); + requireNonNull(lastTriggered, "lastTriggered cannot be null"); + requireNonNull(lastCompleted, "lastCompleted cannot be null"); + requireNonNull(firstFailing, "firstFailing cannot be null"); + requireNonNull(lastSuccess, "lastSuccess cannot be null"); this.type = type; this.jobError = jobError; @@ -74,7 +76,7 @@ public class JobStatus { Version version; String reason; if (type == DeploymentJobs.JobType.component) { // not triggered by us - version = controller.systemVersion(); // TODO jvenstad: Get rid of this, and perhaps all of component info? + version = controller.systemVersion(); reason = "Application commit"; } else if ( ! lastTriggered.isPresent()) { throw new IllegalStateException("Got notified about completion of " + this + @@ -160,16 +162,12 @@ public class JobStatus { private final String reason; private final Instant at; - public JobRun(long id, Version version, ApplicationVersion applicationVersion,String reason, Instant at) { - Objects.requireNonNull(version, "version cannot be null"); - Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); - Objects.requireNonNull(reason, "Reason cannot be null"); - Objects.requireNonNull(at, "at cannot be null"); + public JobRun(long id, Version version, ApplicationVersion applicationVersion, String reason, Instant at) { this.id = id; - this.version = version; - this.applicationVersion = applicationVersion; - this.reason = reason; - this.at = at; + this.version = requireNonNull(version); + this.applicationVersion = requireNonNull(applicationVersion); + this.reason = requireNonNull(reason); + this.at = requireNonNull(at); } /** Returns the id of this run of this job, or -1 if not known */ @@ -187,32 +185,32 @@ public class JobStatus { /** Returns the time if this triggering or completion */ public Instant at() { return at; } - /** Returns whether the job last completed for the given change */ - public boolean lastCompletedWas(Change change) { - if (change.platform().isPresent() && ! change.platform().get().equals(version())) return false; - if (change.application().isPresent() && ! change.application().get().equals(applicationVersion)) return false; - return true; - } - @Override - public int hashCode() { - return Objects.hash(version, applicationVersion, at); + public String toString() { + return "job run " + id + " of version " + version + " " + applicationVersion + " at " + at; } @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof JobRun)) return false; - JobRun jobRun = (JobRun) o; - return id == jobRun.id && - Objects.equals(version, jobRun.version) && - Objects.equals(applicationVersion, jobRun.applicationVersion) && - Objects.equals(at, jobRun.at); + if (!(o instanceof JobRun)) return false; + + JobRun run = (JobRun) o; + + if (id != run.id) return false; + if (!version.equals(run.version)) return false; + if (!applicationVersion.equals(run.applicationVersion)) return false; + return at.equals(run.at); } @Override - public String toString() { return "job run " + id + " of version " + version + " " - + applicationVersion + " at " + at; } + public int hashCode() { + int result = (int) (id ^ (id >>> 32)); + result = 31 * result + version.hashCode(); + result = 31 * result + applicationVersion.hashCode(); + result = 31 * result + at.hashCode(); + return result; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index b8f49ab9ca7..b11b432c27f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.DeploymentSpec.Step; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -18,7 +18,9 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -28,6 +30,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; @@ -35,13 +38,25 @@ import java.util.OptionalLong; import java.util.Set; import java.util.function.Supplier; import java.util.logging.Logger; -import java.util.stream.Collectors; - +import java.util.stream.Stream; + +import static com.yahoo.config.provision.Environment.prod; +import static com.yahoo.config.provision.Environment.staging; +import static com.yahoo.config.provision.Environment.test; +import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; +import static java.util.Optional.empty; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.partitioningBy; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; /** @@ -81,7 +96,7 @@ public class DeploymentTrigger { * Called each time a job completes (successfully or not) to record information used when deciding what to trigger. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.INFO, String.format("Got notified of %s for %s of %s (%d).", + log.log(LogLevel.DEBUG, String.format("Got notified of %s for %s of %s (%d).", report.jobError().map(JobError::toString).orElse("success"), report.jobType(), report.applicationId(), @@ -98,7 +113,7 @@ public class DeploymentTrigger { application = application.withJobCompletion(report, applicationVersion, clock.instant(), controller); application = application.withProjectId(OptionalLong.of(report.projectId())); - if (report.jobType() == JobType.component && report.success()) { + if (report.jobType() == component && report.success()) { if (acceptNewApplicationVersion(application)) // Note that in case of an ongoing upgrade this may result in both the upgrade and application // change being deployed together @@ -116,8 +131,7 @@ public class DeploymentTrigger { * Only one job is triggered each run for test jobs, since their environments have limited capacity. */ public long triggerReadyJobs() { - return computeReadyJobs().stream() - .collect(partitioningBy(job -> job.jobType().isTest())) + return computeReadyJobs().collect(partitioningBy(job -> job.jobType().isTest())) .entrySet().stream() .flatMap(entry -> (entry.getKey() // True for capacity constrained zones -- sort by priority and make a task for each job type. @@ -129,7 +143,7 @@ public class DeploymentTrigger { .collect(groupingBy(Job::jobType)) // False for production jobs -- keep step order and make a task for each application. : entry.getValue().stream() - .collect(groupingBy(Job::id))) + .collect(groupingBy(Job::applicationId))) .values().stream() .map(jobs -> (Supplier<Long>) jobs.stream() .filter(job -> canTrigger(job) && trigger(job)) @@ -144,18 +158,19 @@ public class DeploymentTrigger { * the project id is removed from the application owning the job, to prevent further trigger attemps. */ public boolean trigger(Job job) { - log.log(LogLevel.INFO, String.format("Attempting to trigger %s for %s, deploying %s: %s (platform: %s, application: %s)", job.jobType, job.id, job.change, job.reason, job.platformVersion, job.applicationVersion.id())); + log.log(LogLevel.INFO, String.format("Attempting to trigger %s: %s (%s)", job, job.reason, job.target)); try { - buildService.trigger(new BuildService.BuildJob(job.projectId, job.jobType.jobName())); - applications().lockOrThrow(job.id, application -> applications().store(application.withJobTriggering( - job.jobType, new JobStatus.JobRun(-1, job.platformVersion, job.applicationVersion, job.reason, clock.instant())))); + buildService.trigger(job); + applications().lockOrThrow(job.applicationId(), application -> applications().store(application.withJobTriggering( + job.jobType, new JobRun(-1, job.target.targetPlatform, job.target.targetApplication, job.reason, clock.instant())))); return true; } catch (RuntimeException e) { - log.log(LogLevel.WARNING, String.format("Exception triggering %s for %s (%s): %s", job.jobType, job.id, job.projectId, e)); + log.log(LogLevel.WARNING, "Exception triggering " + job + ": " + e); if (e instanceof NoSuchElementException || e instanceof IllegalArgumentException) - applications().lockOrThrow(job.id, application -> applications().store(application.withProjectId(OptionalLong.empty()))); + applications().lockOrThrow(job.applicationId(), application -> + applications().store(application.withProjectId(OptionalLong.empty()))); return false; } } @@ -189,52 +204,67 @@ public class DeploymentTrigger { }); } + public Map<JobType, ? extends List<? extends BuildJob>> jobsToRun() { + return computeReadyJobs().collect(groupingBy(Job::jobType)); + } + /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ - public List<Job> computeReadyJobs() { + private Stream<Job> computeReadyJobs() { return ApplicationList.from(applications().asList()) .notPullRequest() .withProjectId() .deploying() .idList().stream() .map(this::computeReadyJobs) - .flatMap(List::stream) - .collect(Collectors.toList()); + .flatMap(List::stream); } /** Returns whether the given job is currently running; false if completed since last triggered, asking the build service othewise. */ public boolean isRunning(Application application, JobType jobType) { return ! application.deploymentJobs().statusOf(jobType) .flatMap(job -> job.lastCompleted().map(run -> run.at().isAfter(job.lastTriggered().get().at()))).orElse(false) - && buildService.isRunning(new BuildService.BuildJob(application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); + && buildService.isRunning(BuildJob.of(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); } - public Job forcedDeploymentJob(Application application, JobType jobType, String reason) { - return deploymentJob(application, jobType, reason, clock.instant(), Collections.emptySet()); + public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType) { + Application application = applications().require(applicationId); + if (jobType == component) { + buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); + return singletonList(component); + } + State target = targetFor(application, application.change(), deploymentFor(application, jobType)); + String reason = ">:o:< Triggered by force! (-o-) |-o-| (=oo=)"; + if (isVerified(application, target, jobType)) { + trigger(deploymentJob(application, target, application.change(), jobType, reason, clock.instant(), Collections.emptySet())); + return singletonList(jobType); + } + List<Job> testJobs = testJobsFor(application, target, reason, clock.instant()); + testJobs.forEach(this::trigger); + return testJobs.stream().map(Job::jobType).collect(toList()); } - private Job deploymentJob(Application application, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith) { + private Job deploymentJob(Application application, State target, Change change, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith) { boolean isRetry = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::jobError) .filter(JobError.outOfCapacity::equals).isPresent(); if (isRetry) reason += "; retrying on out of capacity"; - Change change = application.change(); - // For both versions, use the newer of the change's and the currently deployed versions, or a fallback if none of these exist. - Version platform = jobType == JobType.component - ? Version.emptyVersion - : deploymentFor(application, jobType).map(Deployment::version) - .filter(version -> ! change.upgrades(version)) - .orElse(change.platform() - .orElse(application.oldestDeployedPlatform() - .orElse(controller.systemVersion()))); - ApplicationVersion applicationVersion = jobType == JobType.component - ? ApplicationVersion.unknown - : deploymentFor(application, jobType).map(Deployment::applicationVersion) - .filter(version -> ! change.upgrades(version)) - .orElse(change.application() - .orElseGet(() -> application.oldestDeployedApplication() - .orElseThrow(() -> new IllegalArgumentException("Cannot determine application version to use for " + jobType)))); - - return new Job(application, jobType, reason, availableSince, concurrentlyWith, isRetry, change, platform, applicationVersion); + return new Job(application, target, jobType, reason, availableSince, concurrentlyWith, isRetry, change.application().isPresent()); + } + + private Version targetPlatform(Application application, Change change, Optional<Deployment> deployment) { + return max(deployment.map(Deployment::version), change.platform()) + .orElse(application.oldestDeployedPlatform() + .orElse(controller.systemVersion())); + } + + private ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { + return max(deployment.map(Deployment::applicationVersion), change.application()) + .orElse(application.oldestDeployedApplication() + .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().applicationVersion())); + } + + private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { + return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; } /** @@ -243,41 +273,99 @@ public class DeploymentTrigger { private List<Job> computeReadyJobs(ApplicationId id) { List<Job> jobs = new ArrayList<>(); applications().get(id).ifPresent(application -> { - List<DeploymentSpec.Step> steps = application.deploymentSpec().steps().isEmpty() - ? Collections.singletonList(new DeploymentSpec.DeclaredZone(Environment.test)) + List<Step> steps = application.deploymentSpec().steps().isEmpty() + ? singletonList(new DeploymentSpec.DeclaredZone(test)) : application.deploymentSpec().steps(); + List<Step> productionSteps = steps.stream().filter(step -> step.deploysTo(prod) || step.zones().isEmpty()).collect(toList()); - Optional<Instant> completedAt = Optional.of(clock.instant()); - String reason = "Deploying " + application.change(); + Optional<Instant> completedAt = application.deploymentJobs().statusOf(stagingTest) + .flatMap(JobStatus::lastSuccess).map(JobRun::at); + String reason = "New change available"; + List<Job> testJobs = null; - for (DeploymentSpec.Step step : steps) { + for (Step step : productionSteps) { Set<JobType> stepJobs = step.zones().stream().map(order::toJob).collect(toSet()); - Set<JobType> remainingJobs = stepJobs.stream().filter(job -> ! completedAt(application.change(), application, job).isPresent()).collect(toSet()); - if (remainingJobs.isEmpty()) { // All jobs are complete -- find the time of completion of this step. + Map<Optional<Instant>, List<JobType>> jobsByCompletion = stepJobs.stream().collect(groupingBy(job -> completedAt(application.change(), application, job))); + if (jobsByCompletion.containsKey(empty())) { // Step not complete, because some jobs remain -- trigger these if the previous step was done. + for (JobType job : jobsByCompletion.get(empty())) { + State target = targetFor(application, application.change(), deploymentFor(application, job)); + if (isVerified(application, target, job)) { + if (completedAt.isPresent()) + jobs.add(deploymentJob(application, target, application.change(), job, reason, completedAt.get(), stepJobs)); + } + else if (testJobs == null) { + if ( ! alreadyTriggered(application, target)) + testJobs = testJobsFor(application, target, "Testing deployment for " + job.jobName(), completedAt.orElse(clock.instant())); + else + testJobs = emptyList(); + } + } + } + else { // All jobs are complete -- find the time of completion of this step. if (stepJobs.isEmpty()) { // No jobs means this is delay step. Duration delay = ((DeploymentSpec.Delay) step).duration(); completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> ! at.isAfter(clock.instant())); reason += " after a delay of " + delay; } else { - completedAt = stepJobs.stream().map(job -> completedAt(application.change(), application, job).get()).max(naturalOrder()); + completedAt = jobsByCompletion.keySet().stream().map(Optional::get).max(naturalOrder()); reason = "Available change in " + stepJobs.stream().map(JobType::jobName).collect(joining(", ")); } } - else if (completedAt.isPresent()) { // Step not complete, because some jobs remain -- trigger these if the previous step was done. - for (JobType job : remainingJobs) - jobs.add(deploymentJob(application, job, reason, completedAt.get(), stepJobs)); - completedAt = Optional.empty(); - break; - } } + + if (testJobs == null) + testJobs = testJobsFor(application, targetFor(application, application.change(), empty()), "Testing last changes outside prod", clock.instant()); + jobs.addAll(testJobs); + // TODO jvenstad: Replace with completion of individual parts of Change. - if (completedAt.isPresent()) + if (steps.stream().flatMap(step -> step.zones().stream()).map(order::toJob) + .allMatch(job -> completedAt(application.change(), application, job).isPresent())) applications().lockIfPresent(id, lockedApplication -> applications().store(lockedApplication.withChange(Change.empty()))); }); return jobs; } + private List<Job> testJobsFor(Application application, State target, String reason, Instant availableSince) { + List<Step> steps = application.deploymentSpec().steps(); + if (steps.isEmpty()) steps = singletonList(new DeploymentSpec.DeclaredZone(test)); + List<Job> jobs = new ArrayList<>(); + for (Step step : steps.stream().filter(step -> step.deploysTo(test) || step.deploysTo(staging)).collect(toList())) { + for (JobType jobType : step.zones().stream().map(order::toJob).collect(toList())) { + Optional<JobRun> completion = successOn(application, jobType, target); + if (completion.isPresent()) + availableSince = completion.get().at(); + else if (isVerified(application, target, jobType)) + jobs.add(deploymentJob(application, target, application.change(), jobType, reason, availableSince, emptySet())); + } + } + return jobs; + } + + private boolean isVerified(Application application, State state, JobType jobType) { + if (jobType.environment() == staging) + return successOn(application, systemTest, state).isPresent(); + if (jobType.environment() == prod) + return successOn(application, stagingTest, state).isPresent() + || ! JobList.from(application).production() + .lastTriggered().on(state.targetPlatform) + .lastTriggered().on(state.targetApplication) + .isEmpty(); + return true; + } + + private Optional<Instant> testedAt(Application application, State target) { + return max(successOn(application, systemTest, target).map(JobRun::at), + successOn(application, stagingTest, target).map(JobRun::at)); + } + + private boolean alreadyTriggered(Application application, State target) { + return ! JobList.from(application).production() + .lastTriggered().on(target.targetPlatform) + .lastTriggered().on(target.targetApplication) + .isEmpty(); + } + /** * Returns the instant when the given change is complete for the given application for the given job. * @@ -287,9 +375,10 @@ public class DeploymentTrigger { * part is a downgrade, regardless of the status of the job. */ private Optional<Instant> completedAt(Change change, Application application, JobType jobType) { - Optional<Instant> lastSuccess = application.deploymentJobs().successAt(change, jobType); + State target = targetFor(application, change, deploymentFor(application, jobType)); + Optional<JobRun> lastSuccess = successOn(application, jobType, target); if (lastSuccess.isPresent() || ! jobType.isProduction()) - return lastSuccess; + return lastSuccess.map(JobRun::at); return deploymentFor(application, jobType) .filter(deployment -> ! ( change.upgrades(deployment.version()) @@ -299,12 +388,14 @@ public class DeploymentTrigger { .map(Deployment::at); } - private boolean canTrigger(Job job) { - Application application = applications().require(job.id); - // TODO jvenstad: Check versions, not change. - if ( ! application.deploymentJobs().isDeployableTo(job.jobType.environment(), application.change())) - return false; + private Optional<JobRun> successOn(Application application, JobType jobType, State target) { + return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess) + .filter(last -> target.targetPlatform.equals(last.version()) + && target.targetApplication.equals(last.applicationVersion())); + } + private boolean canTrigger(Job job) { + Application application = applications().require(job.applicationId()); if (isRunning(application, job.jobType)) return false; @@ -324,7 +415,7 @@ public class DeploymentTrigger { return application.deploymentJobs().jobStatus().keySet().parallelStream() .filter(job -> job.isProduction()) .filter(job -> isRunning(application, job)) - .collect(Collectors.toList()); + .collect(toList()); } private ApplicationController applications() { @@ -342,44 +433,65 @@ public class DeploymentTrigger { return Optional.ofNullable(application.deployments().get(jobType.zone(controller.system()).get())); } - public static class Job { + private State targetFor(Application application, Change change, Optional<Deployment> deployment) { + return new State(targetPlatform(application, change, deployment), + targetApplication(application, change, deployment), + deployment.map(Deployment::version), + deployment.map(Deployment::applicationVersion)); + } + + + private static class Job extends BuildJob { - private final ApplicationId id; private final JobType jobType; - private final long projectId; private final String reason; private final Instant availableSince; private final Collection<JobType> concurrentlyWith; private final boolean isRetry; private final boolean isApplicationUpgrade; - private final Change change; - private final Version platformVersion; - private final ApplicationVersion applicationVersion; + private final State target; - private Job(Application application, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith, boolean isRetry, Change change, Version platformVersion, ApplicationVersion applicationVersion) { - this.id = application.id(); + private Job(Application application, State target, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith, boolean isRetry, boolean isApplicationUpgrade) { + super(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName()); this.jobType = jobType; - this.projectId = application.deploymentJobs().projectId().getAsLong(); this.availableSince = availableSince; this.concurrentlyWith = concurrentlyWith; this.reason = reason; this.isRetry = isRetry; - this.isApplicationUpgrade = change.application().isPresent(); - this.change = change; - this.platformVersion = platformVersion; - this.applicationVersion = applicationVersion; + this.isApplicationUpgrade = isApplicationUpgrade; + this.target = target; } - public ApplicationId id() { return id; } - public JobType jobType() { return jobType; } - public long projectId() { return projectId; } - public String reason() { return reason; } - public Instant availableSince() { return availableSince; } - public boolean isRetry() { return isRetry; } - public boolean applicationUpgrade() { return isApplicationUpgrade; } - public Change change() { return change; } - public Version platform() { return platformVersion; } - public ApplicationVersion application() { return applicationVersion; } + JobType jobType() { return jobType; } + Instant availableSince() { return availableSince; } + boolean isRetry() { return isRetry; } + boolean applicationUpgrade() { return isApplicationUpgrade; } + + } + + + private static class State { + + private final Version targetPlatform; + private final ApplicationVersion targetApplication; + private final Optional<Version> sourcePlatform; + private final Optional<ApplicationVersion> sourceApplication; + + public State(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, Optional<ApplicationVersion> sourceApplication) { + this.targetPlatform = targetPlatform; + this.targetApplication = targetApplication; + this.sourcePlatform = sourcePlatform; + this.sourceApplication = sourceApplication; + } + + @Override + public String toString() { + return String.format("platform %s %s, application %s %s", + targetPlatform, + sourcePlatform.map(v -> "(from " + v + ")").orElse(""), + targetApplication.id(), + sourceApplication.map(v -> "(from " + v.id() + ")").orElse("")); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 0f7d5b0eab2..3139a7efb29 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.screwdriver; +import com.google.common.base.Joiner; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -10,6 +11,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -19,7 +21,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.io.InputStream; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -28,6 +29,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; /** * This API lists deployment jobs that are queued for execution on Screwdriver. @@ -70,7 +72,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { return vespaVersion(); } if (path.matches("/screwdriver/v1/jobsToRun")) - return buildJobs(controller.applications().deploymentTrigger().computeReadyJobs().stream().collect(groupingBy(job -> job.jobType()))); + return buildJobs(controller.applications().deploymentTrigger().jobsToRun()); return notFound(request); } @@ -88,14 +90,13 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { .map(JobType::fromJobName) .orElse(JobType.component); - Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, "default")); - controller.applications().deploymentTrigger().trigger(controller.applications().deploymentTrigger().forcedDeploymentJob(application, - jobType, - "Triggered from screwdriver/v1")); + String triggered = controller.applications().deploymentTrigger() + .forceTrigger(ApplicationId.from(tenantName, applicationName, "default"), jobType) + .stream().map(JobType::jobName).collect(joining(", ")); Slime slime = new Slime(); Cursor cursor = slime.setObject(); - cursor.setString("message", "Triggered " + jobType.jobName() + " for " + application.id()); + cursor.setString("message", "Triggered " + triggered + " for " + tenantName + "." + applicationName); return new SlimeJsonResponse(slime); } @@ -112,20 +113,15 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse buildJobs(Map<JobType, List<DeploymentTrigger.Job>> jobLists) { + private HttpResponse buildJobs(Map<JobType, ? extends List<? extends BuildService.BuildJob>> jobLists) { Slime slime = new Slime(); Cursor jobTypesObject = slime.setObject(); jobLists.forEach((jobType, jobs) -> { Cursor jobArray = jobTypesObject.setArray(jobType.jobName()); jobs.forEach(job -> { Cursor buildJobObject = jobArray.addObject(); - buildJobObject.setString("id", job.id().toString()); + buildJobObject.setString("applicationId", job.applicationId().toString()); buildJobObject.setLong("projectId", job.projectId()); - buildJobObject.setString("reason", job.reason()); - buildJobObject.setString("change", job.change().toString()); - buildJobObject.setLong("availableSince", job.availableSince().toEpochMilli()); - buildJobObject.setString("platform", job.platform().toString()); - buildJobObject.setString("application", job.application().toString()); }); }); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 81228749323..9c82fa62da8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -53,6 +53,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import static com.yahoo.vespa.hosted.controller.ControllerTester.buildJob; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; @@ -179,7 +180,7 @@ public class ControllerTest { JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1); assertNotNull("Deployment job was not removed", jobStatus); assertEquals(42, jobStatus.lastCompleted().get().id()); - assertEquals("Available change in staging-test", jobStatus.lastCompleted().get().reason()); + assertEquals("New change available", jobStatus.lastCompleted().get().reason()); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() @@ -440,33 +441,33 @@ public class ControllerTest { assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app2.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.add(buildJob(app2, stagingTest)); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app1.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.add(buildJob(app1, stagingTest)); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app3.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.add(buildJob(app3, stagingTest)); assertEquals(jobs, tester.buildService().jobs()); // Remove the jobs for app1 and app2, and then let app3 fail with outOfCapacity. // All three jobs are now eligible, but the one for app3 should trigger first as an outOfCapacity-retry. - tester.buildService().removeJob(app1.deploymentJobs().projectId().getAsLong(), stagingTest.jobName()); - tester.buildService().removeJob(app2.deploymentJobs().projectId().getAsLong(), stagingTest.jobName()); + tester.buildService().remove(buildJob(app1, stagingTest)); + tester.buildService().remove(buildJob(app2, stagingTest)); tester.clock().advance(Duration.ofHours(13)); - jobs.remove(new BuildService.BuildJob(app1.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); - jobs.remove(new BuildService.BuildJob(app2.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.remove(buildJob(app1, stagingTest)); + jobs.remove(buildJob(app2, stagingTest)); tester.jobCompletion(stagingTest).application(app3).error(JobError.outOfCapacity).submit(); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app2.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.add(buildJob(app2, stagingTest)); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app1.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); + jobs.add(buildJob(app1, stagingTest)); assertEquals(jobs, tester.buildService().jobs()); // Finish deployment for apps 2 and 3, then release a new version, leaving only app1 with an application upgrade. @@ -486,21 +487,21 @@ public class ControllerTest { // Let the last system test job start, then remove the ones for apps 1 and 2, and let app3 fail with outOfCapacity again. tester.readyJobTrigger().maintain(); - tester.buildService().removeJob(app1.deploymentJobs().projectId().getAsLong(), systemTest.jobName()); - tester.buildService().removeJob(app2.deploymentJobs().projectId().getAsLong(), systemTest.jobName()); + tester.buildService().remove(buildJob(app1, systemTest)); + tester.buildService().remove(buildJob(app2, systemTest)); tester.clock().advance(Duration.ofHours(13)); jobs.clear(); - jobs.add(new BuildService.BuildJob(app1.deploymentJobs().projectId().getAsLong(), stagingTest.jobName())); - jobs.add(new BuildService.BuildJob(app3.deploymentJobs().projectId().getAsLong(), systemTest.jobName())); + jobs.add(buildJob(app1, stagingTest)); + jobs.add(buildJob(app3, systemTest)); tester.jobCompletion(systemTest).application(app3).error(JobError.outOfCapacity).submit(); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app1.deploymentJobs().projectId().getAsLong(), systemTest.jobName())); + jobs.add(buildJob(app1, systemTest)); assertEquals(jobs, tester.buildService().jobs()); tester.readyJobTrigger().maintain(); - jobs.add(new BuildService.BuildJob(app2.deploymentJobs().projectId().getAsLong(), systemTest.jobName())); + jobs.add(buildJob(app2, systemTest)); assertEquals(jobs, tester.buildService().jobs()); } @@ -539,26 +540,6 @@ public class ControllerTest { } @Test - public void testDeployUntestedChangeFails() { - DeploymentTester tester = new DeploymentTester(); - ApplicationController applications = tester.controller().applications(); - TenantName tenant = tester.controllerTester().createTenant("tenant1", "domain1", 11L); - Application app = tester.controllerTester().createApplication(tenant, "app1", "default", 1); - tester.deployCompletely(app, applicationPackage); - - tester.controller().applications().lockOrThrow(app.id(), application -> { - application = application.withChange(Change.of(Version.fromString("6.3"))); - applications.store(application); - try { - tester.controllerTester().deploy(app, ZoneId.from("prod", "corp-us-east-1"), applicationPackage); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.corp-us-east-1 as upgrade to 6.3 is not tested", e.getMessage()); - } - }); - } - - @Test public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index b4bc8140aca..614310d9221 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrgani import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; @@ -122,6 +123,10 @@ public final class ControllerTester { }); } + public static BuildService.BuildJob buildJob(Application application, DeploymentJobs.JobType jobType) { + return BuildService.BuildJob.of(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName()); + } + public Controller controller() { return controller; } public CuratorDb curator() { return curator; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index d5154c95cdc..ba0694e242b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -272,10 +272,10 @@ public class DeploymentTester { } private void notifyJobCompletion(DeploymentJobs.JobReport report) { - if (report.jobType() != JobType.component && ! buildService().removeJob(report.projectId(), report.jobType().jobName())) + if (report.jobType() != JobType.component && !buildService().remove(report.buildJob())) throw new IllegalArgumentException(report.jobType() + " is not running for " + report.applicationId()); assertFalse("Unexpected entry '" + report.jobType() + "@" + report.projectId() + " in: " + buildService().jobs(), - buildService().removeJob(report.projectId(), report.jobType().jobName())); + buildService().remove(report.buildJob())); clock().advance(Duration.ofMillis(1)); applications().deploymentTrigger().notifyOfCompletion(report); @@ -292,11 +292,7 @@ public class DeploymentTester { } public void assertRunning(ApplicationId id, JobType jobType) { - assertRunning(application(id).deploymentJobs().projectId().getAsLong(), jobType); - } - - public void assertRunning(long projectId, JobType jobType) { - assertTrue(buildService().jobs().contains(new BuildService.BuildJob(projectId, jobType.jobName()))); + assertTrue(buildService().jobs().contains(BuildService.BuildJob.of(id, application(id).deploymentJobs().projectId().getAsLong(), jobType.jobName()))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 28d6a516dbf..b2c109b1b2f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -26,6 +25,7 @@ import java.util.Optional; import java.util.function.Supplier; import static com.yahoo.config.provision.SystemName.main; +import static com.yahoo.vespa.hosted.controller.ControllerTester.buildJob; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionEuWest1; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsCentral1; @@ -34,6 +34,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; import static java.util.Collections.singletonList; +import static java.util.Optional.empty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -42,6 +43,7 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth * @author mpolden + * @author jvenstad */ public class DeploymentTriggerTest { @@ -131,14 +133,15 @@ public class DeploymentTriggerTest { // Test jobs pass tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); - tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); - assertTrue("No more jobs triggered at this time", mockBuildService.jobs().isEmpty()); + tester.deploymentTrigger().triggerReadyJobs(); - // 30 seconds pass, us-west-1 is triggered + // No jobs have started yet, as 30 seconds have not yet passed. + assertEquals(0, mockBuildService.jobs().size()); tester.clock().advance(Duration.ofSeconds(30)); tester.deploymentTrigger().triggerReadyJobs(); + // 30 seconds later, the first jobs may trigger. assertEquals(1, mockBuildService.jobs().size()); tester.assertRunning(application.id(), productionUsWest1); @@ -324,8 +327,7 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(2)); // ---------------- Exit block window: 20:30 tester.deploymentTrigger().triggerReadyJobs(); // Schedules the blocked production job(s) - assertEquals(singletonList(new BuildService.BuildJob(app.deploymentJobs().projectId().getAsLong(), "production-us-west-1")), - tester.buildService().jobs()); + assertEquals(singletonList(buildJob(app, productionUsWest1)), tester.buildService().jobs()); } @Test @@ -362,8 +364,8 @@ public class DeploymentTriggerTest { tester.completeDeploymentWithError(application, applicationPackage, BuildJob.defaultBuildNumber + 1, productionUsCentral1); // deployAndNotify doesn't actually deploy if the job fails, so we need to do that manually. - tester.deployAndNotify(application, Optional.empty(), false, productionUsCentral1); - tester.deploy(productionUsCentral1, application, Optional.empty(), false); + tester.deployAndNotify(application, empty(), false, productionUsCentral1); + tester.deploy(productionUsCentral1, application, empty(), false); ApplicationVersion appVersion1 = ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber + 1); assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion()); @@ -380,7 +382,18 @@ public class DeploymentTriggerTest { Version version1 = new Version("6.2"); tester.upgradeSystem(version1); tester.jobCompletion(productionUsCentral1).application(application).unsuccessful().submit(); - tester.completeUpgrade(application, version1, applicationPackage); + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, empty(), false, productionUsCentral1); + + // The last job has a different target, and the tests need to run again. + // These may now start, since the first job has been triggered once, and thus is verified already. + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); + + // Finally, the two production jobs complete, in order. + tester.deployAndNotify(application, empty(), true, productionUsCentral1); + tester.deployAndNotify(application, empty(), true, productionEuWest1); assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion()); } @@ -427,6 +440,7 @@ public class DeploymentTriggerTest { assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); assertEquals((Long) 42L, app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); + // TODO jvenstad: Fails here now, because job isn't triggered any more, as deploy target is not verified. assertNotEquals(triggered, app.get().deploymentJobs().jobStatus().get(productionUsCentral1).lastTriggered().get().at()); // Change has a higher application version than what is deployed -- deployment should trigger. @@ -435,11 +449,66 @@ public class DeploymentTriggerTest { assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); assertEquals((Long) 43L, app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); - // Change is again strictly dominated, and us-central-1 should be skipped, even though it is still a failure. + // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. tester.deployAndNotify(application, applicationPackage, false, productionUsCentral1); + + // Last job has a different deployment target, so tests need to run again. + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); tester.deployAndNotify(application, applicationPackage, true, productionEuWest1); assertFalse(app.get().change().isPresent()); assertFalse(app.get().deploymentJobs().jobStatus().get(productionUsCentral1).isSuccess()); } + @Test + public void eachDeployTargetIsTested() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + Supplier<Application> app = () -> tester.application(application.id()); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-central-1") + .parallel("eu-west-1", "us-east-3") + .build(); + // Application version 42 and platform version 6.1. + tester.deployCompletely(application, applicationPackage); + + // Success in first prod zone, change cancelled between triggering and deployment to two parallel zones. + // One of the parallel zones get a deployment, but both fail their jobs. + Version v1 = new Version("6.1"); + Version v2 = new Version("6.2"); + tester.upgradeSystem(v2); + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, empty(), true, productionUsCentral1); + tester.deploymentTrigger().cancelChange(application.id(), true); + tester.deploy(productionEuWest1, application, applicationPackage); + tester.deployAndNotify(application, applicationPackage, false, productionEuWest1); + tester.deployAndNotify(application, applicationPackage, false, productionUsEast3); + assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); + assertEquals(v2, app.get().deployments().get(productionEuWest1.zone(main).get()).version()); + assertEquals(v1, app.get().deployments().get(productionUsEast3.zone(main).get()).version()); + + // New application version should run system and staging tests first against 6.2, then against 6.1. + tester.jobCompletion(component).application(application).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + assertEquals(v2, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().version()); + tester.deployAndNotify(application, empty(), true, systemTest); + assertEquals(v2, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().version()); + tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, empty(), true, productionUsCentral1); + assertEquals(v1, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().version()); + tester.deployAndNotify(application, empty(), true, systemTest); + assertEquals(v1, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().version()); + tester.deployAndNotify(application, empty(), true, stagingTest); + + // The production job on version 6.2 fails and must retry -- this is OK, even though staging now has a different version. + tester.deployAndNotify(application, empty(), false, productionEuWest1); + tester.deployAndNotify(application, empty(), true, productionUsEast3); + tester.deployAndNotify(application, empty(), true, productionEuWest1); + assertFalse(app.get().change().isPresent()); + assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionUsCentral1).lastSuccess().get().applicationVersion().buildNumber().get().longValue()); + assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionEuWest1).lastSuccess().get().applicationVersion().buildNumber().get().longValue()); + assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionUsEast3).lastSuccess().get().applicationVersion().buildNumber().get().longValue()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index d28a70eb838..5dd2eab72c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -85,7 +85,7 @@ public class FailureRedeployerTest { // Production job fails again, and is retried tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3); tester.readyJobTrigger().maintain(); - assertEquals("Job is retried", Collections.singletonList(new BuildService.BuildJob(app.deploymentJobs().projectId().getAsLong(), productionUsEast3.jobName())), tester.buildService().jobs()); + assertEquals("Job is retried", Collections.singletonList(ControllerTester.buildJob(app, productionUsEast3)), tester.buildService().jobs()); // Production job finally succeeds tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); @@ -125,7 +125,7 @@ public class FailureRedeployerTest { tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); - tester.buildService().removeJob((long) 1, systemTest.jobName()); + tester.buildService().remove(ControllerTester.buildJob(app, systemTest)); tester.upgrader().maintain(); tester.readyJobTrigger().maintain(); assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 424141137b0..5b1c99ec727 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -123,8 +123,8 @@ public class UpgraderTest { tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); - tester.buildService().removeJob(canary0.deploymentJobs().projectId().getAsLong(), stagingTest.jobName()); - tester.buildService().removeJob(canary1.deploymentJobs().projectId().getAsLong(), systemTest.jobName()); + tester.buildService().remove(ControllerTester.buildJob(canary0, stagingTest)); + tester.buildService().remove(ControllerTester.buildJob(canary1, systemTest)); tester.readyJobTrigger().maintain(); tester.readyJobTrigger().maintain(); @@ -504,6 +504,9 @@ public class UpgraderTest { tester.readyJobTrigger().maintain(); tester.readyJobTrigger().maintain(); tester.readyJobTrigger().maintain(); + tester.readyJobTrigger().maintain(); + tester.readyJobTrigger().maintain(); + tester.readyJobTrigger().maintain(); // Canaries upgrade and raise confidence of V2 tester.completeUpgrade(canary0, v2, "canary"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 5184eeacc33..e7cb92e2264 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.time.Duration; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static org.junit.Assert.assertFalse; /** @@ -125,10 +126,10 @@ public class ContainerControllerTester { private void notifyJobCompletion(DeploymentJobs.JobReport report) { MockBuildService buildService = (MockBuildService) containerTester.container().components().getComponent(MockBuildService.class.getName()); - if (report.jobType() != DeploymentJobs.JobType.component && ! buildService.removeJob(report.projectId(), report.jobType().jobName())) + if (report.jobType() != component && ! buildService.remove(report.buildJob())) throw new IllegalArgumentException(report.jobType() + " is not running for " + report.applicationId()); assertFalse("Unexpected entry '" + report.jobType() + "@" + report.projectId() + " in: " + buildService.jobs(), - buildService.removeJob(report.projectId(), report.jobType().jobName())); + buildService.remove(report.buildJob())); try { Thread.sleep(1); } catch (InterruptedException e) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index e1ff2712e4a..1a88a7691f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -48,7 +48,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.101-commit1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" }, "lastCompleted": { @@ -62,7 +62,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.101-commit1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" }, "lastSuccess": { @@ -76,7 +76,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.101-commit1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" } }, @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-us-west-1", "at": "(ignore)" }, "lastCompleted": { @@ -108,7 +108,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-us-west-1", "at": "(ignore)" }, "lastSuccess": { @@ -122,7 +122,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-us-west-1", "at": "(ignore)" } }, @@ -140,7 +140,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "lastCompleted": { @@ -154,7 +154,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "lastSuccess": { @@ -168,7 +168,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" } }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 2491aec22d1..e3d7c86c051 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -58,7 +58,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastCompleted": { @@ -72,7 +72,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastSuccess": { @@ -86,7 +86,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" } }, @@ -104,7 +104,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastCompleted": { @@ -118,7 +118,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastSuccess": { @@ -132,7 +132,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" } }, @@ -150,7 +150,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "lastCompleted": { @@ -164,7 +164,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "firstFailing": { @@ -178,7 +178,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 8598ff37333..4bc7511c695 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -58,7 +58,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastCompleted": { @@ -72,7 +72,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastSuccess": { @@ -86,7 +86,7 @@ "gitCommit": "commit1" } }, - "reason": "Deploying application change to 1.0.42-commit1", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" } }, @@ -104,7 +104,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastCompleted": { @@ -118,7 +118,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" }, "lastSuccess": { @@ -132,7 +132,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in system-test", + "reason": "Testing deployment for production-corp-us-east-1", "at": "(ignore)" } }, @@ -150,7 +150,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "lastCompleted": { @@ -164,7 +164,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" }, "firstFailing": { @@ -178,7 +178,7 @@ "gitCommit": "commit1" } }, - "reason": "Available change in staging-test", + "reason": "New change available", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 6e79f53cae6..a0cbf39e5ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.controller.restapi.screwdriver; import com.yahoo.application.container.handler.Request; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -10,6 +13,7 @@ import org.junit.Test; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.Optional; import java.util.OptionalLong; /** @@ -57,12 +61,32 @@ public class ScrewdriverApiTest extends ControllerContainerTest { app.id().tenant().value() + "/application/" + app.id().application().value(), new byte[0], Request.Method.POST), 200, "{\"message\":\"Triggered component for tenant1.application1\"}"); + tester.controller().applications().deploymentTrigger().notifyOfCompletion(new JobReport(app.id(), + DeploymentJobs.JobType.component, + 1, + 42, + Optional.of(new SourceRevision("repo", "branch", "commit")), + Optional.empty())); - // Triggers specific job when given -- fails here because the job has never run before, and so application version can't be resolved. + // Triggers specific job when given -- triggers the prerequisites here, since they are not yet fulfilled. assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + app.id().tenant().value() + "/application/" + app.id().application().value(), "staging-test".getBytes(StandardCharsets.UTF_8), Request.Method.POST), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot determine application version to use for stagingTest\"}"); + 200, "{\"message\":\"Triggered system-test for tenant1.application1\"}"); + + tester.controller().applications().deploymentTrigger().notifyOfCompletion(new JobReport(app.id(), + DeploymentJobs.JobType.systemTest, + 1, + 42, + Optional.empty(), + Optional.empty())); + + // Triggers specific job when given, and when it is verified. + assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + + app.id().tenant().value() + "/application/" + app.id().application().value(), + "staging-test".getBytes(StandardCharsets.UTF_8), Request.Method.POST), + 200, "{\"message\":\"Triggered staging-test for tenant1.application1\"}"); + } } |