diff options
14 files changed, 167 insertions, 66 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index eddf4e6b3a1..bd10a884213 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -113,6 +113,12 @@ public class LockedApplication { ownershipIssueId, metrics, rotation, rotationStatus); } + public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { + return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, + deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, + ownershipIssueId, metrics, rotation, rotationStatus); + } + public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, Optional<DeploymentJobs.JobError> jobError) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 1fa579684de..0fb6459611c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -69,6 +69,15 @@ public final class Change { return new Change(platform, Optional.of(applicationVersion)); } + /** Returns the change obtained when overwriting elements of the given change with any present in this */ + public Change onTopOf(Change other) { + if (platform.isPresent()) + other = other.with(platform.get()); + if (application.isPresent()) + other = other.with(application.get()); + return other; + } + @Override public int hashCode() { return Objects.hash(platform, application); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 3703616cb68..6e9318042b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -71,6 +71,15 @@ public class DeploymentJobs { return new DeploymentJobs(projectId, status, issueId, builtInternally); } + public DeploymentJobs withPause(JobType jobType, OptionalLong pausedUntil) { + Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); + status.compute(jobType, (__, job) -> { + if (job == null) job = JobStatus.initial(jobType); + return job.withPause(pausedUntil); + }); + return new DeploymentJobs(projectId, status, issueId, builtInternally); + } + public DeploymentJobs withProjectId(OptionalLong projectId) { return new DeploymentJobs(projectId, status, issueId, builtInternally); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index a06a3e00340..59220d38821 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.time.Instant; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import static java.util.Objects.requireNonNull; @@ -25,6 +26,7 @@ public class JobStatus { private final Optional<JobRun> lastCompleted; private final Optional<JobRun> firstFailing; private final Optional<JobRun> lastSuccess; + private final OptionalLong pausedUntil; private final Optional<DeploymentJobs.JobError> jobError; @@ -34,27 +36,23 @@ public class JobStatus { */ public JobStatus(JobType type, Optional<DeploymentJobs.JobError> jobError, Optional<JobRun> lastTriggered, Optional<JobRun> lastCompleted, - Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess) { - requireNonNull(type, "jobType cannot be null"); - requireNonNull(jobError, "jobError cannot be null"); - requireNonNull(lastTriggered, "lastTriggered cannot be null"); - requireNonNull(lastCompleted, "lastCompleted cannot be null"); - requireNonNull(firstFailing, "firstFailing cannot be null"); - requireNonNull(lastSuccess, "lastSuccess cannot be null"); - - this.type = type; - this.jobError = jobError; + Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess, + OptionalLong pausedUntil) { + this.type = requireNonNull(type, "jobType cannot be null"); + this.jobError = requireNonNull(jobError, "jobError cannot be null"); // Never say we triggered component because we don't: - this.lastTriggered = type == JobType.component ? Optional.empty() : lastTriggered; - this.lastCompleted = lastCompleted; - this.firstFailing = firstFailing; - this.lastSuccess = lastSuccess; + this.lastTriggered = type == JobType.component ? Optional.empty() : requireNonNull(lastTriggered, "lastTriggered cannot be null"); + this.lastCompleted = requireNonNull(lastCompleted, "lastCompleted cannot be null"); + this.firstFailing = requireNonNull(firstFailing, "firstFailing cannot be null"); + this.lastSuccess = requireNonNull(lastSuccess, "lastSuccess cannot be null"); + this.pausedUntil = requireNonNull(pausedUntil, "pausedUntil cannot be null"); + } /** Returns an empty job status */ public static JobStatus initial(JobType type) { - return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), OptionalLong.empty()); } public JobStatus withTriggering(Version platform, ApplicationVersion application, Optional<Deployment> deployment, String reason, Instant triggeredAt) { @@ -62,7 +60,7 @@ public class JobStatus { } public JobStatus withTriggering(JobRun jobRun) { - return new JobStatus(type, jobError, Optional.of(jobRun), lastCompleted, firstFailing, lastSuccess); + return new JobStatus(type, jobError, Optional.of(jobRun), lastCompleted, firstFailing, lastSuccess, OptionalLong.empty()); } public JobStatus withCompletion(long runId, Optional<DeploymentJobs.JobError> jobError, Instant completion) { @@ -80,7 +78,11 @@ public class JobStatus { firstFailing = Optional.empty(); } - return new JobStatus(type, jobError, lastTriggered, Optional.of(completion), firstFailing, lastSuccess); + return new JobStatus(type, jobError, lastTriggered, Optional.of(completion), firstFailing, lastSuccess, pausedUntil); + } + + public JobStatus withPause(OptionalLong pausedUntil) { + return new JobStatus(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess, pausedUntil); } public JobType type() { return type; } @@ -113,17 +115,21 @@ public class JobStatus { /** Returns the run when this last succeeded, or empty if it has never succeeded */ public Optional<JobRun> lastSuccess() { return lastSuccess; } + /** Returns the time until which this job is paused, if currently paused */ + public OptionalLong pausedUntil() { return pausedUntil; } + @Override public String toString() { return "job status of " + type + "[ " + "last triggered: " + lastTriggered.map(JobRun::toString).orElse("(never)") + ", last completed: " + lastCompleted.map(JobRun::toString).orElse("(never)") + ", first failing: " + firstFailing.map(JobRun::toString).orElse("(not failing)") + - ", lastSuccess: " + lastSuccess.map(JobRun::toString).orElse("(never)") + "]"; + ", lastSuccess: " + lastSuccess.map(JobRun::toString).orElse("(never)") + + ", pausedUntil: " + (pausedUntil.isPresent() ? pausedUntil.getAsLong() : "(not paused)") + "]"; } @Override - public int hashCode() { return Objects.hash(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess); } + public int hashCode() { return Objects.hash(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess, pausedUntil); } @Override public boolean equals(Object o) { @@ -135,7 +141,8 @@ public class JobStatus { Objects.equals(lastTriggered, other.lastTriggered) && Objects.equals(lastCompleted, other.lastCompleted) && Objects.equals(firstFailing, other.firstFailing) && - Objects.equals(lastSuccess, other.lastSuccess); + Objects.equals(lastSuccess, other.lastSuccess) && + Objects.equals(pausedUntil, other.pausedUntil); } /** Information about a particular triggering or completion of a run of a job. This is immutable. */ @@ -202,7 +209,7 @@ public class JobStatus { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof JobRun)) return false; + if ( ! (o instanceof JobRun)) return false; JobRun run = (JobRun) o; @@ -224,6 +231,7 @@ public class JobStatus { result = 31 * result + at.hashCode(); return result; } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 6dbdefb0913..69a42866a9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -66,6 +66,8 @@ import static java.util.stream.Collectors.toList; */ public class DeploymentTrigger { + public static final Duration maxPause = Duration.ofDays(3); + private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName()); private final Controller controller; @@ -222,31 +224,33 @@ public class DeploymentTrigger { .map(Job::jobType).collect(toList()); } + /** Prevents jobs of the given type from starting, until the given time. */ + public void pauseJob(ApplicationId id, JobType jobType, Instant until) { + if (until.isAfter(clock.instant().plus(maxPause))) + throw new IllegalArgumentException("Pause only allowed for up to " + maxPause); + + applications().lockOrThrow(id, application -> + applications().store(application.withJobPause(jobType, OptionalLong.of(until.toEpochMilli())))); + } + /** Triggers a change of this application, unless it already has a change. */ public void triggerChange(ApplicationId applicationId, Change change) { applications().lockOrThrow(applicationId, application -> { - if ( ! application.get().change().isPresent()) { - if (change.application().isPresent()) - application = application.withOutstandingChange(Change.empty()); - - applications().store(application.withChange(change)); - } + if ( ! application.get().change().isPresent()) + forceChange(applicationId, change); }); } /** Overrides the given application's platform and application changes with any contained in the given change. */ public void forceChange(ApplicationId applicationId, Change change) { applications().lockOrThrow(applicationId, application -> { - Change current = application.get().change(); - if (change.platform().isPresent()) - current = current.with(change.platform().get()); if (change.application().isPresent()) - current = current.with(change.application().get()); - applications().store(application.withChange(current)); + application = application.withOutstandingChange(Change.empty()); + applications().store(application.withChange(change.onTopOf(application.get().change()))); }); } - /** Cancels a platform upgrade of the given application, and an application upgrade as well if {@code keepApplicationChange}. */ + /** Cancels the indicated part of the given application's change. */ public void cancelChange(ApplicationId applicationId, ChangesToCancel cancellation) { applications().lockOrThrow(applicationId, application -> { Change change; @@ -313,7 +317,7 @@ public class DeploymentTrigger { for (Step step : steps.production()) { List<JobType> stepJobs = steps.toJobs(step); List<JobType> remainingJobs = stepJobs.stream().filter(job -> ! isComplete(change, application, job)).collect(toList()); - if (!remainingJobs.isEmpty()) { // Change is incomplete; trigger remaining jobs if ready, or their test jobs if untested. + if ( ! remainingJobs.isEmpty()) { // Change is incomplete; trigger remaining jobs if ready, or their test jobs if untested. for (JobType job : remainingJobs) { Versions versions = Versions.from(change, application, deploymentFor(application, job), controller.systemVersion()); @@ -321,7 +325,7 @@ public class DeploymentTrigger { if (completedAt.isPresent() && canTrigger(job, versions, application, stepJobs)) { jobs.add(deploymentJob(application, versions, change, job, reason, completedAt.get())); } - if (!alreadyTriggered(application, versions)) { + if ( ! alreadyTriggered(application, versions)) { testJobs = emptyList(); } } @@ -348,10 +352,7 @@ public class DeploymentTrigger { } } if (testJobs == null) { // If nothing to test, but outstanding commits, test those. - Change latestChange = application.outstandingChange().application().isPresent() - ? change.with(application.outstandingChange().application().get()) - : change; - testJobs = testJobs(application, Versions.from(latestChange, + testJobs = testJobs(application, Versions.from(application.outstandingChange().onTopOf(application.change()), application, steps.sortedDeployments(application.productionDeployments().values()).stream().findFirst(), controller.systemVersion()), @@ -392,11 +393,12 @@ public class DeploymentTrigger { /** Returns whether the given job can trigger at the given instant */ public boolean triggerAt(Instant instant, JobType job, Versions versions, Application application) { Optional<JobStatus> jobStatus = application.deploymentJobs().statusOf(job); - if (!jobStatus.isPresent()) return true; + if ( ! jobStatus.isPresent()) return true; + if (jobStatus.get().pausedUntil().isPresent() && jobStatus.get().pausedUntil().getAsLong() > clock.instant().toEpochMilli()) return false; if (jobStatus.get().isSuccess()) return true; // Success - if (!jobStatus.get().lastCompleted().isPresent()) return true; // Never completed - if (!jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job - if (!versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed + if ( ! jobStatus.get().lastCompleted().isPresent()) return true; // Never completed + if ( ! jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job + if ( ! versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed if (application.deploymentSpec().upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries Instant firstFailing = jobStatus.get().firstFailing().get().at(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 7b615249a65..365d74babb5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -95,6 +95,7 @@ public class ApplicationSerializer { private final String lastCompletedField = "lastCompleted"; private final String firstFailingField = "firstFailing"; private final String lastSuccessField = "lastSuccess"; + private final String pausedUntilField = "pausedUntil"; // JobRun fields private final String jobRunIdField = "id"; @@ -252,6 +253,7 @@ public class ApplicationSerializer { jobStatus.lastCompleted().ifPresent(run -> jobRunToSlime(run, object, lastCompletedField)); jobStatus.lastSuccess().ifPresent(run -> jobRunToSlime(run, object, lastSuccessField)); jobStatus.firstFailing().ifPresent(run -> jobRunToSlime(run, object, firstFailingField)); + jobStatus.pausedUntil().ifPresent(until -> object.setLong(pausedUntilField, until)); } private void jobRunToSlime(JobStatus.JobRun jobRun, Cursor parent, String jobRunObjectName) { @@ -440,11 +442,13 @@ public class ApplicationSerializer { if (object.field(errorField).valid()) jobError = Optional.of(JobError.valueOf(object.field(errorField).asString())); - return Optional.of(new JobStatus(jobType.get(), jobError, - jobRunFromSlime(object.field(lastTriggeredField)), - jobRunFromSlime(object.field(lastCompletedField)), - jobRunFromSlime(object.field(firstFailingField)), - jobRunFromSlime(object.field(lastSuccessField)))); + return Optional.of(new JobStatus(jobType.get(), + jobError, + jobRunFromSlime(object.field(lastTriggeredField)), + jobRunFromSlime(object.field(lastCompletedField)), + jobRunFromSlime(object.field(firstFailingField)), + jobRunFromSlime(object.field(lastSuccessField)), + optionalLong(object.field(pausedUntilField)))); } private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index ec5cfaacc4e..dac65f16832 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -65,6 +65,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; @@ -88,6 +89,7 @@ import java.net.URISyntaxException; import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -98,6 +100,7 @@ import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; +import static java.util.stream.Collectors.joining; /** * This implements the application/v4 API which is used to deploy and manage applications @@ -201,6 +204,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -366,6 +371,19 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) { + String triggered = controller.applications().deploymentTrigger() + .forceTrigger(id, type, request.getJDiscRequest().getUserPrincipal().getName()) + .stream().map(JobType::jobName).collect(joining(", ")); + return new MessageResponse("Triggered " + triggered + " for " + id); + } + + private HttpResponse pause(ApplicationId id, JobType type) { + Instant until = controller.clock().instant().plus(DeploymentTrigger.maxPause); + controller.applications().deploymentTrigger().pauseJob(id, type, until); + return new MessageResponse(type.jobName() + " for " + id + " paused for " + DeploymentTrigger.maxPause); + } + private HashMap<String, String> getParameters(String query) { HashMap<String, String> keyValPair = new HashMap<>(); Arrays.stream(query.split("&")).forEach(pair -> { @@ -738,8 +756,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - /** Trigger deployment of the last built application package, on a given version */ - // TODO Consider move to API for maintenance related operations + /** + * Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9", + * or the latest known commit of the application if "commit" is given, + * or an upgrade to the system version if no data is provided. + */ private HttpResponse deploy(String tenantName, String applicationName, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); String requestVersion = readToString(request.getData()); @@ -758,17 +779,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .stream() .map(VespaVersion::versionNumber) .map(Version::toString) - .collect(Collectors.joining(", "))); + .collect(joining(", "))); change = Change.of(version); } - controller.applications().deploymentTrigger().forceChange(application.get().id(), change); - response.append("Triggered " + change + " for " + application); + controller.applications().deploymentTrigger().forceChange(id, change); + response.append("Triggered " + change + " for " + id); }); return new MessageResponse(response.toString()); } /** Cancel any ongoing change for given application */ - // TODO Consider move to API for maintenance related operations private HttpResponse cancelDeploy(String tenantName, String applicationName) { ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); Application application = controller.applications().require(id); @@ -777,7 +797,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse("No deployment in progress for " + application + " at this time"); controller.applications().lockOrThrow(id, lockedApplication -> - controller.applications().deploymentTrigger().cancelChange(id, ALL)); + controller.applications().deploymentTrigger().cancelChange(id, ALL)); return new MessageResponse("Cancelled " + change + " for " + application); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index b9f91a35790..b1e3f8799d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -117,8 +117,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static boolean isHostedOperatorOperation(Path path, Method method) { if (isWhiteListedOperation(path, method)) return false; - return path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") || - path.matches("/controller/v1/{*}") || + return path.matches("/controller/v1/{*}") || path.matches("/provision/v2/{*}") || path.matches("/screwdriver/v1/trigger/tenant/{*}") || path.matches("/os/v1/{*}") || @@ -131,7 +130,8 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { if (isHostedOperatorOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}") || path.matches("/application/v4/tenant/{tenant}/application/{application}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{job}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{job}/{*}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 6633eafc509..14bb89520d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -26,6 +26,7 @@ import java.util.logging.Logger; import static java.util.stream.Collectors.joining; +// TODO jvenstad: Only useful method has been moved to application API. Delete this when users have updated. /** * This API lists deployment jobs that are queued for execution on Screwdriver. * diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 53ab488949e..db58ef1830f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -526,6 +526,37 @@ public class DeploymentTriggerTest { } @Test + public void testJobPause() { + Application app = tester.createAndDeploy("app", 3, "default"); + tester.upgradeSystem(new Version("9.8.7")); + + tester.applications().deploymentTrigger().pauseJob(app.id(), productionUsWest1, tester.clock().instant().plus(Duration.ofSeconds(1))); + tester.applications().deploymentTrigger().pauseJob(app.id(), productionUsEast3, tester.clock().instant().plus(Duration.ofSeconds(3))); + + // us-west-1 does not trigger when paused. + tester.deployAndNotify(app, true, systemTest); + tester.deployAndNotify(app, true, stagingTest); + tester.assertNotRunning(productionUsWest1, app.id()); + + // us-west-1 triggers when no longer paused, but does not retry when paused again. + tester.clock().advance(Duration.ofMillis(1500)); + tester.readyJobTrigger().run(); + tester.assertRunning(productionUsWest1, app.id()); + tester.applications().deploymentTrigger().pauseJob(app.id(), productionUsWest1, tester.clock().instant().plus(Duration.ofSeconds(1))); + tester.deployAndNotify(app, false, productionUsWest1); + tester.assertNotRunning(productionUsWest1, app.id()); + tester.clock().advance(Duration.ofMillis(1000)); + tester.readyJobTrigger().run(); + tester.deployAndNotify(app, true, productionUsWest1); + + // us-east-3 does not automatically trigger when paused, but does when forced. + tester.assertNotRunning(productionUsEast3, app.id()); + tester.deploymentTrigger().forceTrigger(app.id(), productionUsEast3, "mrTrigger"); + tester.assertRunning(productionUsEast3, app.id()); + assertFalse(tester.application(app.id()).deploymentJobs().jobStatus().get(productionUsEast3).pausedUntil().isPresent()); + } + + @Test public void testUpgradingButNoJobStarted() { ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), Duration.ofHours(1), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 094f8989530..e06578a545f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -83,7 +83,8 @@ public class ApplicationSerializerTest { statusList.add(JobStatus.initial(JobType.systemTest) .withTriggering(Version.fromString("5.6.7"), ApplicationVersion.unknown, empty(), "Test", Instant.ofEpochMilli(7)) - .withCompletion(30, empty(), Instant.ofEpochMilli(8))); + .withCompletion(30, empty(), Instant.ofEpochMilli(8)) + .withPause(OptionalLong.of(1L << 32))); statusList.add(JobStatus.initial(JobType.stagingTest) .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, empty(), "Test 2", Instant.ofEpochMilli(5)) .withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 09ea360feeb..b82462ad595 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentMetricsMaintainer; @@ -347,15 +348,25 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (cancel) again is a no-op tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) - .userIdentity(HOSTED_VESPA_OPERATOR), + .userIdentity(USER_ID), new File("application-deployment-cancelled-no-op.json")); // POST triggering of a full deployment to an application (if version is omitted, current system version is used) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST) - .userIdentity(HOSTED_VESPA_OPERATOR) + .userIdentity(USER_ID) .data("6.1.0"), new File("application-deployment.json")); + // POST a pause to a production job + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) + .userIdentity(USER_ID), + "{\"message\":\"production-us-west-1 for tenant1.application1 paused for " + DeploymentTrigger.maxPause + "\"}"); + + // POST a triggering to the same production job + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1", POST) + .userIdentity(USER_ID), + "{\"message\":\"Triggered production-us-west-1 for tenant1.application1\"}"); + // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), @@ -497,9 +508,8 @@ public class ApplicationApiTest extends ControllerContainerTest { private void addIssues(ContainerControllerTester tester, ApplicationId id) { tester.controller().applications().lockOrThrow(id, application -> - tester.controller().applications().store(application - .withDeploymentIssueId(IssueId.from("123")) - .withOwnershipIssueId(IssueId.from("321")))); + tester.controller().applications().store(application.withDeploymentIssueId(IssueId.from("123")) + .withOwnershipIssueId(IssueId.from("321")))); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json index e3c0bbe0679..d2531638a93 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json @@ -1 +1 @@ -{"message":"Triggered upgrade to 6.1 for application 'tenant1.application1'"}
\ No newline at end of file +{"message":"Triggered upgrade to 6.1 for tenant1.application1"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index 22a527bf3d3..19aa247edb4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -70,7 +70,7 @@ public class ControllerAuthorizationFilterTest { List<AthenzIdentity> allowed = singletonList(HOSTED_OPERATOR); List<AthenzIdentity> forbidden = singletonList(USER); - testApiAccess(PUT, "/application/v4/tenant/mytenant/application/myapp/deploying", + testApiAccess(PUT, "/zone/v2/hello-proxy-path", allowed, forbidden, filter); testApiAccess(POST, "/screwdriver/v1/trigger/tenant/mytenant/application/myapp/", allowed, forbidden, filter); |