summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-01-26 17:35:13 +0100
committerJon Bratseth <bratseth@oath.com>2018-01-26 17:35:13 +0100
commit49bf23d2adacfb38b826ceb5c86dfeaccf824f5d (patch)
treed8aa214418dc7d397a92f4c15c8e616200682398
parentb0e92a15e902416d40909bc856a488eae12822d3 (diff)
Simplify Change
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java12
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java164
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java9
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java45
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java20
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java10
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java17
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java10
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java8
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java23
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java16
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json2
19 files changed, 159 insertions, 222 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index abfd94f87a2..e47f5519fae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
import com.yahoo.vespa.hosted.controller.application.ApplicationRotation;
import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.Change.VersionChange;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
import com.yahoo.vespa.hosted.controller.rotation.RotationId;
@@ -152,10 +151,7 @@ public class Application {
/** Returns the version a new deployment to this zone should use for this application */
public Version deployVersionIn(ZoneId zone, Controller controller) {
- if (deploying() instanceof VersionChange)
- return ((Change.VersionChange) deploying()).version();
-
- return versionIn(zone, controller);
+ return deploying.platform().orElse(versionIn(zone, controller));
}
/** Returns the current version this application has, or if none; should use, in the given zone */
@@ -166,8 +162,8 @@ public class Application {
/** Returns the application version a deployment to this zone should use, or empty if we don't know */
public Optional<ApplicationVersion> deployApplicationVersionIn(ZoneId zone) {
- if (deploying() instanceof Change.ApplicationChange) {
- ApplicationVersion version = ((Change.ApplicationChange) deploying()).version();
+ if (deploying().application().isPresent()) {
+ ApplicationVersion version = deploying().application().get();
if (version == ApplicationVersion.unknown)
return Optional.empty();
else
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index dc39268c48d..fed70b8d175 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -39,7 +39,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerato
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
@@ -323,8 +322,8 @@ public class ApplicationController {
// TODO: Remove after introducing new application version number
if ( ! options.deployCurrentVersion && applicationPackageFromDeployer.isPresent()) {
- if (application.deploying() instanceof Change.ApplicationChange) {
- application = application.withDeploying(Change.ApplicationChange.of(applicationVersion));
+ if (application.deploying().application().isPresent()) {
+ application = application.withDeploying(application.deploying().with(applicationVersion));
}
if (!canDeployDirectlyTo(zone, options) && jobType.isPresent()) {
// Update with (potentially) missing information about what we triggered:
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index dd7c238053c..fe085a6bfa4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -54,7 +54,7 @@ public class ApplicationList {
/** Returns the subset of applications which are currently upgrading (to any version) */
public ApplicationList upgrading() {
- return listOf(list.stream().filter(ApplicationList::isUpgrading));
+ return listOf(list.stream().filter(application -> application.deploying().platform().isPresent()));
}
/** Returns the subset of applications which are currently upgrading to the given version */
@@ -170,16 +170,8 @@ public class ApplicationList {
// ----------------------------------- Internal helpers
- private static boolean isUpgrading(Application application) {
- if ( ! (application.deploying().isPresent()) ) return false;
- if ( ! (application.deploying() instanceof Change.VersionChange) ) return false;
- return true;
- }
-
private static boolean isUpgradingTo(Version version, Application application) {
- if ( ! (application.deploying().isPresent()) ) return false;
- if ( ! (application.deploying() instanceof Change.VersionChange) ) return false;
- return ((Change.VersionChange)application.deploying()).version().equals(version);
+ return application.deploying().platform().equals(Optional.of(version));
}
private static boolean failingOn(Version version, Application application) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
index 5e3952f5ad7..7c1049da587 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
@@ -6,125 +6,93 @@ import com.yahoo.config.application.api.DeploymentSpec;
import java.time.Instant;
import java.util.Objects;
+import java.util.Optional;
/**
- * A change to an application
+ * The changes to an application we currently wish to complete deploying.
+ * A goal of the system is to deploy platform and application versions separately.
+ * However, this goal must some times be traded against others, so a change can
+ * consist of both an application and platform version change.
+ *
+ * This is immutable.
*
* @author bratseth
*/
-public abstract class Change {
-
- private static NoChange none = new NoChange();
-
- /** Returns true if this change is blocked by the given spec at the given instant */
- public abstract boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant);
-
- public abstract boolean isPresent();
-
- public static Change empty() { return none; }
-
- public static class NoChange extends Change {
+public final class Change {
- private NoChange() { }
+ private static Change empty = new Change(Optional.empty(), Optional.empty());
- @Override
- public boolean isPresent() { return false; }
+ /** The platform version we are upgrading to, or empty if none */
+ private final Optional<Version> platform;
- @Override
- public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) {
- return false;
- }
+ /** The application version we are changing to, or empty if none */
+ private final Optional<ApplicationVersion> application;
+ private Change(Optional<Version> platform, Optional<ApplicationVersion> application) {
+ Objects.requireNonNull(platform, "platform cannot be null");
+ Objects.requireNonNull(application, "application cannot be null");
+ this.platform = platform;
+ this.application = application;
}
- /** A change to the application package version of an application */
- public static class ApplicationChange extends Change {
-
- private final ApplicationVersion version;
-
- private ApplicationChange(ApplicationVersion version) {
- Objects.requireNonNull(version, "version cannot be null");
- this.version = version;
- }
-
- @Override
- public boolean isPresent() { return true; }
-
- /** The application package version in this change, or empty if not known yet */
- public ApplicationVersion version() { return version; }
-
- @Override
- public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) {
- return ! deploymentSpec.canChangeRevisionAt(instant);
- }
-
- @Override
- public int hashCode() { return version.hashCode(); }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) return true;
- if ( ! (other instanceof ApplicationChange)) return false;
- return ((ApplicationChange)other).version.equals(this.version);
- }
-
- /**
- * Creates an application change which we don't know anything about.
- * We are notified that a change has occurred by completion of the component job
- * but do not get to know about what the change is until a subsequent deployment
- * happens.
- */
- public static ApplicationChange unknown() {
- return new ApplicationChange(ApplicationVersion.unknown);
- }
-
- public static ApplicationChange of(ApplicationVersion version) {
- return new ApplicationChange(version);
- }
-
- @Override
- public String toString() {
- return "application change to " + version;
- }
-
+ /** Returns true if this change is blocked by the given spec at the given instant */
+ public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) {
+ if (platform.isPresent() && ! deploymentSpec.canUpgradeAt(instant)) return true;
+ if (application.isPresent() && ! deploymentSpec.canChangeRevisionAt(instant)) return true;
+ return false;
}
- /** A change to the Vespa version running an application */
- public static class VersionChange extends Change {
-
- private final Version version;
+ /** Returns whether a change shoudl currently be deployed */
+ public boolean isPresent() {
+ return platform.isPresent() || application.isPresent();
+ }
- public VersionChange(Version version) {
- Objects.requireNonNull(version, "version cannot be null");
- this.version = version;
- }
+ /** Returns the platform version change which should currently be deployed, if any */
+ public Optional<Version> platform() { return platform; }
- @Override
- public boolean isPresent() { return true; }
+ /** Returns the application version change which should currently be deployed, if any */
+ public Optional<ApplicationVersion> application() { return application; }
- /** The Vespa version this changes to */
- public Version version() { return version; }
+ /** Returns an instance representing no change */
+ public static Change empty() { return empty; }
- @Override
- public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) {
- return ! deploymentSpec.canUpgradeAt(instant);
- }
+ /** Returns a version of this change which replaces or adds this application change */
+ public Change with(ApplicationVersion applicationVersion) {
+ return new Change(platform, Optional.of(applicationVersion));
+ }
- @Override
- public int hashCode() { return version.hashCode(); }
+ @Override
+ public int hashCode() { return Objects.hash(platform, application); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) return true;
+ if ( ! (other instanceof Change)) return false;
+ Change o = (Change)other;
+ if ( ! o.platform.equals(this.platform)) return false;
+ if ( ! o.application.equals(this.application)) return false;
+ return true;
+ }
- @Override
- public boolean equals(Object other) {
- if (this == other) return true;
- if ( ! (other instanceof VersionChange)) return false;
- return ((VersionChange)other).version.equals(this.version);
- }
+ @Override
+ public String toString() {
+ String platformString = platform.map(v -> "upgrade to " + v).orElse(null);
+ String applicationString = application.map(v -> "application change to " + v).orElse(null);
+ if (platformString != null && applicationString != null)
+ return platformString + " and " + applicationString;
+ if (platformString != null)
+ return platformString;
+ if (applicationString != null)
+ return applicationString;
+ return "no change";
+ }
- @Override
- public String toString() {
- return "version change to " + version;
- }
+ public static Change of(ApplicationVersion applicationVersion) {
+ return new Change(Optional.empty(), Optional.of(applicationVersion));
+ }
+ public static Change of(Version platformChange) {
+ return new Change(Optional.of(platformChange), Optional.empty());
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index 542343d66c0..bb7b39eed0f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -75,7 +75,7 @@ public class DeploymentJobs {
if (job == null) job = JobStatus.initial(jobType);
return job.withTriggering(version,
applicationVersion,
- change instanceof Change.VersionChange,
+ change.platform().isPresent(),
reason,
triggerTime);
});
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
index 10463e6d296..e165d3c9fe5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
@@ -209,12 +209,9 @@ public class JobStatus {
// TODO: Consider a version and application version for each JobStatus, to compare against a Target (instead of Change, which is, really, a Target).
/** Returns whether the job last completed for the given change */
public boolean lastCompletedWas(Change change) {
- if (change instanceof Change.ApplicationChange) {
- return ((Change.ApplicationChange) change).version().equals(applicationVersion);
- } else if (change instanceof Change.VersionChange) {
- return version().equals(((Change.VersionChange) change).version());
- }
- throw new IllegalArgumentException("Unexpected change: " + change.getClass());
+ if (change.platform().isPresent() && ! change.platform().get().equals(version())) return false;
+ if (change.application().isPresent() && ! change.application().get().equals(applicationVersion)) return false;
+ return true;
}
@Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 18838f243ad..f3aeeabd3d4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.ApplicationList;
import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.Change.VersionChange;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
@@ -87,16 +86,11 @@ public class DeploymentTrigger {
if (report.jobType() == JobType.component) {
if (acceptNewApplicationVersionNow(application)) {
// Set this as the change we are doing, unless we are already pushing a platform change
- if ( ! ( application.deploying() instanceof Change.VersionChange)) {
- Change.ApplicationChange applicationChange = Change.ApplicationChange.unknown();
- // TODO: Remove guard when source is always reported by component
- if (report.sourceRevision().isPresent()) {
- applicationChange = Change.ApplicationChange.of(
- ApplicationVersion.from(report.sourceRevision().get(),
- report.buildNumber())
- );
- }
- application = application.withDeploying(applicationChange);
+ if ( ! ( application.deploying().platform().isPresent())) {
+ ApplicationVersion applicationVersion = ApplicationVersion.unknown;
+ if (report.sourceRevision().isPresent())
+ applicationVersion = ApplicationVersion.from(report.sourceRevision().get(), report.buildNumber());
+ application = application.withDeploying(Change.of(applicationVersion));
}
}
else { // postpone
@@ -140,11 +134,11 @@ public class DeploymentTrigger {
if (deployment == null) return false;
// Check actual job outcome (the deployment)
- if (change instanceof VersionChange) {
- if (((VersionChange)change).version().isAfter(deployment.version())) return false; // later is ok
+ if (change.platform().isPresent()) {
+ if (change.platform().get().isAfter(deployment.version())) return false; // later is ok
}
- else if (((Change.ApplicationChange)change).version() != ApplicationVersion.unknown) {
- if ( ! ((Change.ApplicationChange)change).version().equals(deployment.applicationVersion())) return false;
+ else if (change.application().isPresent() && change.application().get() != ApplicationVersion.unknown) { // TODO: Should not be "else"
+ if ( ! change.application().get().equals(deployment.applicationVersion())) return false;
}
else {
return false; // If we don't yet know the application version we are deploying, then we are not complete
@@ -172,8 +166,8 @@ public class DeploymentTrigger {
// Should the first step be triggered?
if ( ! jobs.isEmpty() && jobs.get(0).equals(JobType.systemTest) ) {
JobStatus systemTestStatus = application.deploymentJobs().jobStatus().get(JobType.systemTest);
- if (application.deploying() instanceof Change.VersionChange) {
- Version target = ((Change.VersionChange) application.deploying()).version();
+ if (application.deploying().platform().isPresent()) {
+ Version target = application.deploying().platform().get();
if (systemTestStatus == null
|| ! systemTestStatus.lastTriggered().isPresent()
|| ! systemTestStatus.isSuccess()
@@ -219,10 +213,8 @@ public class DeploymentTrigger {
if ( ! application.deploying().isPresent()) return false;
if (next == null) return true;
- Change change = application.deploying();
-
- if (change instanceof Change.VersionChange) { // Propagate upgrade while making sure we never downgrade
- Version targetVersion = ((Change.VersionChange)change).version();
+ if (application.deploying().platform().isPresent()) { // Propagate upgrade while making sure we never downgrade
+ Version targetVersion = application.deploying().platform().get();
if (next.type().isTest()) {
// Is it not yet this job's turn to upgrade?
@@ -271,10 +263,9 @@ public class DeploymentTrigger {
throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " +
application.deploying() + " is already in progress");
application = application.withDeploying(change);
- if (change instanceof Change.ApplicationChange)
+ if (change.application().isPresent())
application = application.withOutstandingChange(false);
- application = trigger(JobType.systemTest, application, false,
- (change instanceof Change.VersionChange ? "Upgrading to " + ((Change.VersionChange)change).version() : "Deploying " + change));
+ application = trigger(JobType.systemTest, application, false, change.toString());
applications().store(application);
});
}
@@ -381,8 +372,8 @@ public class DeploymentTrigger {
application.deploying().blockedBy(application.deploymentSpec(), clock.instant())) return false;
// Don't downgrade or redeploy the same version in production needlessly
- if (application.deploying() instanceof VersionChange &&
- jobType.isProduction() && alreadyDeployed(((VersionChange) application.deploying()).version(), application, jobType)) return false;
+ if (application.deploying().platform().isPresent() &&
+ jobType.isProduction() && alreadyDeployed((application.deploying().platform().get()), application, jobType)) return false;
if (application.deploymentJobs().isRunning(jobType, jobTimeoutLimit())) return false;
if ( ! hasJob(jobType, application)) return false;
@@ -418,7 +409,7 @@ public class DeploymentTrigger {
private boolean acceptNewApplicationVersionNow(LockedApplication application) {
if ( ! application.deploying().isPresent()) return true;
- if (application.deploying() instanceof Change.ApplicationChange) return true; // more changes are ok
+ if (application.deploying().application().isPresent()) return true; // more application changes are ok
if (application.deploymentJobs().hasFailures()) return true; // allow changes to fix upgrade problems
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
index 4485a603f61..0752b90eca9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
@@ -4,17 +4,18 @@ package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.application.Change;
import java.time.Duration;
/**
* Deploys application changes which have been postponed due to an ongoing upgrade
- *
+ *
* @author bratseth
*/
public class OutstandingChangeDeployer extends Maintainer {
-
+
public OutstandingChangeDeployer(Controller controller, Duration interval, JobControl jobControl) {
super(controller, interval, jobControl);
}
@@ -25,7 +26,7 @@ public class OutstandingChangeDeployer extends Maintainer {
for (Application application : applications.asList()) {
if (application.hasOutstandingChange() && ! application.deploying().isPresent())
controller().applications().deploymentTrigger().triggerChange(application.id(),
- Change.ApplicationChange.unknown());
+ Change.of(ApplicationVersion.unknown));
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index 5b87f9eaa86..75f348904dd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -20,7 +20,7 @@ import java.util.logging.Logger;
/**
* Maintenance job which schedules applications for Vespa version upgrade
- *
+ *
* @author bratseth
* @author mpolden
*/
@@ -39,7 +39,7 @@ public class Upgrader extends Maintainer {
* Schedule application upgrades. Note that this implementation must be idempotent.
*/
@Override
- public void maintain() {
+ public void maintain() {
// Determine target versions for each upgrade policy
Optional<Version> canaryTarget = controller().versionStatus().systemVersion().map(VespaVersion::versionNumber);
Optional<Version> defaultTarget = newestVersionWithConfidence(VespaVersion.Confidence.normal);
@@ -66,26 +66,25 @@ public class Upgrader extends Maintainer {
defaultTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.defaultPolicy), target));
conservativeTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.conservative), target));
}
-
+
private Optional<Version> newestVersionWithConfidence(VespaVersion.Confidence confidence) {
return reversed(controller().versionStatus().versions()).stream()
.filter(v -> v.confidence().equalOrHigherThan(confidence))
.findFirst()
.map(VespaVersion::versionNumber);
}
-
+
private List<VespaVersion> reversed(List<VespaVersion> versions) {
List<VespaVersion> reversed = new ArrayList<>(versions.size());
for (int i = 0; i < versions.size(); i++)
reversed.add(versions.get(versions.size() - 1 - i));
return reversed;
}
-
+
/** Returns a list of all applications */
private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); }
-
+
private void upgrade(ApplicationList applications, Version version) {
- Change.VersionChange change = new Change.VersionChange(version);
applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade
applications = applications.hasProductionDeployment();
applications = applications.onLowerVersionThan(version);
@@ -96,7 +95,7 @@ public class Upgrader extends Maintainer {
applications = applications.first(numberOfApplicationsToUpgrade()); // throttle upgrades
for (Application application : applications.asList()) {
try {
- controller().applications().deploymentTrigger().triggerChange(application.id(), change);
+ controller().applications().deploymentTrigger().triggerChange(application.id(), Change.of(version));
} catch (IllegalArgumentException e) {
log.log(Level.INFO, "Could not trigger change: " + Exceptions.toMessageString(e));
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index b626fda5be1..41641a59363 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -247,10 +247,10 @@ public class ApplicationSerializer {
if ( ! deploying.isPresent()) return;
Cursor object = parentObject.setObject(deployingField);
- if (deploying instanceof Change.VersionChange)
- object.setString(versionField, ((Change.VersionChange)deploying).version().toString());
- else if (((Change.ApplicationChange)deploying).version() != ApplicationVersion.unknown)
- toSlime(((Change.ApplicationChange)deploying).version(), object);
+ if (deploying.platform().isPresent())
+ object.setString(versionField, deploying.platform().get().toString());
+ if (deploying.application().isPresent() && deploying.application().get() != ApplicationVersion.unknown)
+ toSlime(deploying.application().get(), object);
}
// ------------------ Deserialization
@@ -366,12 +366,14 @@ public class ApplicationSerializer {
private Change changeFromSlime(Inspector object) {
if ( ! object.valid()) return Change.empty();
Inspector versionFieldValue = object.field(versionField);
+ Change change = Change.empty();
if (versionFieldValue.valid())
- return new Change.VersionChange(Version.fromString(versionFieldValue.asString()));
- else if (object.field(applicationPackageHashField).valid())
- return Change.ApplicationChange.of(applicationVersionFromSlime(object));
- else
- return Change.ApplicationChange.unknown();
+ change = Change.of(Version.fromString(versionFieldValue.asString()));
+ if (object.field(applicationPackageHashField).valid())
+ change = change.with(applicationVersionFromSlime(object));
+ if ( ! change.isPresent()) // A deploy object with no fields -> unknown application change
+ change = Change.of(ApplicationVersion.unknown);
+ return change;
}
private List<JobStatus> jobStatusListFromSlime(Inspector array) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 00f1fe3790c..c262782ae37 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -350,10 +350,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
// Currently deploying change
if (application.deploying().isPresent()) {
Cursor deployingObject = object.setObject("deploying");
- if (application.deploying() instanceof Change.VersionChange)
- deployingObject.setString("version", ((Change.VersionChange)application.deploying()).version().toString());
- else if (((Change.ApplicationChange)application.deploying()).version() != ApplicationVersion.unknown)
- toSlime(((Change.ApplicationChange)application.deploying()).version(), deployingObject.setObject("revision"));
+ application.deploying().platform().ifPresent(v -> deployingObject.setString("version", v.toString()));
+ application.deploying().application()
+ .filter(v -> v != ApplicationVersion.unknown)
+ .ifPresent(v -> toSlime(v, deployingObject.setObject("revision")));
}
// Jobs sorted according to deployment spec
@@ -715,7 +715,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " +
application.deploying() + " is in progress");
- controller.applications().deploymentTrigger().triggerChange(application.id(), new Change.VersionChange(version));
+ controller.applications().deploymentTrigger().triggerChange(application.id(), Change.of(version));
});
return new MessageResponse("Triggered deployment of application '" + id + "' on version " + version);
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 3b4c3dda191..8d3fe4dbbfc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -103,12 +103,12 @@ public class ControllerTest {
tester.notifyJobCompletion(component, app1, true);
assertEquals("Application version is currently not known",
ApplicationVersion.unknown,
- ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying()).version());
+ tester.controller().applications().require(app1.id()).deploying().application().get());
tester.deployAndNotify(app1, applicationPackage, true, systemTest);
tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
- ApplicationVersion applicationVersion = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying()).version();
+ ApplicationVersion applicationVersion = tester.controller().applications().require(app1.id()).deploying().application().get();
assertTrue("Application version has been set during deployment", applicationVersion != ApplicationVersion.unknown);
assertStatus(JobStatus.initial(stagingTest)
.withTriggering(version1, applicationVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1)))
@@ -221,9 +221,9 @@ public class ControllerTest {
tester.artifactRepository().put(app1.id(), applicationPackage, expectedVersionString);
tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 37);
ApplicationVersion expectedVersion = ApplicationVersion.from(source, 37);
- assertEquals(expectedVersionString, ((Change.ApplicationChange) tester.controller().applications()
- .require(app1.id())
- .deploying()).version().id());
+ assertEquals(expectedVersionString, tester.controller().applications()
+ .require(app1.id())
+ .deploying().application().get().id());
// Deploy without application package
tester.deployAndNotify(app1, true, systemTest);
@@ -380,8 +380,7 @@ public class ControllerTest {
assertFalse("Change deployed", app1.deploying().isPresent());
// Version upgrade changes system version
- Change.VersionChange change = new Change.VersionChange(newSystemVersion);
- applications.deploymentTrigger().triggerChange(app1.id(), change);
+ applications.deploymentTrigger().triggerChange(app1.id(), Change.of(newSystemVersion));
tester.deployAndNotify(app1, applicationPackage, true, systemTest);
tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
@@ -669,13 +668,13 @@ public class ControllerTest {
Application app = tester.createApplication(tenant, "app1", "default", 1);
tester.controller().applications().lockOrThrow(app.id(), application -> {
- application = application.withDeploying(new Change.VersionChange(Version.fromString("6.3")));
+ application = application.withDeploying(Change.of(Version.fromString("6.3")));
applications.store(application);
try {
tester.deploy(app, ZoneId.from("prod", "us-east-3"));
fail("Expected exception");
} catch (IllegalArgumentException e) {
- assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as version change to 6.3 is not tested", e.getMessage());
+ assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as upgrade to 6.3 is not tested", e.getMessage());
}
});
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 247ec83aff2..dc8bd3e6423 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -94,12 +94,6 @@ public class DeploymentTester {
return controller().applications().require(application);
}
- public Optional<Change.VersionChange> versionChange(ApplicationId application) {
- Change change = application(application).deploying();
- if (change instanceof Change.VersionChange) return Optional.of((Change.VersionChange)change);
- return Optional.empty();
- }
-
public void updateVersionStatus() {
controller().updateVersionStatus(VersionStatus.compute(controller(), tester.controller().systemVersion()));
}
@@ -226,7 +220,7 @@ public class DeploymentTester {
public void completeUpgrade(Application application, Version version, ApplicationPackage applicationPackage) {
assertTrue(application + " has a deployment", applications().require(application.id()).deploying().isPresent());
- assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying());
+ assertEquals(Change.of(version), applications().require(application.id()).deploying());
completeDeployment(application, applicationPackage, Optional.empty(), true);
}
@@ -240,7 +234,7 @@ public class DeploymentTester {
private void completeUpgradeWithError(Application application, Version version, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) {
assertTrue(applications().require(application.id()).deploying().isPresent());
- assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying());
+ assertEquals(Change.of(version), applications().require(application.id()).deploying());
completeDeployment(application, applicationPackage, failOnJob, true);
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 4bc64a8f213..73c6f1c1126 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -320,7 +320,7 @@ public class DeploymentTriggerTest {
new JobControl(tester.controllerTester().curator()));
LockedApplication app = (LockedApplication)tester.createAndDeploy("default0", 3, "default");
// Store that we are upgrading but don't start the system-tests job
- tester.controller().applications().store(app.withDeploying(new Change.VersionChange(Version.fromString("6.2"))));
+ tester.controller().applications().store(app.withDeploying(Change.of(Version.fromString("6.2"))));
assertEquals(0, tester.buildSystem().jobs().size());
readyJobsTrigger.run();
assertEquals(1, tester.buildSystem().jobs().size());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index b200e2d7e18..b89fcec11a5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -58,7 +58,7 @@ public class FailureRedeployerTest {
tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress
tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size());
- assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+ assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).deploying().platform().get());
// Another version is released, which cancels any pending upgrades to lower versions
version = Version.fromString("5.2");
@@ -66,7 +66,7 @@ public class FailureRedeployerTest {
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); // Finish previous production job.
tester.upgrader().maintain();
assertEquals("Application starts upgrading to new version", 1, tester.buildSystem().jobs().size());
- assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+ assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).deploying().platform().get());
// Failure redeployer does not retry failing job for prod.us-east-3 as there's an ongoing deployment
tester.clock().advance(Duration.ofMinutes(1));
@@ -150,7 +150,7 @@ public class FailureRedeployerTest {
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
- assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+ assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).deploying().platform().get());
// system-test fails and exhausts all immediate retries
tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest);
@@ -164,7 +164,7 @@ public class FailureRedeployerTest {
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
- assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+ assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).deploying().platform().get());
// Consume system-test job for 5.2
tester.buildSystem().takeJobsToRun();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
index d904e1f87c0..690e5c6d512 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
@@ -31,9 +31,9 @@ public class OutstandingChangeDeployerTest {
tester.createAndDeploy("app2", 22, "default");
Version version = new Version(6, 2);
- tester.deploymentTrigger().triggerChange(tester.application("app1").id(), new Change.VersionChange(version));
+ tester.deploymentTrigger().triggerChange(tester.application("app1").id(), Change.of(version));
- assertEquals(new Change.VersionChange(version), tester.application("app1").deploying());
+ assertEquals(Change.of(version), tester.application("app1").deploying());
assertFalse(tester.application("app1").hasOutstandingChange());
tester.notifyJobCompletion(DeploymentJobs.JobType.component, tester.application("app1"), true);
assertTrue(tester.application("app1").hasOutstandingChange());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 63043d39b55..c435aed96ee 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
@@ -174,11 +173,11 @@ public class UpgraderTest {
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size());
- assertEquals(version54, ((Change.VersionChange)tester.application(default0.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying()).version());
+ assertEquals(version54, tester.application(default0.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default1.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default2.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default3.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default4.id()).deploying().platform().get());
tester.completeUpgrade(default0, version54, "default");
// State: Default applications started upgrading to 5.4 (and one completed)
Version version55 = Version.fromString("5.5");
@@ -190,11 +189,11 @@ public class UpgraderTest {
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size());
- assertEquals(version55, ((Change.VersionChange)tester.application(default0.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying()).version());
- assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying()).version());
+ assertEquals(version55, tester.application(default0.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default1.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default2.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default3.id()).deploying().platform().get());
+ assertEquals(version54, tester.application(default4.id()).deploying().platform().get());
tester.completeUpgrade(default1, version54, "default");
tester.completeUpgrade(default2, version54, "default");
tester.completeUpgradeWithError(default3, version54, "default", DeploymentJobs.JobType.stagingTest);
@@ -216,7 +215,7 @@ public class UpgraderTest {
assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken: " +
"This is default3 since it failed upgrade on both 5.4 and 5.5",
1, tester.buildSystem().jobs().size());
- assertEquals("5.4", ((Change.VersionChange)tester.application(default3.id()).deploying()).version().toString());
+ assertEquals("5.4", tester.application(default3.id()).deploying().platform().get().toString());
}
@Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index f674e7be02e..46389b288c4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -82,7 +82,7 @@ public class ApplicationSerializerTest {
deploymentSpec,
validationOverrides,
deployments, deploymentJobs,
- new Change.VersionChange(Version.fromString("6.7")),
+ Change.of(Version.fromString("6.7")),
true,
Optional.of(IssueId.from("1234")),
new MetricsService.ApplicationMetrics(0.5, 0.9),
@@ -145,18 +145,18 @@ public class ApplicationSerializerTest {
assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
{ // test more deployment serialization cases
- Application original2 = writable(original).withDeploying(Change.ApplicationChange.of(ApplicationVersion.from("hash1")));
+ Application original2 = writable(original).withDeploying(Change.of(ApplicationVersion.from("hash1")));
Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2));
assertEquals(original2.deploying(), serialized2.deploying());
- assertEquals(((Change.ApplicationChange)serialized2.deploying()).version().source(),
- ((Change.ApplicationChange)original2.deploying()).version().source());
+ assertEquals(serialized2.deploying().application().get().source(),
+ original2.deploying().application().get().source());
- Application original3 = writable(original).withDeploying(Change.ApplicationChange.of(ApplicationVersion.from("hash1",
- new SourceRevision("a", "b", "c"))));
+ Application original3 = writable(original).withDeploying(Change.of(ApplicationVersion.from("hash1",
+ new SourceRevision("a", "b", "c"))));
Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3));
assertEquals(original3.deploying(), serialized2.deploying());
- assertEquals(((Change.ApplicationChange)serialized3.deploying()).version().source(),
- ((Change.ApplicationChange)original3.deploying()).version().source());
+ assertEquals(serialized3.deploying().application().get().source(),
+ original3.deploying().application().get().source());
Application original4 = writable(original).withDeploying(Change.empty());
Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
index d1e1ebe94fd..3b6d8ed71e9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
@@ -1 +1 @@
-{"message":"Cancelled version change to 6.1 for application 'tenant1.application1'"}
+{"message":"Cancelled upgrade to 6.1 for application 'tenant1.application1'"}