diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2018-12-06 15:01:55 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-06 15:01:55 +0100 |
commit | 1bfae09471539303fa587f7c103baf4eb1c04e48 (patch) | |
tree | 6008092c3b91a7091c18a12aa78dc9439c2efdd8 | |
parent | 0a5f4ae6d09e91d0073924ee7997be57018480c0 (diff) | |
parent | 2291fa04d5cd67bf6fc9e3b7faf15fd352c9b00b (diff) |
Merge pull request #7892 from vespa-engine/jvenstad/allow-emergency-deployments-through-controller
Allow operator deployments to prod of known packages
3 files changed, 82 insertions, 17 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index aed2667c811..8108a9e086c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -301,11 +301,18 @@ public class ApplicationController { } } + public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, + Optional<ApplicationPackage> applicationPackageFromDeployer, + DeployOptions options) { + return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); + } + /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg // TODO jvenstad: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, + Optional<ApplicationVersion> applicationVersionFromDeployer, DeployOptions options) { if (applicationId.instance().isTester()) throw new IllegalArgumentException("'" + applicationId + "' is a tester application!"); @@ -324,7 +331,7 @@ public class ApplicationController { ApplicationPackage applicationPackage; if (canDeployDirectly) { platformVersion = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); - applicationVersion = ApplicationVersion.unknown; + applicationVersion = applicationVersionFromDeployer.orElse(ApplicationVersion.unknown); applicationPackage = applicationPackageFromDeployer.orElseThrow( () -> new IllegalArgumentException("Application package must be given when deploying to " + zone)); } @@ -343,18 +350,7 @@ public class ApplicationController { ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - try { - applicationPackage = application.get().deploymentJobs().deployedInternally() - ? new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion)) - : new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())); - } - catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. - log.info("Fetching application package for " + applicationId + " from alternate repository; it is now deployed " - + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); - applicationPackage = application.get().deploymentJobs().deployedInternally() - ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())) - : new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion)); - } + applicationPackage = getApplicationPackage(application.get(), applicationVersion); validateRun(application.get(), zone, platformVersion, applicationVersion); } @@ -386,6 +382,22 @@ public class ApplicationController { } } + /** Fetches the requested application package from the artifact store(s). */ + public ApplicationPackage getApplicationPackage(Application application, ApplicationVersion version) { + try { + return application.deploymentJobs().deployedInternally() + ? new ApplicationPackage(applicationStore.get(application.id(), version)) + : new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), version.id())); + } + catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. + log.info("Fetching application package for " + application.id() + " from alternate repository; it is now deployed " + + (application.deploymentJobs().deployedInternally() ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); + return application.deploymentJobs().deployedInternally() + ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), version.id())) + : new ApplicationPackage(applicationStore.get(application.id(), version)); + } + } + /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ public LockedApplication storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { validate(applicationPackage.deploymentSpec()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index aa0d4857818..d90f565b90e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,11 +49,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -63,7 +64,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -831,6 +831,21 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<ApplicationPackage> applicationPackage = Optional.ofNullable(dataParts.get("applicationZip")) .map(ApplicationPackage::new); + Inspector sourceRevision = deployOptions.field("sourceRevision"); + Inspector buildNumber = deployOptions.field("buildNumber"); + if (sourceRevision.valid() != buildNumber.valid()) + throw new IllegalArgumentException("Source revision and build number must both be provided, or not"); + + Optional<ApplicationVersion> applicationVersion = Optional.empty(); + if (sourceRevision.valid()) { + if (applicationPackage.isPresent()) + throw new IllegalArgumentException("Application version and application package can't both be provided."); + + applicationVersion = Optional.of(ApplicationVersion.from(toSourceRevision(sourceRevision).get(), + buildNumber.asLong())); + applicationPackage = Optional.of(controller.applications().getApplicationPackage(controller.applications().require(applicationId), applicationVersion.get())); + } + // TODO: get rid of the json object DeployOptions deployOptionsJsonClass = new DeployOptions(deployOptions.field("deployDirectly").asBool(), optional("vespaVersion", deployOptions).map(Version::new), @@ -839,6 +854,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ActivateResult result = controller.applications().deploy(applicationId, zone, applicationPackage, + applicationVersion, deployOptionsJsonClass); return new SlimeJsonResponse(toSlime(result)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index b6cf05d6990..4ced2ef06f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -206,6 +207,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("deploy-result.json")); + // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. ApplicationId id = ApplicationId.from("tenant1", "application1", "default"); long screwdriverProjectId = 123; @@ -259,6 +261,27 @@ public class ApplicationApiTest extends ControllerContainerTest { .unsuccessful() .submit(); + // POST an application deployment to a production zone - operator emergency deployment - fails since package is unknown + entity = createApplicationDeployData(Optional.empty(), + Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, + BuildJob.defaultBuildNumber - 1)), + true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"NullPointerException\"}", + 500); + + // POST an application deployment to a production zone - operator emergency deployment - works with known package + entity = createApplicationDeployData(Optional.empty(), + Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, + BuildJob.defaultBuildNumber)), + true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + new File("deploy-result.json")); + // POST (create) another application ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) @@ -1141,8 +1164,13 @@ public class ApplicationApiTest extends ControllerContainerTest { } private HttpEntity createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, boolean deployDirectly) { + return createApplicationDeployData(applicationPackage, Optional.empty(), deployDirectly); + } + + private HttpEntity createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, + Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.addTextBody("deployOptions", deployOptions(deployDirectly), ContentType.APPLICATION_JSON); + builder.addTextBody("deployOptions", deployOptions(deployDirectly, applicationVersion), ContentType.APPLICATION_JSON); applicationPackage.ifPresent(ap -> builder.addBinaryBody("applicationZip", ap.zippedContent())); return builder.build(); } @@ -1157,10 +1185,19 @@ public class ApplicationApiTest extends ControllerContainerTest { return builder.build(); } - private String deployOptions(boolean deployDirectly) { + private String deployOptions(boolean deployDirectly, Optional<ApplicationVersion> applicationVersion) { return "{\"vespaVersion\":null," + "\"ignoreValidationErrors\":false," + "\"deployDirectly\":" + deployDirectly + + applicationVersion.map(version -> + "," + + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + + "\"sourceRevision\":{" + + "\"repository\":\"" + version.source().get().repository() + "\"," + + "\"branch\":\"" + version.source().get().branch() + "\"," + + "\"commit\":\"" + version.source().get().commit() + "\"" + + "}" + ).orElse("") + "}"; } |