From 292ac35f8a4bd3ed9dae0fd88a6e30f906cadf28 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 6 Dec 2018 14:38:56 +0100 Subject: Allow operator deployments to prod of known packages --- .../hosted/controller/ApplicationController.java | 38 +++++++++++------- .../restapi/application/ApplicationApiHandler.java | 19 +++++++++ .../restapi/application/ApplicationApiTest.java | 45 ++++++++++++++++++++-- 3 files changed, 86 insertions(+), 16 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index aed2667c811..8108a9e086c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -301,11 +301,18 @@ public class ApplicationController { } } + public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, + Optional applicationPackageFromDeployer, + DeployOptions options) { + return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); + } + /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg // TODO jvenstad: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, Optional applicationPackageFromDeployer, + Optional applicationVersionFromDeployer, DeployOptions options) { if (applicationId.instance().isTester()) throw new IllegalArgumentException("'" + applicationId + "' is a tester application!"); @@ -324,7 +331,7 @@ public class ApplicationController { ApplicationPackage applicationPackage; if (canDeployDirectly) { platformVersion = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); - applicationVersion = ApplicationVersion.unknown; + applicationVersion = applicationVersionFromDeployer.orElse(ApplicationVersion.unknown); applicationPackage = applicationPackageFromDeployer.orElseThrow( () -> new IllegalArgumentException("Application package must be given when deploying to " + zone)); } @@ -343,18 +350,7 @@ public class ApplicationController { ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - try { - applicationPackage = application.get().deploymentJobs().deployedInternally() - ? new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion)) - : new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())); - } - catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. - log.info("Fetching application package for " + applicationId + " from alternate repository; it is now deployed " - + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); - applicationPackage = application.get().deploymentJobs().deployedInternally() - ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())) - : new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion)); - } + applicationPackage = getApplicationPackage(application.get(), applicationVersion); validateRun(application.get(), zone, platformVersion, applicationVersion); } @@ -386,6 +382,22 @@ public class ApplicationController { } } + /** Fetches the requested application package from the artifact store(s). */ + public ApplicationPackage getApplicationPackage(Application application, ApplicationVersion version) { + try { + return application.deploymentJobs().deployedInternally() + ? new ApplicationPackage(applicationStore.get(application.id(), version)) + : new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), version.id())); + } + catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. + log.info("Fetching application package for " + application.id() + " from alternate repository; it is now deployed " + + (application.deploymentJobs().deployedInternally() ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); + return application.deploymentJobs().deployedInternally() + ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), version.id())) + : new ApplicationPackage(applicationStore.get(application.id(), version)); + } + } + /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ public LockedApplication storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { validate(applicationPackage.deploymentSpec()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index aa0d4857818..1374dbb83e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -18,6 +18,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -81,6 +82,7 @@ import com.yahoo.yolean.Exceptions; import javax.ws.rs.ForbiddenException; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotAuthorizedException; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -94,6 +96,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Scanner; import java.util.logging.Level; @@ -831,6 +834,21 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional applicationPackage = Optional.ofNullable(dataParts.get("applicationZip")) .map(ApplicationPackage::new); + Inspector sourceRevision = deployOptions.field("sourceRevision"); + Inspector buildNumber = deployOptions.field("buildNumber"); + if (sourceRevision.valid() != buildNumber.valid()) + throw new IllegalArgumentException("Source revision and build number must both be provided, or not"); + + Optional applicationVersion = Optional.empty(); + if (sourceRevision.valid()) { + if (applicationPackage.isPresent()) + throw new IllegalArgumentException("Application version and application package can't both be provided."); + + applicationVersion = Optional.of(ApplicationVersion.from(toSourceRevision(sourceRevision).get(), + buildNumber.asLong())); + applicationPackage = Optional.of(controller.applications().getApplicationPackage(controller.applications().require(applicationId), applicationVersion.get())); + } + // TODO: get rid of the json object DeployOptions deployOptionsJsonClass = new DeployOptions(deployOptions.field("deployDirectly").asBool(), optional("vespaVersion", deployOptions).map(Version::new), @@ -839,6 +857,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ActivateResult result = controller.applications().deploy(applicationId, zone, applicationPackage, + applicationVersion, deployOptionsJsonClass); return new SlimeJsonResponse(toSlime(result)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index b6cf05d6990..6467b50a4cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -27,12 +27,15 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; @@ -206,6 +209,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("deploy-result.json")); + // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. ApplicationId id = ApplicationId.from("tenant1", "application1", "default"); long screwdriverProjectId = 123; @@ -259,6 +263,27 @@ public class ApplicationApiTest extends ControllerContainerTest { .unsuccessful() .submit(); + // POST an application deployment to a production zone - operator emergency deployment - fails since package is unknown + entity = createApplicationDeployData(Optional.empty(), + Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, + BuildJob.defaultBuildNumber - 1)), + true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"NullPointerException\"}", + 500); + + // POST an application deployment to a production zone - operator emergency deployment - works with known package + entity = createApplicationDeployData(Optional.empty(), + Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, + BuildJob.defaultBuildNumber)), + true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + new File("deploy-result.json")); + // POST (create) another application ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) @@ -1141,8 +1166,13 @@ public class ApplicationApiTest extends ControllerContainerTest { } private HttpEntity createApplicationDeployData(Optional applicationPackage, boolean deployDirectly) { + return createApplicationDeployData(applicationPackage, Optional.empty(), deployDirectly); + } + + private HttpEntity createApplicationDeployData(Optional applicationPackage, + Optional applicationVersion, boolean deployDirectly) { MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.addTextBody("deployOptions", deployOptions(deployDirectly), ContentType.APPLICATION_JSON); + builder.addTextBody("deployOptions", deployOptions(deployDirectly, applicationVersion), ContentType.APPLICATION_JSON); applicationPackage.ifPresent(ap -> builder.addBinaryBody("applicationZip", ap.zippedContent())); return builder.build(); } @@ -1157,10 +1187,19 @@ public class ApplicationApiTest extends ControllerContainerTest { return builder.build(); } - private String deployOptions(boolean deployDirectly) { + private String deployOptions(boolean deployDirectly, Optional applicationVersion) { return "{\"vespaVersion\":null," + "\"ignoreValidationErrors\":false," + "\"deployDirectly\":" + deployDirectly + + applicationVersion.map(version -> + "," + + "\"buildNumber\":" + version.buildNumber().getAsLong() + "," + + "\"sourceRevision\":{" + + "\"repository\":\"" + version.source().get().repository() + "\"," + + "\"branch\":\"" + version.source().get().branch() + "\"," + + "\"commit\":\"" + version.source().get().commit() + "\"" + + "}" + ).orElse("") + "}"; } -- cgit v1.2.3 From 2291fa04d5cd67bf6fc9e3b7faf15fd352c9b00b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 6 Dec 2018 14:43:19 +0100 Subject: Optimize imports --- .../controller/restapi/application/ApplicationApiHandler.java | 7 ++----- .../hosted/controller/restapi/application/ApplicationApiTest.java | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1374dbb83e9..d90f565b90e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -18,7 +18,6 @@ import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -50,11 +49,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -64,7 +64,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -82,7 +81,6 @@ import com.yahoo.yolean.Exceptions; import javax.ws.rs.ForbiddenException; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotAuthorizedException; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -96,7 +94,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalLong; import java.util.Scanner; import java.util.logging.Level; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 6467b50a4cb..4ced2ef06f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -27,15 +27,13 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; -- cgit v1.2.3