diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2020-07-21 15:38:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-07-21 15:38:29 +0200 |
commit | 58d31227d188ee6a11b545851d6b152766cbc7cf (patch) | |
tree | 36f2c0fa17f3ed5c3b82ebeac20467d80d8694ec /controller-server | |
parent | 80ff9faa96b23bc56353c7233ad659655a8b0dd8 (diff) | |
parent | 1034436255c66d6b00b7ee6dc1c38f71701766b0 (diff) |
Merge pull request #13929 from vespa-engine/freva/delete-all-deployments
Add endpoint to delete all prod deployments
Diffstat (limited to 'controller-server')
9 files changed, 106 insertions, 19 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index 083984bd13c..d0e8a0f2816 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -5,11 +5,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; @@ -19,7 +21,10 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UncheckedIOException; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -184,4 +189,27 @@ public class ApplicationPackage { } + /** Creates a valid application package that will remove all application's deployments */ + public static ApplicationPackage deploymentRemoval() { + DeploymentSpec deploymentSpec = DeploymentSpec.empty; + ValidationOverrides validationOverrides = allValidationOverrides(); + try (ZipBuilder zipBuilder = new ZipBuilder(deploymentSpec.xmlForm().length() + validationOverrides.xmlForm().length() + 500)) { + zipBuilder.add("validation-overrides.xml", validationOverrides.xmlForm().getBytes(UTF_8)); + zipBuilder.add("deployment.xml", deploymentSpec.xmlForm().getBytes(UTF_8)); + + zipBuilder.close(); + return new ApplicationPackage(zipBuilder.toByteArray()); + } + } + + private static ValidationOverrides allValidationOverrides() { + String until = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().plus(Duration.ofDays(25)).atZone(ZoneOffset.UTC)); + StringBuilder validationOverridesContents = new StringBuilder(1000); + validationOverridesContents.append("<validation-overrides version=\"1.0\">\n"); + for (ValidationId validationId: ValidationId.values()) + validationOverridesContents.append("\t<allow until=\"").append(until).append("\">").append(validationId.value()).append("</allow>\n"); + validationOverridesContents.append("</validation-overrides>\n"); + + return ValidationOverrides.fromXml(validationOverridesContents.toString()); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java index b245718171f..3e72936575d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java @@ -51,7 +51,8 @@ public class ApplicationPackageValidator { /** Verify that we have the security/clients.pem file for public systems */ private void validateSecurityClientsPem(ApplicationPackage applicationPackage) { - if (controller.system().isPublic() && applicationPackage.trustedCertificates().isEmpty()) + if (!controller.system().isPublic() || applicationPackage.deploymentSpec().steps().isEmpty()) return; + if (applicationPackage.trustedCertificates().isEmpty()) throw new IllegalArgumentException("Missing required file 'security/clients.pem'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 9a5f9b1074a..ac19b5dd5e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -47,7 +47,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -58,7 +57,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; -import static java.util.stream.Collectors.toUnmodifiableMap; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -379,8 +377,8 @@ public class JobController { * Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, - Optional<String> sourceUrl, Optional<String> commit, - long projectId, ApplicationPackage applicationPackage, byte[] testPackageBytes) { + Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, + byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockApplicationOrThrow(id, application -> { long run = 1 + application.get().latestVersion() @@ -390,7 +388,7 @@ public class JobController { applicationPackage.compileVersion(), applicationPackage.buildTime(), sourceUrl, - commit)); + revision.map(SourceRevision::commit))); controller.applications().applicationStore().put(id.tenant(), id.application(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index f2897f856ce..15d34828734 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -284,6 +284,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); @@ -1901,12 +1902,18 @@ public class ApplicationApiHandler extends LoggingRequestHandler { sourceRevision, authorEmail, sourceUrl, - commit, projectId, applicationPackage, dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } + private HttpResponse removeAllProdDeployments(String tenant, String application) { + JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, + Optional.empty(), Optional.empty(), Optional.empty(), 1, + ApplicationPackage.deploymentRemoval(), new byte[0]); + return new MessageResponse("All deployments removed"); + } + private static Map<String, byte[]> parseDataParts(HttpRequest request) { String contentHash = request.getHeader("x-Content-Hash"); if (contentHash == null) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index f72439b694a..0e25ee1fe85 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -525,13 +525,12 @@ class JobControllerApiHandlerHelper { */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, - Optional<String> sourceUrl, Optional<String> commit, long projectId, + Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), sourceRevision, authorEmail, sourceUrl, - commit, projectId, applicationPackage, testPackage); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java new file mode 100644 index 00000000000..f7f0c9ce58e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java @@ -0,0 +1,27 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; +import org.junit.Test; + +import java.time.Instant; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author valerijf + */ +public class ApplicationPackageTest { + @Test + public void test_createEmptyForDeploymentRemoval() { + ApplicationPackage app = ApplicationPackage.deploymentRemoval(); + assertEquals(DeploymentSpec.empty, app.deploymentSpec()); + assertEquals(List.of(), app.trustedCertificates()); + + for (ValidationId validationId : ValidationId.values()) { + assertTrue(app.validationOverrides().allows(validationId, Instant.now())); + } + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 7a5853c08da..d90eb715499 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -244,7 +244,7 @@ public class DeploymentContext { .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), - Optional.empty(), projectId, applicationPackage, new byte[0]); + projectId, applicationPackage, new byte[0]); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index fe33d728b7c..88eab642a60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -91,7 +91,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); try { @@ -122,7 +122,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -229,7 +229,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -266,7 +266,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId instanceId = appId.defaultInstance(); JobId jobId = new JobId(instanceId, systemTest); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { @@ -361,7 +361,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); @@ -378,7 +378,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); for (RunStatus status : RunStatus.values()) { if (status == success) continue; // Status not used for steps. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 388ca65dc40..10682218353 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -70,11 +70,9 @@ import org.junit.Before; import org.junit.Test; import java.io.File; -import java.math.BigDecimal; import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.time.YearMonth; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; @@ -84,7 +82,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -975,6 +972,36 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test + public void testRemovingAllDeployments() { + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") + .region("us-west-1") + .region("us-east-3") + .region("eu-west-1") + .endpoint("eu", "default", "eu-west-1") + .endpoint("default", "default", "us-west-1", "us-east-3") + .build(); + + deploymentTester.controllerTester().createTenant("tenant1", ATHENZ_TENANT_DOMAIN.getName(), 432L); + + // Create tenant and deploy + var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); + app.submit(applicationPackage).deploy(); + tester.controller().jobController().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); + + assertEquals(Set.of(ZoneId.from("prod.us-west-1"), ZoneId.from("prod.us-east-3"), ZoneId.from("prod.eu-west-1"), ZoneId.from("dev.us-east-1")), + app.instance().deployments().keySet()); + + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deployment", DELETE) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"message\":\"All deployments removed\"}"); + + assertEquals(Set.of(ZoneId.from("dev.us-east-1")), app.instance().deployments().keySet()); + } + + @Test public void testErrorResponses() throws Exception { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); |