Merge pull request #13929 from vespa-engine/freva/delete-all-deployments

Add endpoint to delete all prod deployments

9 files changed, 106 insertions, 19 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
index 083984bd13c..d0e8a0f2816 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
@@ -5,11 +5,13 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.hash.Hashing;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayInputStream;
@@ -19,7 +21,10 @@ import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UncheckedIOException;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -184,4 +189,27 @@ public class ApplicationPackage {
 
     }
 
+    /** Creates a valid application package that will remove all application's deployments */
+    public static ApplicationPackage deploymentRemoval() {
+        DeploymentSpec deploymentSpec = DeploymentSpec.empty;
+        ValidationOverrides validationOverrides = allValidationOverrides();
+        try (ZipBuilder zipBuilder = new ZipBuilder(deploymentSpec.xmlForm().length() + validationOverrides.xmlForm().length() + 500)) {
+            zipBuilder.add("validation-overrides.xml", validationOverrides.xmlForm().getBytes(UTF_8));
+            zipBuilder.add("deployment.xml", deploymentSpec.xmlForm().getBytes(UTF_8));
+
+            zipBuilder.close();
+            return new ApplicationPackage(zipBuilder.toByteArray());
+        }
+    }
+
+    private static ValidationOverrides allValidationOverrides() {
+        String until = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().plus(Duration.ofDays(25)).atZone(ZoneOffset.UTC));
+        StringBuilder validationOverridesContents = new StringBuilder(1000);
+        validationOverridesContents.append("<validation-overrides version=\"1.0\">\n");
+        for (ValidationId validationId: ValidationId.values())
+            validationOverridesContents.append("\t<allow until=\"").append(until).append("\">").append(validationId.value()).append("</allow>\n");
+        validationOverridesContents.append("</validation-overrides>\n");
+
+        return ValidationOverrides.fromXml(validationOverridesContents.toString());
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
index b245718171f..3e72936575d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
@@ -51,7 +51,8 @@ public class ApplicationPackageValidator {
 
     /** Verify that we have the security/clients.pem file for public systems */
     private void validateSecurityClientsPem(ApplicationPackage applicationPackage) {
-        if (controller.system().isPublic() && applicationPackage.trustedCertificates().isEmpty())
+        if (!controller.system().isPublic() || applicationPackage.deploymentSpec().steps().isEmpty()) return;
+        if (applicationPackage.trustedCertificates().isEmpty())
             throw new IllegalArgumentException("Missing required file 'security/clients.pem'");
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 9a5f9b1074a..ac19b5dd5e0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -47,7 +47,6 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 import java.util.logging.Level;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.google.common.collect.ImmutableList.copyOf;
@@ -58,7 +57,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableList;
-import static java.util.stream.Collectors.toUnmodifiableMap;
 
 /**
  * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs.
@@ -379,8 +377,8 @@ public class JobController {
      * Accepts and stores a new application package and test jar pair under a generated application version key.
      */
     public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail,
-                                     Optional<String> sourceUrl, Optional<String> commit,
-                                     long projectId, ApplicationPackage applicationPackage, byte[] testPackageBytes) {
+                                     Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage,
+                                     byte[] testPackageBytes) {
         AtomicReference<ApplicationVersion> version = new AtomicReference<>();
         controller.applications().lockApplicationOrThrow(id, application -> {
             long run = 1 + application.get().latestVersion()
@@ -390,7 +388,7 @@ public class JobController {
                                                 applicationPackage.compileVersion(),
                                                 applicationPackage.buildTime(),
                                                 sourceUrl,
-                                                commit));
+                                                revision.map(SourceRevision::commit)));
 
             controller.applications().applicationStore().put(id.tenant(),
                                                              id.application(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index f2897f856ce..15d34828734 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -284,6 +284,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all");
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request);
@@ -1901,12 +1902,18 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                             sourceRevision,
                                                             authorEmail,
                                                             sourceUrl,
-                                                            commit,
                                                             projectId,
                                                             applicationPackage,
                                                             dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP));
     }
 
+    private HttpResponse removeAllProdDeployments(String tenant, String application) {
+        JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application,
+                Optional.empty(), Optional.empty(), Optional.empty(), 1,
+                ApplicationPackage.deploymentRemoval(), new byte[0]);
+        return new MessageResponse("All deployments removed");
+    }
+
     private static Map<String, byte[]> parseDataParts(HttpRequest request) {
         String contentHash = request.getHeader("x-Content-Hash");
         if (contentHash == null)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index f72439b694a..0e25ee1fe85 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -525,13 +525,12 @@ class JobControllerApiHandlerHelper {
      */
     static HttpResponse submitResponse(JobController jobController, String tenant, String application,
                                        Optional<SourceRevision> sourceRevision, Optional<String> authorEmail,
-                                       Optional<String> sourceUrl, Optional<String> commit, long projectId,
+                                       Optional<String> sourceUrl, long projectId,
                                        ApplicationPackage applicationPackage, byte[] testPackage) {
         ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application),
                                                           sourceRevision,
                                                           authorEmail,
                                                           sourceUrl,
-                                                          commit,
                                                           projectId,
                                                           applicationPackage,
                                                           testPackage);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java
new file mode 100644
index 00000000000..f7f0c9ce58e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java
@@ -0,0 +1,27 @@
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationId;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author valerijf
+ */
+public class ApplicationPackageTest {
+    @Test
+    public void test_createEmptyForDeploymentRemoval() {
+        ApplicationPackage app = ApplicationPackage.deploymentRemoval();
+        assertEquals(DeploymentSpec.empty, app.deploymentSpec());
+        assertEquals(List.of(), app.trustedCertificates());
+
+        for (ValidationId validationId : ValidationId.values()) {
+            assertTrue(app.validationOverrides().allows(validationId, Instant.now()));
+        }
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 7a5853c08da..d90eb715499 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -244,7 +244,7 @@ public class DeploymentContext {
                               .projectId()
                               .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set.
         lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(),
-                                     Optional.empty(), projectId, applicationPackage, new byte[0]);
+                                     projectId, applicationPackage, new byte[0]);
         return this;
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index fe33d728b7c..88eab642a60 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -91,7 +91,7 @@ public class JobRunnerTest {
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
 
         jobs.start(id, systemTest, versions);
         try {
@@ -122,7 +122,7 @@ public class JobRunnerTest {
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
         Supplier<Run> run = () -> jobs.last(id, systemTest).get();
 
         jobs.start(id, systemTest, versions);
@@ -229,7 +229,7 @@ public class JobRunnerTest {
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
 
         RunId runId = new RunId(id, systemTest, 1);
         jobs.start(id, systemTest, versions);
@@ -266,7 +266,7 @@ public class JobRunnerTest {
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId instanceId = appId.defaultInstance();
         JobId jobId = new JobId(instanceId, systemTest);
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
         assertFalse(jobs.lastSuccess(jobId).isPresent());
 
         for (int i = 0; i < jobs.historyLength(); i++) {
@@ -361,7 +361,7 @@ public class JobRunnerTest {
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
 
         jobs.start(id, systemTest, versions);
         tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1)));
@@ -378,7 +378,7 @@ public class JobRunnerTest {
 
         TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id();
         ApplicationId id = appId.defaultInstance();
-        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
+        jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]);
 
         for (RunStatus status : RunStatus.values()) {
             if (status == success) continue; // Status not used for steps.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 388ca65dc40..10682218353 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -70,11 +70,9 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.io.File;
-import java.math.BigDecimal;
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
-import java.time.YearMonth;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Base64;
@@ -84,7 +82,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.function.Supplier;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
@@ -975,6 +972,36 @@ public class ApplicationApiTest extends ControllerContainerTest {
     }
 
     @Test
+    public void testRemovingAllDeployments() {
+        createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID);
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .instances("instance1")
+                .region("us-west-1")
+                .region("us-east-3")
+                .region("eu-west-1")
+                .endpoint("eu", "default", "eu-west-1")
+                .endpoint("default", "default", "us-west-1", "us-east-3")
+                .build();
+
+        deploymentTester.controllerTester().createTenant("tenant1", ATHENZ_TENANT_DOMAIN.getName(), 432L);
+
+        // Create tenant and deploy
+        var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1");
+        app.submit(applicationPackage).deploy();
+        tester.controller().jobController().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage);
+
+        assertEquals(Set.of(ZoneId.from("prod.us-west-1"), ZoneId.from("prod.us-east-3"), ZoneId.from("prod.eu-west-1"), ZoneId.from("dev.us-east-1")),
+                app.instance().deployments().keySet());
+
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deployment", DELETE)
+                        .userIdentity(USER_ID)
+                        .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT),
+                "{\"message\":\"All deployments removed\"}");
+
+        assertEquals(Set.of(ZoneId.from("dev.us-east-1")), app.instance().deployments().keySet());
+    }
+
+    @Test
     public void testErrorResponses() throws Exception {
         createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID);