Merge pull request #8338 from vespa-engine/smorgrav/redeploy_zone_app

Add support for redeployment for the zone application

4 files changed, 85 insertions, 5 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index f9254e9c467..3efb0db8cce 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -378,14 +378,23 @@ public class ApplicationController {
     /** Deploy a system application to given zone */
     public void deploy(SystemApplication application, ZoneId zone, Version version) {
         if (application.hasApplicationPackage()) {
+            deploySystemApplicationPackage(application, zone, version);
+        } else {
+            // Deploy by calling node repository directly
+            application.nodeTypes().forEach(nodeType -> configServer().nodeRepository().upgrade(zone, nodeType, version));
+        }
+    }
+
+    /** Deploy a system application to given zone */
+    public ActivateResult deploySystemApplicationPackage(SystemApplication application, ZoneId zone, Version version) {
+        if (application.hasApplicationPackage()) {
             ApplicationPackage applicationPackage = new ApplicationPackage(
                     artifactRepository.getSystemApplicationPackage(application.id(), zone, version)
             );
             DeployOptions options = withVersion(version, DeployOptions.none());
-            deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
+            return deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
         } else {
-            // Deploy by calling node repository directly
-            application.nodeTypes().forEach(nodeType -> configServer().nodeRepository().upgrade(zone, nodeType, version));
+           throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString());
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index c1b7f965135..19ff6df3ccb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -65,9 +66,11 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.RotationStatus;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
+import com.yahoo.vespa.hosted.controller.maintenance.InfrastructureUpgrader;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
@@ -92,11 +95,13 @@ import java.time.DayOfWeek;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Scanner;
+import java.util.function.Function;
 import java.util.logging.Level;
 
 import static java.util.stream.Collectors.joining;
@@ -856,12 +861,40 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName);
         ZoneId zone = ZoneId.from(environment, region);
 
+        // Get deployOptions
         Map<String, byte[]> dataParts = new MultipartParser().parse(request);
         if ( ! dataParts.containsKey("deployOptions"))
             return ErrorResponse.badRequest("Missing required form part 'deployOptions'");
-
         Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get();
 
+        /*
+         * Special handling of the zone application (the only system application with an application package)
+         * Setting any other deployOptions here is not supported for now (e.g. specifying version), but
+         * this might be handy later to handle emergency downgrades.
+         */
+        boolean isZoneApplication = SystemApplication.zone.id().equals(applicationId);
+        if (isZoneApplication) {
+            // Make it explicit that version is not yet supported here
+            String versionStr = deployOptions.field("vespaVersion").asString();
+            boolean versionPresent = !versionStr.isEmpty() && !versionStr.equals("null");
+            if (versionPresent) {
+                throw new RuntimeException("Version not supported for system applications");
+            }
+            // To avoid second guessing the orchestrated upgrades of system applications
+            // we don't allow to deploy these during an system upgrade (i.e when new vespa is being rolled out)
+            Version version = wantedSystemVersion(zone, SystemApplication.zone);
+            if (!controller.systemVersion().equals(version)) {
+                throw new RuntimeException("Deployment of system applications during a system upgrade is not allowed");
+            }
+            ActivateResult result = controller.applications()
+                    .deploySystemApplicationPackage(SystemApplication.zone, zone, version);
+            return new SlimeJsonResponse(toSlime(result));
+        }
+
+        /*
+         * Normal applications from here
+         */
+
         Optional<ApplicationPackage> applicationPackage = Optional.ofNullable(dataParts.get("applicationZip"))
                                                                   .map(ApplicationPackage::new);
 
@@ -880,11 +913,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             applicationPackage = Optional.of(controller.applications().getApplicationPackage(controller.applications().require(applicationId), applicationVersion.get()));
         }
 
-
         boolean deployDirectly = deployOptions.field("deployDirectly").asBool();
         Optional<Version> vespaVersion = optional("vespaVersion", deployOptions).map(Version::new);
 
+        /*
+         * Deploy direct is when we want to redeploy the current application - retrieve version
+         * info from the application package before deploying
+         */
         if(deployDirectly && !applicationPackage.isPresent() && !applicationVersion.isPresent() && !vespaVersion.isPresent()) {
+
             // Redeploy the existing deployment with the same versions.
             Optional<Deployment> deployment = controller.applications().get(applicationId)
                     .map(Application::deployments)
@@ -907,15 +944,35 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                                  vespaVersion,
                                                                  deployOptions.field("ignoreValidationErrors").asBool(),
                                                                  deployOptions.field("deployCurrentVersion").asBool());
+
+
         ActivateResult result = controller.applications().deploy(applicationId,
                                                                  zone,
                                                                  applicationPackage,
                                                                  applicationVersion,
                                                                  deployOptionsJsonClass,
                                                                  Optional.of(getUserPrincipal(request).getIdentity()));
+
         return new SlimeJsonResponse(toSlime(result));
     }
 
+    /** Find the minimum value of a version field in a zone */
+    private Version wantedSystemVersion(ZoneId zone, SystemApplication application) {
+        try {
+            return controller.configServer()
+                    .nodeRepository()
+                    .list(zone, application.id())
+                    .stream()
+                    .filter(node -> node.state().equals(Node.State.active))
+                    .map(Node::wantedVersion)
+                    .min(Comparator.naturalOrder()).orElseThrow(
+                            () -> new RuntimeException("System version not found in node repo"));
+        } catch (Exception e) {
+            throw new RuntimeException(String.format("Failed to get version for %s in %s: %s",
+                    application.id(), zone, Exceptions.toMessageString(e)));
+        }
+    }
+
     private HttpResponse deleteTenant(String tenantName, HttpRequest request) {
         Optional<Tenant> tenant = controller.tenants().tenant(tenantName);
         if ( ! tenant.isPresent()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); // NOTE: The Jersey implementation would silently ignore this
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index cd804e2ebfb..7697cf00b86 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -668,6 +668,19 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data(entity)
                                       .screwdriverIdentity(SCREWDRIVER_ID),
                               new File("deploy-result.json"));
+
+        // POST (deploy) a system application with an application package
+        HttpEntity noAppEntity = createApplicationDeployData(Optional.empty(), true);
+        tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST)
+                        .data(noAppEntity)
+                        .userIdentity(USER_ID),
+                new File("deploy-result.json"));
+
+        // POST (deploy) a system application without an application package
+        tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/default/deploy", POST)
+                        .data(noAppEntity)
+                        .userIdentity(USER_ID),
+                new File("deploy-no-deployment.json"), 400);
     }
 
     // Tests deployment to config server when using just on API call
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json
new file mode 100644
index 00000000000..f90420f28d7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json
@@ -0,0 +1 @@
+{"error-code":"BAD_REQUEST","message":"Can't redeploy application, no deployment currently exist"}
\ No newline at end of file