Allow partial success for prepareandactivate API, and retry of activation

6 files changed, 401 insertions, 32 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 64e5f80d72c..80edbac9a43 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -15,6 +15,7 @@ import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ActivationContext;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.EndpointsChecker;
@@ -24,6 +25,7 @@ import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.InfraDeployer;
+import com.yahoo.config.provision.ParentHostUnavailableException;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
@@ -71,6 +73,7 @@ import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SecretStoreValidator;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.http.TesterClient;
+import com.yahoo.vespa.config.server.http.v2.PrepareAndActivateResult;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse;
 import com.yahoo.vespa.config.server.http.v2.response.SearchNodeMetricsResponse;
@@ -363,36 +366,40 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return deployment;
     }
 
-    public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) {
+    public PrepareAndActivateResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) {
         DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams);
         File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile();
         ThreadLockStats threadLockStats = LockStats.getForCurrentThread();
-        PrepareResult prepareResult;
+        PrepareAndActivateResult result;
         try {
             threadLockStats.startRecording("deploy of " + prepareParams.getApplicationId().serializedForm());
-            prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, logger);
+            result = deploy(decompressApplication(in, tempDir), prepareParams, logger);
         } finally {
             threadLockStats.stopRecording();
             cleanupTempDirectory(tempDir, logger);
         }
-        return prepareResult;
+        return result;
     }
 
     public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) {
-        return deploy(applicationPackage, prepareParams, DeployHandlerLogger.forPrepareParams(prepareParams));
+        return deploy(applicationPackage, prepareParams, DeployHandlerLogger.forPrepareParams(prepareParams)).deployResult();
     }
 
-    private PrepareResult deploy(File applicationDir, PrepareParams prepareParams, DeployHandlerLogger logger) {
+    private PrepareAndActivateResult deploy(File applicationDir, PrepareParams prepareParams, DeployHandlerLogger logger) {
         long sessionId = createSession(prepareParams.getApplicationId(),
                                        prepareParams.getTimeoutBudget(),
                                        applicationDir,
                                        logger);
         Deployment deployment = prepare(sessionId, prepareParams, logger);
 
-        if ( ! prepareParams.isDryRun())
+        RuntimeException activationFailure = null;
+        if ( ! prepareParams.isDryRun()) try {
             deployment.activate();
-
-        return new PrepareResult(sessionId, deployment.configChangeActions(), logger);
+        }
+        catch (ParentHostUnavailableException | ApplicationLockException e) {
+            activationFailure = e;
+        }
+        return new PrepareAndActivateResult(new PrepareResult(sessionId, deployment.configChangeActions(), logger), activationFailure);
     }
 
     /**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
index 29f2125ac3c..0823f8f75c2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
@@ -3,6 +3,9 @@ package com.yahoo.vespa.config.server.http.v2;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.annotation.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationLockException;
+import com.yahoo.config.provision.ParentHostUnavailableException;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
@@ -11,7 +14,10 @@ import com.yahoo.container.jdisc.utils.MultiPartFormParser;
 import com.yahoo.container.jdisc.utils.MultiPartFormParser.PartItem;
 import com.yahoo.jdisc.application.BindingMatch;
 import com.yahoo.jdisc.http.HttpHeaders;
+import com.yahoo.restapi.MessageResponse;
+import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.TimeoutBudget;
 import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream;
 import com.yahoo.vespa.config.server.http.BadRequestException;
 import com.yahoo.vespa.config.server.http.SessionHandler;
@@ -56,19 +62,27 @@ public class ApplicationApiHandler extends SessionHandler {
 
     private final TenantRepository tenantRepository;
     private final Duration zookeeperBarrierTimeout;
-    private final Zone zone;
     private final long maxApplicationPackageSize;
 
     @Inject
     public ApplicationApiHandler(Context ctx,
                                  ApplicationRepository applicationRepository,
-                                 ConfigserverConfig configserverConfig,
-                                 Zone zone) {
+                                 ConfigserverConfig configserverConfig) {
         super(ctx, applicationRepository);
         this.tenantRepository = applicationRepository.tenantRepository();
         this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout());
         this.maxApplicationPackageSize = configserverConfig.maxApplicationPackageSize();
-        this.zone = zone;
+    }
+
+    @Override
+    protected HttpResponse handlePUT(HttpRequest request) {
+        TenantName tenantName = validateTenant(request);
+        long sessionId = getSessionIdFromRequest(request);
+        ApplicationId app = applicationRepository.activate(tenantRepository.getTenant(tenantName),
+                                                           sessionId,
+                                                           getTimeoutBudget(request, Duration.ofMinutes(2)),
+                                                           shouldIgnoreSessionStaleFailure(request));
+        return new MessageResponse("Session " + sessionId + " for " + app.toFullString() + " activated");
     }
 
     @Override
@@ -112,8 +126,8 @@ public class ApplicationApiHandler extends SessionHandler {
                 .ifPresent(e -> e.addKeyValue("app.id", prepareParams.getApplicationId().toFullString()));
 
         try (compressedStream) {
-            PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams);
-            return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone);
+            PrepareAndActivateResult result = applicationRepository.deploy(compressedStream, prepareParams);
+            return new SessionPrepareAndActivateResponse(result, prepareParams.getApplicationId());
         }
         catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -132,8 +146,18 @@ public class ApplicationApiHandler extends SessionHandler {
     }
 
     public static TenantName getTenantNameFromRequest(HttpRequest request) {
-        BindingMatch<?> bm = Utils.getBindingMatch(request, "http://*/application/v2/tenant/*/prepareandactivate*");
+        BindingMatch<?> bm = Utils.getBindingMatch(request, "http://*/application/v2/tenant/*/prepareandactivate*"); // Gosh, these glob rules aren't good ...
         return TenantName.from(bm.group(2));
     }
 
+    public static long getSessionIdFromRequest(HttpRequest request) {
+        BindingMatch<?> bm = Utils.getBindingMatch(request, "http://*/application/v2/tenant/*/prepareandactivate/*");
+        try {
+            return Long.parseLong(bm.group(3));
+        }
+        catch (NumberFormatException e) {
+            throw new BadRequestException("Session id '" + bm.group(3) + "' is not a number: " + e.getMessage());
+        }
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareAndActivateResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareAndActivateResult.java
new file mode 100644
index 00000000000..3da3a6752cd
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareAndActivateResult.java
@@ -0,0 +1,21 @@
+package com.yahoo.vespa.config.server.http.v2;
+
+import com.yahoo.config.provision.ParentHostUnavailableException;
+
+/**
+ * Allows a partial deployment success, where the application is prepared, but not activated.
+ * This currently only allows the parent-host-not-ready and application-lock cases, as other transient errors are
+ * thrown too early (LB during prepare, cert during validation), but could be expanded to allow
+ * reuse of a prepared session in the future. In that case, users of this result (handler and its client)
+ * must also be updated.
+ *
+ * @author jonmv
+ */
+public record PrepareAndActivateResult(PrepareResult prepareResult, RuntimeException activationFailure) {
+
+    public PrepareResult deployResult() {
+        if (activationFailure != null) throw activationFailure;
+        return prepareResult;
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java
index 1e6f7dfe45e..11ce933b6f3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java
@@ -8,6 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter;
+import com.yahoo.vespa.config.server.http.v2.PrepareAndActivateResult;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 
 /**
@@ -17,25 +18,19 @@ import com.yahoo.vespa.config.server.http.v2.PrepareResult;
  */
 public class SessionPrepareAndActivateResponse extends SlimeJsonResponse {
 
-    public SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) {
-        super(result.deployLogger().slime());
+    public SessionPrepareAndActivateResponse(PrepareAndActivateResult result, ApplicationId applicationId) {
+        super(result.prepareResult().deployLogger().slime());
 
         TenantName tenantName = applicationId.tenant();
-        String message = "Session " + result.sessionId() + " for tenant '" + tenantName.value() + "' prepared and activated.";
+        String message = "Session " + result.prepareResult().sessionId() + " for tenant '" + tenantName.value() + "' prepared" +
+                         (result.activationFailure() == null ? " and activated." : ", but activation failed: " + result.activationFailure().getMessage());
         Cursor root = slime.get();
 
-        root.setString("session-id", Long.toString(result.sessionId()));
         root.setString("message", message);
+        root.setString("sessionId", Long.toString(result.prepareResult().sessionId()));
+        root.setBool("activated", result.activationFailure() == null);
 
-        // TODO: remove unused fields, but add whether activation was successful.
-        root.setString("tenant", tenantName.value());
-        root.setString("url", "http://" + request.getHost() + ":" + request.getPort() +
-                              "/application/v2/tenant/" + tenantName +
-                              "/application/" + applicationId.application().value() +
-                              "/environment/" + zone.environment().value() +
-                              "/region/" + zone.region().value() +
-                              "/instance/" + applicationId.instance().value());
-        new ConfigChangeActionsSlimeConverter(result.configChangeActions()).toSlime(root);
+        new ConfigChangeActionsSlimeConverter(result.prepareResult().configChangeActions()).toSlime(root);
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java
index d93ee19085a..0632da173af 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java
@@ -23,6 +23,7 @@ import java.util.List;
 public class MockProvisioner implements Provisioner {
 
     private boolean transientFailureOnPrepare = false;
+    private RuntimeException activationFailure = null;
     private HostProvisioner hostProvisioner = null;
 
     public MockProvisioner hostProvisioner(HostProvisioner hostProvisioner) {
@@ -35,19 +36,29 @@ public class MockProvisioner implements Provisioner {
         return this;
     }
 
+    public MockProvisioner activationFailure(RuntimeException activationFailure) {
+        this.activationFailure = activationFailure;
+        return this;
+    }
+
     @Override
     public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
-        if (hostProvisioner != null) {
-            return hostProvisioner.prepare(cluster, capacity, logger);
-        }
         if (transientFailureOnPrepare) {
             throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
         }
+        if (hostProvisioner != null) {
+            return hostProvisioner.prepare(cluster, capacity, logger);
+        }
         throw new UnsupportedOperationException("This mock does not support prepare");
     }
 
     @Override
     public void activate(Collection<HostSpec> hosts, ActivationContext context, ApplicationTransaction transaction) {
+        if (activationFailure != null) {
+            RuntimeException toThrow = activationFailure;
+            activationFailure = null;
+            throw toThrow;
+        }
     }
 
     @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java
new file mode 100644
index 00000000000..3fb5aa45a36
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java
@@ -0,0 +1,311 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.model.provision.InMemoryProvisioner;
+import com.yahoo.config.provision.ApplicationLockException;
+import com.yahoo.config.provision.ParentHostUnavailableException;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler.Context;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.MockProvisioner;
+import com.yahoo.vespa.config.server.application.OrchestratorMock;
+import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
+import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author jonmv
+ */
+class ApplicationApiHandlerTest {
+
+    private static final TenantName tenant = TenantName.from("test");
+    private static final Map<String, String> appPackage = Map.of("services.xml",
+                                                                 """
+                                                                 <services version='1.0'>
+                                                                   <container id='jdisc' version='1.0'>
+                                                                     <nodes count='2' />
+                                                                   </container>
+                                                                 </services>
+                                                                 """,
+
+                                                                 "deployment.xml",
+                                                                 """
+                                                                 <deployment version='1.0' />
+                                                                 """);
+    static final String minimalPrepareParams = """
+                                               {
+                                                 "containerEndpoints": [
+                                                   {
+                                                     "clusterId": "jdisc",
+                                                     "scope": "zone",
+                                                     "names": ["zone.endpoint"],
+                                                     "routingMethod": "exclusive",
+                                                     "authMethod": "mtls"
+                                                   }
+                                                 ]
+                                               }
+                                               """;
+
+    private final Curator curator = new MockCurator();
+    private ApplicationRepository applicationRepository;
+
+    private MockProvisioner provisioner;
+    private ConfigserverConfig configserverConfig;
+    private TenantRepository tenantRepository;
+    private ApplicationApiHandler handler;
+
+    @TempDir
+    public Path dbDir, defsDir, refsDir;
+
+    @BeforeEach
+    public void setupRepo() throws IOException {
+        configserverConfig = new ConfigserverConfig.Builder()
+                .hostedVespa(true)
+                .configServerDBDir(dbDir.toString())
+                .configDefinitionsDir(defsDir.toString())
+                .fileReferencesDir(refsDir.toString())
+                .build();
+        Clock clock = new ManualClock();
+        provisioner = new MockProvisioner().hostProvisioner(new InMemoryProvisioner(4, false));
+        tenantRepository = new TestTenantRepository.Builder()
+                .withConfigserverConfig(configserverConfig)
+                .withCurator(curator)
+                .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig))
+                .build();
+        tenantRepository.addTenant(tenant);
+        applicationRepository = new ApplicationRepository.Builder()
+                .withTenantRepository(tenantRepository)
+                .withOrchestrator(new OrchestratorMock())
+                .withClock(clock)
+                .withConfigserverConfig(configserverConfig)
+                .build();
+        handler = new ApplicationApiHandler(new Context(Runnable::run, null),
+                                            applicationRepository,
+                                            configserverConfig);
+    }
+
+    private HttpResponse put(long sessionId, Map<String, String> parameters) throws IOException {
+        var request = com.yahoo.container.jdisc.HttpRequest.createTestRequest("http://host:123/application/v2/tenant/" + tenant + "/prepareandactivate/" + sessionId,
+                                                                              Method.PUT,
+                                                                              InputStream.nullInputStream(),
+                                                                              parameters);
+        return handler.handle(request);
+    }
+
+    private HttpResponse post(String json, byte[] appZip, Map<String, String> parameters) throws IOException {
+        HttpEntity entity = MultipartEntityBuilder.create()
+                                                  .addTextBody("prepareParams", json, ContentType.APPLICATION_JSON)
+                                                  .addBinaryBody("applicationPackage", appZip, ContentType.create("application/zip"), "applicationZip")
+                                                  .build();
+        var request = com.yahoo.container.jdisc.HttpRequest.createTestRequest("http://host:123/application/v2/tenant/" + tenant + "/prepareandactivate",
+                                                                              Method.POST,
+                                                                              entity.getContent(),
+                                                                              parameters);
+        request.getJDiscRequest().headers().add("Content-Type", entity.getContentType());
+        return handler.handle(request);
+    }
+
+    private static byte[] zip(Map<String, String> files) throws IOException {
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(buffer)) {
+            files.forEach((name, content) -> uncheck(() -> {
+                zip.putNextEntry(new ZipEntry(name));
+                zip.write(content.getBytes(UTF_8));
+            }));
+        }
+        return buffer.toByteArray();
+    }
+
+    private static void verifyResponse(HttpResponse response, int expectedStatusCode, String expectedBody) throws IOException {
+        String body = new ByteArrayOutputStream() {{ response.render(this); }}.toString(UTF_8);
+        assertEquals(expectedStatusCode, response.getStatus(), "Status code should match. Response was:\n" + body);
+        assertEquals(SlimeUtils.toJson(SlimeUtils.jsonToSlimeOrThrow(expectedBody).get(), false),
+                     SlimeUtils.toJson(SlimeUtils.jsonToSlimeOrThrow(body).get(), false));
+    }
+
+    @Test
+    void testMinimalDeployment() throws Exception {
+        verifyResponse(post(minimalPrepareParams, zip(appPackage), Map.of()),
+                       200,
+                       """
+                       {
+                         "log": [ ],
+                         "message": "Session 2 for tenant 'test' prepared and activated.",
+                         "sessionId": "2",
+                         "activated": true,
+                         "configChangeActions": {
+                           "restart": [ ],
+                           "refeed": [ ],
+                           "reindex": [ ]
+                         }
+                       }
+                       """);
+    }
+
+    @Test
+    void testBadZipDeployment() throws Exception {
+        verifyResponse(post("{ }", Arrays.copyOf(zip(appPackage), 13), Map.of()),
+                       400,
+                       """
+                       {
+                         "error-code": "BAD_REQUEST",
+                         "message": "Error preprocessing application package for test.default, session 2: services.xml does not exist in application package"
+                       }
+                       """);
+    }
+
+    @Test
+    void testPrepareFailure() throws Exception {
+        provisioner.transientFailureOnPrepare();
+        verifyResponse(post(minimalPrepareParams, zip(appPackage), Map.of()),
+                       409,
+                       """
+                       {
+                         "error-code": "LOAD_BALANCER_NOT_READY",
+                         "message": "Unable to create load balancer: some internal exception"
+                       }
+                       """);
+    }
+
+    @Test
+    void testActivateInvalidSession() throws Exception {
+        verifyResponse(put(2, Map.of()),
+                       404,
+                       """
+                       {
+                         "error-code": "NOT_FOUND",
+                         "message": "Local session 2 for 'test' was not found"
+                       }
+                       """);
+    }
+
+    @Test
+    void testActivationFailuresAndRetries() throws Exception {
+        // Prepare session 2, but fail on hosts; this session will be activated later.
+        provisioner.activationFailure(new ParentHostUnavailableException("host still booting"));
+        verifyResponse(post(minimalPrepareParams, zip(appPackage), Map.of()),
+                       200,
+                       """
+                       {
+                         "log": [ ],
+                         "message": "Session 2 for tenant 'test' prepared, but activation failed: host still booting",
+                         "sessionId": "2",
+                         "activated": false,
+                         "configChangeActions": {
+                           "restart": [ ],
+                           "refeed": [ ],
+                           "reindex": [ ]
+                         }
+                       }
+                       """);
+
+        // Prepare session 3, but fail on lock; this session will become outdated later.
+        provisioner.activationFailure(new ApplicationLockException("lock timeout"));
+        verifyResponse(post(minimalPrepareParams, zip(appPackage), Map.of()),
+                       200,
+                       """
+                       {
+                         "log": [ ],
+                         "message": "Session 3 for tenant 'test' prepared, but activation failed: lock timeout",
+                         "sessionId": "3",
+                         "activated": false,
+                         "configChangeActions": {
+                           "restart": [ ],
+                           "refeed": [ ],
+                           "reindex": [ ]
+                         }
+                       }
+                       """);
+
+        // Prepare session 4, but fail with some other exception, which we won't retry.
+        provisioner.activationFailure(new RuntimeException("some other exception"));
+        verifyResponse(post(minimalPrepareParams, zip(appPackage), Map.of()),
+                       500,
+                       """
+                       {
+                         "error-code": "INTERNAL_SERVER_ERROR",
+                         "message": "some other exception"
+                       }
+                       """);
+
+        // Retry only activation of session 2, but fail again with hosts.
+        provisioner.activationFailure(new ParentHostUnavailableException("host still booting"));
+        verifyResponse(put(2, Map.of()),
+                       409,
+                       """
+                       {
+                         "error-code": "PARENT_HOST_NOT_READY",
+                         "message": "host still booting"
+                       }
+                       """);
+
+        // Retry only activation of session 2, but fail again with lock.
+        provisioner.activationFailure(new ApplicationLockException("lock timeout"));
+        verifyResponse(put(2, Map.of()),
+                       500,
+                       """
+                       {
+                         "error-code": "APPLICATION_LOCK_FAILURE",
+                         "message": "lock timeout"
+                       }
+                       """);
+
+        // Retry only activation of session 2, and succeed!
+        provisioner.activationFailure(null);
+        verifyResponse(put(2, Map.of()),
+                       200,
+                       """
+                       {
+                         "message": "Session 2 for test.default.default activated"
+                       }
+                       """);
+
+        // Retry only activation of session 3, but fail because it is now based on an outdated session.
+        verifyResponse(put(3, Map.of()),
+                       409,
+                       """
+                       {
+                         "error-code": "ACTIVATION_CONFLICT",
+                         "message": "app:test.default.default Cannot activate session 3 because the currently active session (2) has changed since session 3 was created (was empty at creation time)"
+                       }
+                       """);
+
+        // Retry activation of session 2 again, and fail.
+        verifyResponse(put(2, Map.of()),
+                       400,
+                       """
+                       {
+                         "error-code": "BAD_REQUEST",
+                         "message": "app:test.default.default Session 2 is already active"
+                       }
+                       """);
+    }
+
+}