Try to finish delete if a previous delete failed

If we timeout waiting for an application to be deleted, make sure that
the next attempt to delete (where we have an active session, but the
remote session has been deleted) is allowed to remove the provisioned
nodes

2 files changed, 28 insertions, 10 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index d2f26738301..7ee6febfac1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -26,6 +26,7 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream;
@@ -361,15 +362,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             // until the config server where the deployment happened picks it up and deletes
             // the local session
             long sessionId = activeSession.get();
-            RemoteSession remoteSession = getRemoteSession(tenant, sessionId);
-            remoteSession.createDeleteTransaction().commit();
-            log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted");
 
-
-            if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) {
-                log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted");
-            } else {
-                throw new InternalServerException("Session " + sessionId + " was not deleted (waited " + waitTime + ")");
+            RemoteSession remoteSession;
+            try {
+                remoteSession = getRemoteSession(tenant, sessionId);
+                Transaction deleteTransaction = remoteSession.createDeleteTransaction();
+                deleteTransaction.commit();
+                log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted");
+
+                if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) {
+                    log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted");
+                } else {
+                    deleteTransaction.rollbackOrLog();
+                    throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId);
+                }
+            } catch (NotFoundException e) {
+                // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below
+                log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup");
             }
 
             NestedTransaction transaction = new NestedTransaction();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index fb745bbb76b..d924d22cb39 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -266,13 +266,22 @@ public class ApplicationRepositoryTest {
         }
 
         {
+            PrepareResult prepareResult = deployApp(testApp);
             try {
-                deployApp(testApp);
                 applicationRepository.delete(applicationId(), Duration.ZERO);
                 fail("Should have gotten an exception");
             } catch (InternalServerException e) {
-                assertEquals("Session 5 was not deleted (waited PT0S)", e.getMessage());
+                assertEquals("test1.testapp was not deleted (waited PT0S), session " + prepareResult.sessionId(), e.getMessage());
             }
+
+            // No active session or remote session (deleted in step above), but an exception was thrown above
+            // A new delete should cleanup and be successful
+            LocalSession activeSession = applicationRepository.getActiveSession(applicationId());
+            assertNull(activeSession);
+            Tenant tenant = tenantRepository.getTenant(applicationId().tenant());
+            assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId()));
+
+            assertTrue(applicationRepository.delete(applicationId()));
         }
     }