Wait until application has been removed before responding

On the config server where the request for deleting an application
happens, wait until applications has been removed. We still don't know
anything about the other servers, but hopefully they have also gotten the
ZooKeeper event and removed the application

2 files changed, 22 insertions, 3 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 5d46f3dc240..ec1516f1267 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -74,7 +74,6 @@ import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantMetaData;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.stats.LockStats;
 import com.yahoo.vespa.curator.stats.ThreadLockStats;
@@ -505,7 +504,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         NestedTransaction transaction = new NestedTransaction();
         Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId))
                                                                                  .map(lock -> new ApplicationTransaction(lock, transaction));
-        try (var sessionLock = tenantApplications.lock(applicationId)) {
+        try (var applicationLock = tenantApplications.lock(applicationId)) {
             Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId);
             if (activeSession.isEmpty()) return false;
 
@@ -533,6 +532,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             } else {
                 transaction.commit();
             }
+            waitForApplicationRemoved(tenantApplications, applicationId);
             return true;
         } finally {
             applicationTransaction.ifPresent(ApplicationTransaction::close);
@@ -591,6 +591,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return fileReferencesToDelete;
     }
 
+    private void waitForApplicationRemoved(TenantApplications applications, ApplicationId applicationId) {
+        log.log(Level.INFO, "Waiting for " + applicationId + " to be deleted");
+        Duration duration = Duration.ofSeconds(5);
+        Instant end = Instant.now().plus(duration);
+        do {
+            if ( ! (applications.hasApplication(applicationId)))
+                return;
+            log.log(Level.INFO, "Application " + applicationId + " not deleted yet, will retry");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException interruptedException) {/* ignore */}
+        } while (Instant.now().isBefore(end));
+        log.log(Level.INFO, "Application " + applicationId + " not deleted after " + duration + ", giving up");
+    }
+
     private Set<String> getFileReferencesInUse() {
         Set<String> fileReferencesInUse = new HashSet<>();
         // Intentionally skip applications that we for some reason do not find
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 6f626464c63..f9c1f831049 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -259,7 +259,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica
                 return;
             }
 
-            if (applicationMapper.hasApplication(applicationId, clock.instant())) {
+            if (hasApplication(applicationId)) {
                 applicationMapper.remove(applicationId);
                 hostRegistry.removeHostsForKey(applicationId);
                 reloadListenersOnRemove(applicationId);
@@ -270,6 +270,10 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica
         }
     }
 
+    public boolean hasApplication(ApplicationId applicationId) {
+        return applicationMapper.hasApplication(applicationId, clock.instant());
+    }
+
     public void removeApplicationsExcept(Set<ApplicationId> applications) {
         for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) {
             if ( ! applications.contains(activeApplication)) {