Synchronise access to session status of an application, and eliminate some races

9 files changed, 140 insertions, 79 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index bea087f6ae9..f2e1410e9a9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -302,34 +302,35 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         if (tenant == null) return false;
 
         TenantApplications tenantApplications = tenant.getApplicationRepo();
-        if (!tenantApplications.activeApplications().contains(applicationId)) return false;
-
-        // Deleting an application is done by deleting the remote session and waiting
-        // until the config server where the deployment happened picks it up and deletes
-        // the local session
-        long sessionId = tenantApplications.requireActiveSessionOf(applicationId);
-        RemoteSession remoteSession = getRemoteSession(tenant, sessionId);
-        remoteSession.createDeleteTransaction().commit();
-
-        log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted");
-        // TODO: Add support for timeout in request
-        Duration waitTime = Duration.ofSeconds(60);
-        if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) {
-            log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted");
-        } else {
-            log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")");
-            return false;
-        }
+        try (Lock lock = tenantApplications.lock(applicationId)) {
+            if ( ! tenantApplications.exists(applicationId)) return false;
+            // Deleting an application is done by deleting the remote session and waiting
+            // until the config server where the deployment happened picks it up and deletes
+            // the local session
+            long sessionId = tenantApplications.requireActiveSessionOf(applicationId);
+            RemoteSession remoteSession = getRemoteSession(tenant, sessionId);
+            remoteSession.createDeleteTransaction().commit();
+
+            log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted");
+            // TODO: Add support for timeout in request
+            Duration waitTime = Duration.ofSeconds(60);
+            if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) {
+                log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted");
+            } else {
+                log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")");
+                return false;
+            }
 
-        NestedTransaction transaction = new NestedTransaction();
-        transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested
-        // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server
-        // this is done asynchronously in application maintenance by the node repository)
-        transaction.add(tenantApplications.createDeleteTransaction(applicationId));
+            NestedTransaction transaction = new NestedTransaction();
+            transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested
+            // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server
+            // this is done asynchronously in application maintenance by the node repository)
+            transaction.add(tenantApplications.createDeleteTransaction(applicationId));
 
-        hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId));
-        transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId));
-        transaction.commit();
+            hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId));
+            transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId));
+            transaction.commit();
+        }
 
         return true;
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 348e8f1cafc..a27c2b3603e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -11,14 +11,18 @@ import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.ReloadHandler;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
 
+import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.logging.Logger;
@@ -27,9 +31,9 @@ import java.util.stream.Collectors;
 /**
  * The applications of a tenant, backed by ZooKeeper.
  *
- * Each application is stored under /config/v2/tenants/<tenant>/applications/<applications>,
- * the root contains the currently active session, if any, and sub-paths /preparing contains the session id
- * of whatever session may be activated next, if any, and /lock is used for synchronizing writes to all these paths.
+ * Each application is stored under /config/v2/tenants/<tenant>/applications/<application>,
+ * the root contains the currently active session, if any. Locks for synchronising writes to these paths, and changes
+ * to the config of this application, are found under /config/v2/tenants/<tenant>/locks/<application>.
  *
  * @author Ulf Lilleengen
  * @author jonmv
@@ -40,17 +44,20 @@ public class TenantApplications {
 
     private final Curator curator;
     private final Path applicationsPath;
+    private final Path locksPath;
     // One thread pool for all instances of this class
     private static final ExecutorService pathChildrenExecutor =
             Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(TenantApplications.class.getName()));
     private final Curator.DirectoryCache directoryCache;
     private final ReloadHandler reloadHandler;
     private final TenantName tenant;
+    private final Map<ApplicationId, Lock> locks;
 
-    private TenantApplications(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) {
+    private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant) {
         this.curator = curator;
-        this.applicationsPath = applicationsPath;
-        curator.create(applicationsPath);
+        this.applicationsPath = TenantRepository.getApplicationsPath(tenant);
+        this.locksPath = TenantRepository.getLocksPath(tenant);
+        this.locks = new ConcurrentHashMap<>(2);
         this.reloadHandler = reloadHandler;
         this.tenant = tenant;
         this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor);
@@ -59,7 +66,7 @@ public class TenantApplications {
     }
 
     public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) {
-        return new TenantApplications(curator, TenantRepository.getApplicationsPath(tenant), reloadHandler, tenant);
+        return new TenantApplications(curator, reloadHandler, tenant);
     }
 
     /**
@@ -86,13 +93,17 @@ public class TenantApplications {
             try {
                 curator.delete(applicationsPath.append(appNode));
             }
-            catch (Exception e) {
+            catch (RuntimeException e) {
                 log.log(LogLevel.WARNING, TenantRepository.logPre(tenant) + "Failed to clean up stray node '" + appNode + "'!", e);
             }
             return false;
         }
     }
 
+    public boolean exists(ApplicationId id) {
+        return curator.exists(applicationPath(id));
+    }
+
     /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */
     public OptionalLong activeSessionOf(ApplicationId id) {
         String data = curator.getData(applicationPath(id)).map(Utf8::toString)
@@ -114,7 +125,9 @@ public class TenantApplications {
      * Creates a node for the given application, marking its existence.
      */
     public void createApplication(ApplicationId id) {
-        curator.create(applicationPath(id));
+        try (Lock lock = lock(id)) {
+            curator.create(applicationPath(id));
+        }
     }
 
     /**
@@ -133,7 +146,7 @@ public class TenantApplications {
      * Returns a transaction which deletes this application.
      */
     public CuratorTransaction createDeleteTransaction(ApplicationId applicationId) {
-        return CuratorTransaction.from(CuratorOperations.delete(applicationPath(applicationId).getAbsolute()), curator);
+        return CuratorTransaction.from(CuratorOperations.deleteAll(applicationPath(applicationId).getAbsolute(), curator), curator);
     }
 
     /**
@@ -150,6 +163,14 @@ public class TenantApplications {
         directoryCache.close();
     }
 
+    /** Returns the lock for changing the session status of the given application. */
+    public Lock lock(ApplicationId id) {
+        curator.create(lockPath(id));
+        Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator));
+        lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
+        return lock;
+    }
+
     private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) {
         switch (event.getType()) {
             case CHILD_ADDED:
@@ -183,4 +204,8 @@ public class TenantApplications {
         return applicationsPath.append(id.serializedForm());
     }
 
+    private Path lockPath(ApplicationId id) {
+        return locksPath.append(id.serializedForm());
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 01bb4e2dc76..b6e1d1873c9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -119,9 +119,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
             prepare();
 
         TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout);
-        long sessionId = session.getSessionId();
-        validateSessionStatus(session);
-        try (Lock lock = tenant.getSessionLock(timeout)) {
+
+        try (Lock lock = tenant.getApplicationRepo().lock(session.getApplicationId())) {
+            validateSessionStatus(session);
             NestedTransaction transaction = new NestedTransaction();
             transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure));
 
@@ -129,13 +129,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
                 hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts());
             }
             transaction.commit();
-            session.waitUntilActivated(timeoutBudget);
         } catch (RuntimeException e) {
             throw e;
         } catch (Exception e) {
             throw new InternalServerException("Error activating application", e);
         }
-        log.log(LogLevel.INFO, session.logPre() + "Session " + sessionId + 
+
+        session.waitUntilActivated(timeoutBudget);
+
+        log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() +
                                " activated successfully using " +
                                ( hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner" ) +
                                ". Config generation " + session.getMetaData().getGeneration());
@@ -153,7 +155,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
 
     /** Exposes the session of this for testing only */
     public LocalSession session() { return session; }
-    
+
     private long validateSessionStatus(LocalSession localSession) {
         long sessionId = localSession.getSessionId();
         if (Session.Status.NEW.equals(localSession.getStatus())) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
index 8838daeb32e..26f437920ad 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
@@ -70,10 +70,7 @@ public class RemoteSession extends Session {
     }
 
     public synchronized ApplicationSet ensureApplicationLoaded() {
-        if (applicationSet == null) {
-            applicationSet = loadApplication();
-        }
-        return applicationSet;
+        return applicationSet == null ? applicationSet = loadApplication() : applicationSet;
     }
 
     public Session.Status getStatus() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
index a68f4a396cd..88e71d7ddd1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
@@ -11,10 +11,8 @@ import com.yahoo.vespa.config.server.session.LocalSessionRepo;
 import com.yahoo.vespa.config.server.session.RemoteSessionRepo;
 import com.yahoo.vespa.config.server.session.SessionFactory;
 import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.Lock;
 import org.apache.zookeeper.data.Stat;
 
-import java.time.Duration;
 import java.time.Instant;
 import java.util.Optional;
 
@@ -30,7 +28,7 @@ public class Tenant implements TenantHandlerProvider {
 
     static final String SESSIONS = "sessions";
     static final String APPLICATIONS = "applications";
-    static final String SESSION_LOCK_PATH = "activateLock";
+    static final String LOCKS = "locks";
 
     private final TenantName name;
     private final RemoteSessionRepo remoteSessionRepo;
@@ -38,7 +36,6 @@ public class Tenant implements TenantHandlerProvider {
     private final SessionFactory sessionFactory;
     private final LocalSessionRepo localSessionRepo;
     private final TenantApplications applicationRepo;
-    private final Lock sessionLock;
     private final RequestHandler requestHandler;
     private final ReloadHandler reloadHandler;
     private final TenantFileSystemDirs tenantFileSystemDirs;
@@ -61,7 +58,6 @@ public class Tenant implements TenantHandlerProvider {
         this.remoteSessionRepo = remoteSessionRepo;
         this.sessionFactory = sessionFactory;
         this.localSessionRepo = localSessionRepo;
-        this.sessionLock = createLock(curator, path);
         this.applicationRepo = applicationRepo;
         this.tenantFileSystemDirs = tenantFileSystemDirs;
         this.curator = curator;
@@ -110,14 +106,6 @@ public class Tenant implements TenantHandlerProvider {
         return localSessionRepo;
     }
 
-    /**
-     * This lock allows activation and deactivation of sessions under this tenant.
-     */
-    public Lock getSessionLock(Duration timeout) {
-        sessionLock.acquire(timeout);
-        return sessionLock;
-    }
-    
     @Override
     public String toString() {
         return getName().value();
@@ -165,10 +153,4 @@ public class Tenant implements TenantHandlerProvider {
         curator.delete(path);
     }
 
-    private static Lock createLock(Curator curator, Path tenantPath) {
-        Path lockPath = tenantPath.append(SESSION_LOCK_PATH);
-        curator.create(lockPath);
-        return new Lock(lockPath.getAbsolute(), curator);
-    }
-
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
index 9c74c9c1e67..c37db0e0df5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
@@ -262,7 +262,10 @@ public class TenantRepository {
      */
     private synchronized void writeTenantPath(TenantName name) {
         Path tenantPath = getTenantPath(name);
-        curator.createAtomically(tenantPath, tenantPath.append(Tenant.SESSIONS), tenantPath.append(Tenant.APPLICATIONS));
+        curator.createAtomically(tenantPath,
+                                 tenantPath.append(Tenant.SESSIONS),
+                                 tenantPath.append(Tenant.APPLICATIONS),
+                                 tenantPath.append(Tenant.LOCKS));
     }
 
     /**
@@ -406,4 +409,11 @@ public class TenantRepository {
         return getTenantPath(tenantName).append(Tenant.APPLICATIONS);
     }
 
+    /**
+     * Gets zookeeper path for locks for a tenant's applications
+     */
+    public static Path getLocksPath(TenantName tenantName) {
+        return getTenantPath(tenantName).append(Tenant.LOCKS);
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
index fe34e6c361d..1430475e486 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
@@ -6,6 +6,7 @@ import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.OptionalLong;
 import java.util.Set;
 
 import com.yahoo.component.Version;
@@ -31,6 +32,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
 import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.Lock;
 
 /**
  * A per tenant request handler, for handling reload (activate application) and getConfig requests for
@@ -99,18 +101,31 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host
      */
     @Override
     public void reloadConfig(ApplicationSet applicationSet) {
-        setLiveApp(applicationSet);
-        notifyReloadListeners(applicationSet);
+        ApplicationId id = applicationSet.getId();
+        try (Lock lock = applications.lock(id)) {
+            if ( ! applications.exists(id))
+                return; // Application was deleted before activation.
+            if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id))
+                return; // Application activated a new session before we got here.
+
+            setLiveApp(applicationSet);
+            notifyReloadListeners(applicationSet);
+        }
     }
 
     @Override
     public void removeApplication(ApplicationId applicationId) {
-        if (applicationMapper.hasApplication(applicationId, clock.instant())) {
-            applicationMapper.remove(applicationId);
-            hostRegistry.removeHostsForKey(applicationId);
-            reloadListenersOnRemove(applicationId);
-            tenantMetricUpdater.setApplications(applicationMapper.numApplications());
-            metrics.removeMetricUpdater(Metrics.createDimensions(applicationId));
+        try (Lock lock = applications.lock(applicationId)) {
+            if (applications.exists(applicationId))
+                return; // Application was deployed again.
+
+            if (applicationMapper.hasApplication(applicationId, clock.instant())) {
+                applicationMapper.remove(applicationId);
+                hostRegistry.removeHostsForKey(applicationId);
+                reloadListenersOnRemove(applicationId);
+                tenantMetricUpdater.setApplications(applicationMapper.numApplications());
+                metrics.removeMetricUpdater(Metrics.createDimensions(applicationId));
+            }
         }
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
index 4046384005d..9bd5c5f1614 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java
@@ -81,6 +81,8 @@ public class TenantRepositoryTest {
 
     @Test
     public void testListenersAdded() throws IOException, SAXException {
+        tenantRepository.getTenant(tenant1).getApplicationRepo().createApplication(ApplicationId.defaultId());
+        tenantRepository.getTenant(tenant1).getApplicationRepo().createPutTransaction(ApplicationId.defaultId(), 4).commit();
         tenantRepository.getTenant(tenant1).getReloadHandler().reloadConfig(ApplicationSet.fromSingle(
                 new Application(new VespaModel(MockApplicationPackage.createEmpty()),
                                 new ServerCache(),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java
index 952f87cbc6d..26a6f5e0c5f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java
@@ -176,6 +176,8 @@ public class TenantRequestHandlerTest {
     public void testReloadConfig() throws IOException {
         ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build();
 
+        server.applications().createApplication(applicationId);
+        server.applications().createPutTransaction(applicationId, 1).commit();
         server.reloadConfig(reloadConfig(1));
         assertThat(listener.reloaded.get(), is(1));
         // Using only payload list for this simple test
@@ -195,6 +197,7 @@ public class TenantRequestHandlerTest {
 
         listener.reloaded.set(0);
         feedApp(app2, 2, defaultApp(), true);
+        server.applications().createPutTransaction(applicationId, 2).commit();
         server.reloadConfig(reloadConfig(2L));
         configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, "");
         assertTrue(configResponse.isInternalRedeploy());
@@ -206,19 +209,34 @@ public class TenantRequestHandlerTest {
 
     @Test
     public void testRemoveApplication() {
+        ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default");
         server.reloadConfig(reloadConfig(1));
+        assertThat(listener.reloaded.get(), is(0));
+
+        server.applications().createApplication(appId);
+        server.applications().createPutTransaction(appId, 1).commit();
+        server.reloadConfig(reloadConfig(1));
+        assertThat(listener.reloaded.get(), is(1));
+
+        assertThat(listener.removed.get(), is(0));
+
+        server.removeApplication(appId);
         assertThat(listener.removed.get(), is(0));
-        server.removeApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build());
+
+        server.applications().createDeleteTransaction(appId).commit();
+        server.removeApplication(appId);
         assertThat(listener.removed.get(), is(1));
     }
 
     @Test
     public void testResolveForAppId() {
         long id = 1L;
-        SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id)));
         ApplicationId appId = new ApplicationId.Builder()
                               .tenant(tenant)
                               .applicationName("myapp").instanceName("myinst").build();
+        server.applications().createApplication(appId);
+        server.applications().createPutTransaction(appId, 1).commit();
+        SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id)));
         zkc.writeApplicationId(appId);
         RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc);
         server.reloadConfig(session.ensureApplicationLoaded());
@@ -256,6 +274,8 @@ public class TenantRequestHandlerTest {
     }
 
     private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException {
+        server.applications().createApplication(appId);
+        server.applications().createPutTransaction(appId, sessionId).commit();
         feedApp(appDir, sessionId, appId, false);
         SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId)));
         zkc.writeApplicationId(appId);
@@ -291,9 +311,11 @@ public class TenantRequestHandlerTest {
     @Test
     public void testHasApplication() {
         assertdefaultAppNotFound();
+        ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default");
+        server.applications().createApplication(appId);
+        server.applications().createPutTransaction(appId, 1).commit();
         server.reloadConfig(reloadConfig(1));
-        assertTrue(server.hasApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(),
-                                         Optional.of(vespaVersion)));
+        assertTrue(server.hasApplication(appId, Optional.of(vespaVersion)));
     }
 
     private void assertdefaultAppNotFound() {
@@ -302,10 +324,13 @@ public class TenantRequestHandlerTest {
 
     @Test
     public void testMultipleApplicationsReload() {
+        ApplicationId appId = ApplicationId.from(tenant.value(), "foo", "default");
         assertdefaultAppNotFound();
+        server.applications().createApplication(appId);
+        server.applications().createPutTransaction(appId, 1).commit();
         server.reloadConfig(reloadConfig(1, "foo"));
         assertdefaultAppNotFound();
-        assertTrue(server.hasApplication(new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(),
+        assertTrue(server.hasApplication(appId,
                                          Optional.of(vespaVersion)));
         assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId()));
         assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder()
@@ -318,6 +343,8 @@ public class TenantRequestHandlerTest {
         assertdefaultAppNotFound();
 
         VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app")));
+        server.applications().createApplication(ApplicationId.defaultId());
+        server.applications().createPutTransaction(ApplicationId.defaultId(), 1).commit();
         server.reloadConfig(ApplicationSet.fromSingle(new Application(model,
                                                                       new ServerCache(),
                                                                       1,