Lock instances on application level

2 files changed, 30 insertions, 7 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index b6a647b434e..690ed0043f4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -57,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.application.JobList;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
 import com.yahoo.vespa.hosted.controller.concurrent.Once;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
@@ -237,7 +238,8 @@ public class ApplicationController {
     public Instance createApplication(ApplicationId id, Optional<Credentials> credentials) {
         if (id.instance().isTester())
             throw new IllegalArgumentException("'" + id + "' is a tester application!");
-        try (Lock lock = lock(id)) {
+        try (Lock lock = lock(TenantAndApplicationId.from(id));
+             Lock oldLock = lock(id)) {
             // Validate only application names which do not already exist.
             if (asList(id.tenant()).stream().noneMatch(application -> application.id().application().equals(id.application())))
                 com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value());
@@ -291,7 +293,8 @@ public class ApplicationController {
             Set<ContainerEndpoint> endpoints;
             Optional<ApplicationCertificate> applicationCertificate;
 
-            try (Lock lock = lock(applicationId)) {
+            try (Lock lock = lock(TenantAndApplicationId.from(applicationId));
+                 Lock oldLock = lock(applicationId)) {
                 LockedInstance application = new LockedInstance(require(applicationId), lock);
 
                 boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed();
@@ -744,7 +747,8 @@ public class ApplicationController {
      * @param action Function which acts on the locked application.
      */
     public void lockIfPresent(ApplicationId applicationId, Consumer<LockedInstance> action) {
-        try (Lock lock = lock(applicationId)) {
+        try (Lock lock = lock(TenantAndApplicationId.from(applicationId));
+             Lock oldLock = lock(applicationId)) {
             get(applicationId).map(application -> new LockedInstance(application, lock)).ifPresent(action);
         }
     }
@@ -757,7 +761,8 @@ public class ApplicationController {
      * @throws IllegalArgumentException when application does not exist.
      */
     public void lockOrThrow(ApplicationId applicationId, Consumer<LockedInstance> action) {
-        try (Lock lock = lock(applicationId)) {
+        try (Lock lock = lock(TenantAndApplicationId.from(applicationId));
+             Lock oldLock = lock(applicationId)) {
             action.accept(new LockedInstance(require(applicationId), lock));
         }
     }
@@ -820,6 +825,15 @@ public class ApplicationController {
      * Any operation which stores an application need to first acquire this lock, then read, modify
      * and store the application, and finally release (close) the lock.
      */
+    Lock lock(TenantAndApplicationId application) {
+        return curator.lock(application);
+    }
+
+    /**
+     * Returns a lock which provides exclusive rights to changing this application.
+     * Any operation which stores an application need to first acquire this lock, then read, modify
+     * and store the application, and finally release (close) the lock.
+     */
     Lock lock(ApplicationId application) {
         return curator.lock(application);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index d439a84af70..a1be19bf1e9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.Applicatio
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
@@ -135,6 +136,10 @@ public class CuratorDb {
         return lock(lockPath(name), defaultLockTimeout.multipliedBy(2));
     }
 
+    public Lock lock(TenantAndApplicationId id) {
+        return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
+    }
+
     public Lock lock(ApplicationId id) {
         return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
     }
@@ -505,10 +510,14 @@ public class CuratorDb {
                 .append(tenant.value());
     }
 
-    private Path lockPath(ApplicationId application) {
+    private Path lockPath(TenantAndApplicationId application) {
         return lockPath(application.tenant())
-                .append(application.application().value())
-                .append(application.instance().value());
+                .append(application.application().value());
+    }
+
+    private Path lockPath(ApplicationId instance) {
+        return lockPath(TenantAndApplicationId.from(instance))
+                .append(instance.instance().value());
     }
 
     private Path lockPath(ApplicationId application, ZoneId zone) {