Acquire provision lock for transactional writes

5 files changed, 83 insertions, 5 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 4fba0204f55..083112d8456 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -132,6 +132,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final TesterClient testerClient;
     private final Metric metric;
     private final BooleanFlag deployWithInternalRestart;
+    private final BooleanFlag acquireProvisionLock;
 
     @Inject
     public ApplicationRepository(TenantRepository tenantRepository,
@@ -182,6 +183,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.testerClient = Objects.requireNonNull(testerClient);
         this.metric = Objects.requireNonNull(metric);
         this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(flagSource);
+        this.acquireProvisionLock = Flags.ALWAYS_ACQUIRE_PROVISION_LOCK.bindTo(flagSource);
     }
 
     public static class Builder {
@@ -508,7 +510,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             // and allocated hosts in model and handlers in RPC server
             transaction.add(tenantApplications.createDeleteTransaction(applicationId));
 
-            hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId));
+            hostProvisioner.ifPresent(provisioner -> {
+                if (acquireProvisionLock.value()) {
+                    try (var provisionLock = provisioner.lock(applicationId)) {
+                        provisioner.remove(transaction, provisionLock);
+                    }
+                } else {
+                    provisioner.remove(transaction, applicationId);
+                }
+            });
             transaction.onCommitted(() -> log.log(Level.INFO, "Deleted " + applicationId));
             transaction.commit();
             return true;
@@ -722,13 +732,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     // ---------------- Session operations ----------------------------------------------------------------
 
-
-
     public CompletionWaiter activate(LocalSession session, Session previousActiveSession, ApplicationId applicationId, boolean force) {
         CompletionWaiter waiter = session.getSessionZooKeeperClient().createActiveWaiter();
         NestedTransaction transaction = new NestedTransaction();
         transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, force));
-        hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts()));
+        hostProvisioner.ifPresent(provisioner -> {
+            if (acquireProvisionLock.value()) {
+                try (var lock = provisioner.lock(applicationId)) {
+                    provisioner.activate(transaction, session.getAllocatedHosts().getHosts(), lock);
+                }
+            } else {
+                provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts());
+            }
+        });
         transaction.commit();
         return waiter;
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index 7553583e70c..87122fdba45 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -22,6 +22,7 @@ import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.TenantName;
@@ -227,15 +228,28 @@ public class DeployTester {
         }
 
         @Override
+        public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) {
+        }
+
+        @Override
         public void remove(NestedTransaction transaction, ApplicationId application) {
             // noop
         }
 
         @Override
+        public void remove(NestedTransaction transaction, ProvisionLock lock) {
+        }
+
+        @Override
         public void restart(ApplicationId application, HostFilter filter) {
             // noop
         }
 
+        @Override
+        public ProvisionLock lock(ApplicationId application) {
+            return null;
+        }
+
     }
 
     private static class FailingModelFactory implements ModelFactory {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
index 9d696ae34e7..32a9e867684 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.container.jdisc.HttpRequest;
@@ -102,17 +103,32 @@ public class SessionHandlerTest {
         }
 
         @Override
+        public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void remove(NestedTransaction transaction, ApplicationId application) {
             removed = true;
             lastApplicationId = application;
         }
 
         @Override
+        public void remove(NestedTransaction transaction, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void restart(ApplicationId application, HostFilter filter) {
             restarted = true;
             lastApplicationId = application;
         }
 
+        @Override
+        public ProvisionLock lock(ApplicationId application) {
+            return null;
+        }
+
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java
index 712242a69e6..d092a3139a0 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.transaction.NestedTransaction;
@@ -95,15 +96,30 @@ class MaintainerTester {
         }
 
         @Override
+        public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void remove(NestedTransaction transaction, ApplicationId application) {
             // noop
         }
 
         @Override
+        public void remove(NestedTransaction transaction, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void restart(ApplicationId application, HostFilter filter) {
             // noop
         }
 
+        @Override
+        public ProvisionLock lock(ApplicationId application) {
+            return null;
+        }
+
     }
 
-}
\ No newline at end of file
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 3858388ed39..43504988d67 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -17,6 +17,7 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.TenantName;
@@ -422,11 +423,26 @@ public class SessionPreparerTest {
         public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { }
 
         @Override
+        public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void remove(NestedTransaction transaction, ApplicationId application) { }
 
         @Override
+        public void remove(NestedTransaction transaction, ProvisionLock lock) {
+
+        }
+
+        @Override
         public void restart(ApplicationId application, HostFilter filter) { }
 
+        @Override
+        public ProvisionLock lock(ApplicationId application) {
+            return null;
+        }
+
     }
 
 }