Make it possible to wait for resources in resources

3 files changed, 66 insertions, 8 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index a341063ddd7..f2f5eb3afa1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -73,7 +73,9 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.stats.LockStats;
 import com.yahoo.vespa.curator.stats.ThreadLockStats;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.BooleanFlag;
 import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 
@@ -138,6 +140,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final TesterClient testerClient;
     private final Metric metric;
     private final ClusterReindexingStatusClient clusterReindexingStatusClient;
+    private final BooleanFlag waitForResourcesInPrepareFlag;
 
     @Inject
     public ApplicationRepository(TenantRepository tenantRepository,
@@ -190,6 +193,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.testerClient = Objects.requireNonNull(testerClient);
         this.metric = Objects.requireNonNull(metric);
         this.clusterReindexingStatusClient = clusterReindexingStatusClient;
+        this.waitForResourcesInPrepareFlag = Flags.WAIT_FOR_RESOURCES_IN_PREPARE.bindTo(flagSource);
     }
 
     public static class Builder {
@@ -397,9 +401,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         SessionRepository sessionRepository = tenant.getSessionRepository();
         DeployLogger logger = new SilentDeployLogger();
         Session newSession = sessionRepository.createSessionFromExisting(activeSession, true, timeoutBudget);
+        boolean waitForResourcesInPrepare = waitForResourcesInPrepareFlag.value();
 
         return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock,
-                                                 false /* don't validate as this is already deployed */, bootstrap));
+                                                 false /* don't validate as this is already deployed */, bootstrap, waitForResourcesInPrepare));
     }
 
     @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index fae06291f8f..56a02f8dd8c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -5,9 +5,15 @@ import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.provision.ActivationContext;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationTransaction;
 import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.Provisioner;
+import com.yahoo.config.provision.TransientException;
+import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer;
 import com.yahoo.vespa.config.server.ApplicationRepository.Activation;
@@ -23,6 +29,7 @@ import java.time.Clock;
 import java.time.Duration;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -38,6 +45,7 @@ import java.util.stream.Collectors;
 public class Deployment implements com.yahoo.config.provision.Deployment {
 
     private static final Logger log = Logger.getLogger(Deployment.class.getName());
+    private static final Duration durationBetweenResourceReadyChecks = Duration.ofSeconds(60);
 
     /** The session containing the application instance to activate */
     private final Session session;
@@ -73,15 +81,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
 
     public static Deployment unprepared(Session session, ApplicationRepository applicationRepository,
                                         Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger,
-                                        Duration timeout, Clock clock, boolean validate, boolean isBootstrap) {
-        Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false);
+                                        Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean waitForResourcesInPrepare) {
+        Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, waitForResourcesInPrepare);
         return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false);
     }
 
     public static Deployment prepared(Session session, ApplicationRepository applicationRepository,
                                       Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger,
                                       Duration timeout, Clock clock, boolean isBootstrap, boolean force) {
-        Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force);
+        Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false);
         return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false, true);
     }
 
@@ -95,6 +103,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
             this.configChangeActions = tenant.getSessionRepository().prepareLocalSession(session, deployLogger, params, clock.instant());
             this.prepared = true;
         }
+
+        waitForResourcesOrTimeout(params, session, provisioner);
     }
 
     /** Activates this. If it is not already prepared, this will call prepare first. */
@@ -195,7 +205,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
      */
     private static Supplier<PrepareParams> createPrepareParams(
             Clock clock, Duration timeout, Session session,
-            boolean isBootstrap, boolean ignoreValidationErrors, boolean force) {
+            boolean isBootstrap, boolean ignoreValidationErrors, boolean force, boolean waitForResourcesInPrepare) {
 
         // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments
         // memoized because we want to create this once for unprepared deployments
@@ -208,7 +218,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
                     .timeoutBudget(timeoutBudget)
                     .ignoreValidationErrors(ignoreValidationErrors)
                     .isBootstrap(isBootstrap)
-                    .force(force);
+                    .force(force)
+                    .waitForResourcesInPrepare(waitForResourcesInPrepare);
             session.getDockerImageRepository().ifPresent(params::dockerImageRepository);
             session.getAthenzDomain().ifPresent(params::athenzDomain);
 
@@ -216,4 +227,33 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         });
     }
 
+    private static void waitForResourcesOrTimeout(PrepareParams params, Session session, Optional<Provisioner> provisioner) {
+        if (!params.waitForResourcesInPrepare() || provisioner.isEmpty()) return;
+
+        Set<HostSpec> preparedHosts = session.getAllocatedHosts().getHosts();
+        ActivationContext context = new ActivationContext(session.getSessionId());
+        ApplicationTransaction transaction = new ApplicationTransaction(
+                new ProvisionLock(session.getApplicationId(), () -> {}), new NestedTransaction());
+        AtomicReference<TransientException> lastException = new AtomicReference<>();
+
+        while (true) {
+            params.getTimeoutBudget().assertNotTimedOut(
+                    () -> "Timeout exceeded while waiting for application resources of '" + session.getApplicationId() + "'" +
+                            Optional.ofNullable(lastException.get()).map(e -> ". Last exception: " + e.getMessage()).orElse(""));
+
+            try {
+                // Call to activate to make sure that everything is ready, but do not commit the transaction
+                provisioner.get().activate(preparedHosts, context, transaction);
+                return;
+            } catch (TransientException e) {
+                lastException.set(e);
+                try {
+                    Thread.sleep(durationBetweenResourceReadyChecks.toMillis());
+                } catch (InterruptedException e1) {
+                    throw new RuntimeException(e1);
+                }
+            }
+        }
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index 139081fde00..b5b7e57e361 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -45,6 +45,7 @@ public final class PrepareParams {
     static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole";
     static final String QUOTA_PARAM_NAME = "quota";
     static final String FORCE_PARAM_NAME = "force";
+    static final String WAIT_FOR_RESOURCES_IN_PREPARE = "waitForResourcesInPrepare";
 
     private final ApplicationId applicationId;
     private final TimeoutBudget timeoutBudget;
@@ -53,6 +54,7 @@ public final class PrepareParams {
     private final boolean verbose;
     private final boolean isBootstrap;
     private final boolean force;
+    private final boolean waitForResourcesInPrepare;
     private final Optional<Version> vespaVersion;
     private final List<ContainerEndpoint> containerEndpoints;
     private final Optional<String> tlsSecretsKeyName;
@@ -67,7 +69,8 @@ public final class PrepareParams {
                           List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName,
                           Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
                           Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain,
-                          Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) {
+                          Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force,
+                          boolean waitForResourcesInPrepare) {
         this.timeoutBudget = timeoutBudget;
         this.applicationId = Objects.requireNonNull(applicationId);
         this.ignoreValidationErrors = ignoreValidationErrors;
@@ -83,6 +86,7 @@ public final class PrepareParams {
         this.applicationRoles = applicationRoles;
         this.quota = quota;
         this.force = force;
+        this.waitForResourcesInPrepare = waitForResourcesInPrepare;
     }
 
     public static class Builder {
@@ -92,6 +96,7 @@ public final class PrepareParams {
         private boolean verbose = false;
         private boolean isBootstrap = false;
         private boolean force = false;
+        private boolean waitForResourcesInPrepare = false;
         private ApplicationId applicationId = null;
         private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60));
         private Optional<Version> vespaVersion = Optional.empty();
@@ -203,6 +208,11 @@ public final class PrepareParams {
             return this;
         }
 
+        public Builder waitForResourcesInPrepare(boolean waitForResourcesInPrepare) {
+            this.waitForResourcesInPrepare = waitForResourcesInPrepare;
+            return this;
+        }
+
         public Builder force(boolean force) {
             this.force = force;
             return this;
@@ -212,7 +222,7 @@ public final class PrepareParams {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
                                      verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName,
                                      endpointCertificateMetadata, dockerImageRepository, athenzDomain,
-                                     applicationRoles, quota, force);
+                                     applicationRoles, quota, force, waitForResourcesInPrepare);
         }
     }
 
@@ -231,6 +241,7 @@ public final class PrepareParams {
                             .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE)))
                             .quota(request.getProperty(QUOTA_PARAM_NAME))
                             .force(request.getBooleanProperty(FORCE_PARAM_NAME))
+                            .waitForResourcesInPrepare(request.getBooleanProperty(WAIT_FOR_RESOURCES_IN_PREPARE))
                             .build();
     }
 
@@ -280,6 +291,8 @@ public final class PrepareParams {
 
     public boolean force() { return force; }
 
+    public boolean waitForResourcesInPrepare() { return waitForResourcesInPrepare; }
+
     public TimeoutBudget getTimeoutBudget() {
         return timeoutBudget;
     }