diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2021-01-21 20:18:28 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-21 20:18:28 +0100 |
commit | ca7e5417d7cb2a73e50d2ad357fd719d258ea4e0 (patch) | |
tree | 98290a09667410a64b512efa69a15876e532e973 /configserver/src | |
parent | aadd34503c768add53c6bd941970e38314870fd9 (diff) | |
parent | 006f2ecfc15adf1e57e61cb99ca82aae0adb3a71 (diff) |
Merge pull request #16147 from vespa-engine/freva/blocking-prepare
Wait for resources in prepare
Diffstat (limited to 'configserver/src')
3 files changed, 66 insertions, 8 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index a341063ddd7..f2f5eb3afa1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -73,7 +73,9 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.stats.LockStats; import com.yahoo.vespa.curator.stats.ThreadLockStats; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -138,6 +140,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final TesterClient testerClient; private final Metric metric; private final ClusterReindexingStatusClient clusterReindexingStatusClient; + private final BooleanFlag waitForResourcesInPrepareFlag; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -190,6 +193,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); this.clusterReindexingStatusClient = clusterReindexingStatusClient; + this.waitForResourcesInPrepareFlag = Flags.WAIT_FOR_RESOURCES_IN_PREPARE.bindTo(flagSource); } public static class Builder { @@ -397,9 +401,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye SessionRepository sessionRepository = tenant.getSessionRepository(); DeployLogger logger = new SilentDeployLogger(); Session newSession = sessionRepository.createSessionFromExisting(activeSession, true, timeoutBudget); + boolean waitForResourcesInPrepare = waitForResourcesInPrepareFlag.value(); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, - false /* don't validate as this is already deployed */, bootstrap)); + false /* don't validate as this is already deployed */, bootstrap, waitForResourcesInPrepare)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index fae06291f8f..ff2bbc12e29 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -5,9 +5,15 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TransientException; +import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.ApplicationRepository.Activation; @@ -23,6 +29,7 @@ import java.time.Clock; import java.time.Duration; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -38,6 +45,7 @@ import java.util.stream.Collectors; public class Deployment implements com.yahoo.config.provision.Deployment { private static final Logger log = Logger.getLogger(Deployment.class.getName()); + private static final Duration durationBetweenResourceReadyChecks = Duration.ofSeconds(60); /** The session containing the application instance to activate */ private final Session session; @@ -73,15 +81,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public static Deployment unprepared(Session session, ApplicationRepository applicationRepository, Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, - Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { - Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false); + Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean waitForResourcesInPrepare) { + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, waitForResourcesInPrepare); return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false); } public static Deployment prepared(Session session, ApplicationRepository applicationRepository, Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { - Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force); + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false); return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false, true); } @@ -95,6 +103,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.configChangeActions = tenant.getSessionRepository().prepareLocalSession(session, deployLogger, params, clock.instant()); this.prepared = true; } + + waitForResourcesOrTimeout(params, session, provisioner); } /** Activates this. If it is not already prepared, this will call prepare first. */ @@ -195,7 +205,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { */ private static Supplier<PrepareParams> createPrepareParams( Clock clock, Duration timeout, Session session, - boolean isBootstrap, boolean ignoreValidationErrors, boolean force) { + boolean isBootstrap, boolean ignoreValidationErrors, boolean force, boolean waitForResourcesInPrepare) { // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments // memoized because we want to create this once for unprepared deployments @@ -208,7 +218,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .timeoutBudget(timeoutBudget) .ignoreValidationErrors(ignoreValidationErrors) .isBootstrap(isBootstrap) - .force(force); + .force(force) + .waitForResourcesInPrepare(waitForResourcesInPrepare); session.getDockerImageRepository().ifPresent(params::dockerImageRepository); session.getAthenzDomain().ifPresent(params::athenzDomain); @@ -216,4 +227,33 @@ public class Deployment implements com.yahoo.config.provision.Deployment { }); } + private static void waitForResourcesOrTimeout(PrepareParams params, Session session, Optional<Provisioner> provisioner) { + if (!params.waitForResourcesInPrepare() || provisioner.isEmpty()) return; + + Set<HostSpec> preparedHosts = session.getAllocatedHosts().getHosts(); + ActivationContext context = new ActivationContext(session.getSessionId()); + ProvisionLock lock = new ProvisionLock(session.getApplicationId(), () -> {}); + AtomicReference<TransientException> lastException = new AtomicReference<>(); + + while (true) { + params.getTimeoutBudget().assertNotTimedOut( + () -> "Timeout exceeded while waiting for application resources of '" + session.getApplicationId() + "'" + + Optional.ofNullable(lastException.get()).map(e -> ". Last exception: " + e.getMessage()).orElse("")); + + try { + // Call to activate to make sure that everything is ready, but do not commit the transaction + ApplicationTransaction transaction = new ApplicationTransaction(lock, new NestedTransaction()); + provisioner.get().activate(preparedHosts, context, transaction); + return; + } catch (TransientException e) { + lastException.set(e); + try { + Thread.sleep(durationBetweenResourceReadyChecks.toMillis()); + } catch (InterruptedException e1) { + throw new RuntimeException(e1); + } + } + } + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 78071cbf89e..1cee80038e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -44,6 +44,7 @@ public final class PrepareParams { static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; static final String QUOTA_PARAM_NAME = "quota"; static final String FORCE_PARAM_NAME = "force"; + static final String WAIT_FOR_RESOURCES_IN_PREPARE = "waitForResourcesInPrepare"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -52,6 +53,7 @@ public final class PrepareParams { private final boolean verbose; private final boolean isBootstrap; private final boolean force; + private final boolean waitForResourcesInPrepare; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; @@ -65,7 +67,8 @@ public final class PrepareParams { List<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, - Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { + Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force, + boolean waitForResourcesInPrepare) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -80,6 +83,7 @@ public final class PrepareParams { this.applicationRoles = applicationRoles; this.quota = quota; this.force = force; + this.waitForResourcesInPrepare = waitForResourcesInPrepare; } public static class Builder { @@ -89,6 +93,7 @@ public final class PrepareParams { private boolean verbose = false; private boolean isBootstrap = false; private boolean force = false; + private boolean waitForResourcesInPrepare = false; private ApplicationId applicationId = null; private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); @@ -193,6 +198,11 @@ public final class PrepareParams { return this; } + public Builder waitForResourcesInPrepare(boolean waitForResourcesInPrepare) { + this.waitForResourcesInPrepare = waitForResourcesInPrepare; + return this; + } + public Builder force(boolean force) { this.force = force; return this; @@ -202,7 +212,7 @@ public final class PrepareParams { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, force); + applicationRoles, quota, force, waitForResourcesInPrepare); } } @@ -220,6 +230,7 @@ public final class PrepareParams { .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .quota(request.getProperty(QUOTA_PARAM_NAME)) .force(request.getBooleanProperty(FORCE_PARAM_NAME)) + .waitForResourcesInPrepare(request.getBooleanProperty(WAIT_FOR_RESOURCES_IN_PREPARE)) .build(); } @@ -269,6 +280,8 @@ public final class PrepareParams { public boolean force() { return force; } + public boolean waitForResourcesInPrepare() { return waitForResourcesInPrepare; } + public TimeoutBudget getTimeoutBudget() { return timeoutBudget; } |