Don't fail on out of capacity on bootstrap

10 files changed, 85 insertions, 43 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index ab7702e26d1..109afe87e89 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -214,7 +214,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
      *         node in the config server cluster)
      */
     @Override
-    public Optional<com.yahoo.config.provision.Deployment> deployFromLocalActive(ApplicationId application, Duration timeout) {
+    public Optional<com.yahoo.config.provision.Deployment> deployFromLocalActive(ApplicationId application,
+                                                                                 Duration timeout) {
         Tenant tenant = tenantRepository.getTenant(application.tenant());
         if (tenant == null) return Optional.empty();
         LocalSession activeSession = getActiveSession(tenant, application);
@@ -583,8 +584,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         // Keep track of deployment per application
         Map<ApplicationId, Future<?>> futures = new HashMap<>();
         Set<ApplicationId> failedDeployments = new HashSet<>();
-        applicationIds.forEach(appId -> deployFromLocalActive(appId).ifPresent(
-                deployment -> futures.put(appId, executor.submit(deployment::activate))));
+
+        for (ApplicationId appId : applicationIds) {
+            Optional<com.yahoo.config.provision.Deployment> deploymentOptional = deployFromLocalActive(appId);
+            if ( ! deploymentOptional.isPresent()) continue;
+
+            Deployment deployment = (Deployment)deploymentOptional.get();
+            deployment.setBootstrap(true); // Only available inside the config server; hence the cast
+            futures.put(appId, executor.submit(deployment::activate));
+        }
+
         for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
             try {
                 f.getValue().get();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 74757032eaa..acc5e8dcf61 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -56,9 +56,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
     private boolean ignoreLockFailure = false;
     private boolean ignoreSessionStaleFailure = false;
 
+    private boolean isBootstrap = false;
+
     private Deployment(LocalSession session, ApplicationRepository applicationRepository,
                        Optional<Provisioner> hostProvisioner, Tenant tenant,
-                       Duration timeout, Clock clock, boolean prepared, boolean validate, Version version) {
+                       Duration timeout, Clock clock, boolean prepared, boolean validate, Version version,
+                       boolean isBootstrap) {
         this.session = session;
         this.applicationRepository = applicationRepository;
         this.hostProvisioner = hostProvisioner;
@@ -68,20 +71,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         this.prepared = prepared;
         this.validate = validate;
         this.version = version;
+        this.isBootstrap = isBootstrap;
     }
 
     public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository,
                                         Optional<Provisioner> hostProvisioner, Tenant tenant,
                                         Duration timeout, Clock clock, boolean validate, Version version) {
         return new Deployment(session, applicationRepository, hostProvisioner, tenant,
-                              timeout, clock, false, validate, version);
+                              timeout, clock, false, validate, version, false);
     }
 
     public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository,
                                       Optional<Provisioner> hostProvisioner, Tenant tenant,
                                       Duration timeout, Clock clock) {
         return new Deployment(session, applicationRepository, hostProvisioner, tenant,
-                              timeout, clock, true, true, session.getVespaVersion());
+                              timeout, clock, true, true, session.getVespaVersion(), false);
     }
 
     public Deployment setIgnoreLockFailure(boolean ignoreLockFailure) {
@@ -105,6 +109,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
                                                    .timeoutBudget(timeoutBudget)
                                                    .ignoreValidationErrors( ! validate)
                                                    .vespaVersion(version.toString())
+                                                   .isBootstrap(isBootstrap)
                                                    .build(),
                         Optional.empty(),
                         tenant.getPath(),
@@ -161,10 +166,14 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
      * This is sometimes needed after activation, but can also be requested without
      * doing prepare and activate in the same session.
      */
+    @Override
     public void restart(HostFilter filter) {
         hostProvisioner.get().restart(session.getApplicationId(), filter);
     }
 
+    /** Set this to true if this deployment is done to bootstrap the config server */
+    public void setBootstrap(boolean isBootstrap) { this.isBootstrap = isBootstrap; }
+
     /** Exposes the session of this for testing only */
     public LocalSession session() { return session; }
     
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index cec879c6e14..28dc0cc8414 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -123,6 +123,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean hostedVespa;
         private final Zone zone;
         private final Set<Rotation> rotations;
+        private final boolean isBootstrap;
 
         public Properties(ApplicationId applicationId,
                           boolean multitenant,
@@ -132,7 +133,8 @@ public class ModelContextImpl implements ModelContext {
                           String athenzDnsSuffix,
                           boolean hostedVespa,
                           Zone zone,
-                          Set<Rotation> rotations) {
+                          Set<Rotation> rotations,
+                          boolean isBootstrap) {
             this.applicationId = applicationId;
             this.multitenant = multitenant;
             this.configServerSpecs = configServerSpecs;
@@ -142,6 +144,7 @@ public class ModelContextImpl implements ModelContext {
             this.hostedVespa = hostedVespa;
             this.zone = zone;
             this.rotations = rotations;
+            this.isBootstrap = isBootstrap;
         }
 
         @Override
@@ -175,6 +178,9 @@ public class ModelContextImpl implements ModelContext {
         @Override
         public Set<Rotation> rotations() { return rotations; }
 
+        @Override
+        public boolean isBootstrap() { return isBootstrap; }
+
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index 651e5a6bbb0..6b872fc4601 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -10,9 +10,11 @@ import com.yahoo.config.model.api.ModelFactory;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Version;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.server.ConfigServerSpec;
 import com.yahoo.vespa.config.server.GlobalComponentRegistry;
 import com.yahoo.vespa.config.server.tenant.Rotations;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
@@ -25,6 +27,7 @@ import com.yahoo.vespa.config.server.session.SessionZooKeeperClient;
 import com.yahoo.vespa.config.server.session.SilentDeployLogger;
 import com.yahoo.vespa.curator.Curator;
 
+import java.net.URI;
 import java.time.Instant;
 import java.util.Map;
 import java.util.Optional;
@@ -104,11 +107,16 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
     }
 
     private ModelContext.Properties createModelContextProperties(ApplicationId applicationId) {
-        return createModelContextProperties(
-                applicationId,
-                configserverConfig,
-                zone(),
-                new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId));
+        return new ModelContextImpl.Properties(applicationId,
+                                               configserverConfig.multitenant(),
+                                               ConfigServerSpec.fromConfig(configserverConfig),
+                                               HostName.from(configserverConfig.loadBalancerAddress()),
+                                               configserverConfig.ztsUrl() != null ? URI.create(configserverConfig.ztsUrl()) : null,
+                                               configserverConfig.athenzDnsSuffix(),
+                                               configserverConfig.hostedVespa(),
+                                               zone(),
+                                               new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId),
+                                               false); // We may be bootstrapping, but we only know and care during prepare
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 560f650dcba..b2eea704a5e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -2,29 +2,22 @@
 package com.yahoo.vespa.config.server.modelfactory;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
-import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.api.HostProvisioner;
-import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelFactory;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.OutOfCapacityException;
-import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.Version;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.lang.SettableOptional;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.config.server.ConfigServerSpec;
-import com.yahoo.vespa.config.server.deploy.ModelContextImpl;
 import com.yahoo.vespa.config.server.http.InternalServerException;
 import com.yahoo.vespa.config.server.http.UnknownVespaVersionException;
 import com.yahoo.vespa.config.server.provision.StaticProvisioner;
 
-import java.net.URI;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -209,22 +202,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                                                      Optional<AllocatedHosts> allocatedHosts,
                                                      Instant now);
 
-    protected ModelContext.Properties createModelContextProperties(ApplicationId applicationId,
-                                                                   ConfigserverConfig configserverConfig,
-                                                                   Zone zone,
-                                                                   Set<Rotation> rotations) {
-        return new ModelContextImpl.Properties(applicationId,
-                                               configserverConfig.multitenant(),
-                                               ConfigServerSpec.fromConfig(configserverConfig),
-                                               HostName.from(configserverConfig.loadBalancerAddress()),
-                                               configserverConfig.ztsUrl() != null ? URI.create(configserverConfig.ztsUrl()) : null,
-                                               configserverConfig.athenzDnsSuffix(),
-                                               configserverConfig.hostedVespa(),
-                                               zone,
-                                               rotations);
-    }
-
-    /** 
+    /**
      * Returns a host provisioner returning the previously allocated hosts if available and when on hosted Vespa,
      * returns empty otherwise, which may either mean that no hosts are allocated or that we are running
      * non-hosted and should default to use hosts defined in the application package, depending on context
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index f53a48b3783..b44896ecb89 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -35,16 +35,18 @@ public final class PrepareParams {
     private final boolean ignoreValidationErrors;
     private final boolean dryRun;
     private final boolean verbose;
+    private final boolean isBootstrap;
     private final Optional<Version> vespaVersion;
     private final Set<Rotation> rotations;
 
     private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
-                         boolean dryRun, boolean verbose, Optional<Version> vespaVersion, Set<Rotation> rotations) {
+                         boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, Set<Rotation> rotations) {
         this.timeoutBudget = timeoutBudget;
         this.applicationId = applicationId;
         this.ignoreValidationErrors = ignoreValidationErrors;
         this.dryRun = dryRun;
         this.verbose = verbose;
+        this.isBootstrap = isBootstrap;
         this.vespaVersion = vespaVersion;
         this.rotations = rotations;
     }
@@ -54,6 +56,7 @@ public final class PrepareParams {
         private boolean ignoreValidationErrors = false;
         private boolean dryRun = false;
         private boolean verbose = false;
+        private boolean isBootstrap = false;
         private ApplicationId applicationId = ApplicationId.defaultId();
         private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(30));
         private Optional<Version> vespaVersion = Optional.empty();
@@ -81,6 +84,11 @@ public final class PrepareParams {
             return this;
         }
 
+        public Builder isBootstrap(boolean isBootstrap) {
+            this.isBootstrap = isBootstrap;
+            return this;
+        }
+
         public Builder timeoutBudget(TimeoutBudget timeoutBudget) {
             this.timeoutBudget = timeoutBudget;
             return this;
@@ -113,7 +121,7 @@ public final class PrepareParams {
 
         public PrepareParams build() {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, 
-                                     verbose, vespaVersion, rotations);
+                                     verbose, isBootstrap, vespaVersion, rotations);
         }
 
     }
@@ -170,6 +178,8 @@ public final class PrepareParams {
         return verbose;
     }
 
+    public boolean isBootstrap() { return isBootstrap; }
+
     public TimeoutBudget getTimeoutBudget() {
         return timeoutBudget;
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 114ad936eda..0d9f8ce64b1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -157,7 +157,8 @@ public class SessionPreparer {
                                                               configserverConfig.athenzDnsSuffix(),
                                                               configserverConfig.hostedVespa(),
                                                               zone,
-                                                              rotationsSet);
+                                                              rotationsSet,
+                                                              params.isBootstrap());
             this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry,
                                                                    permanentApplicationPackage,
                                                                    configDefinitionRepo,
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 082c3058598..f9c99b323af 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -2,6 +2,10 @@
 package com.yahoo.vespa.config.server;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.provision.InMemoryProvisioner;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Version;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.container.jdisc.state.StateMonitor;
@@ -34,9 +38,10 @@ public class ConfigServerBootstrapTest {
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
     @Test
-    public void testBootStrap() throws Exception {
+    public void testBootstrap() throws Exception {
         ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
-        DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig);
+        InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3");
+        DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig, provisioner);
         tester.deployApp("myApp", "4.5.6", Instant.now());
 
         File versionFile = temporaryFolder.newFile();
@@ -45,6 +50,13 @@ public class ConfigServerBootstrapTest {
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
         VipStatus vipStatus = new VipStatus();
+        // Take a host away so that there are too few for the application, to verify we can still bootstrap
+        ClusterSpec contentCluster = ClusterSpec.from(ClusterSpec.Type.content,
+                                                      ClusterSpec.Id.from("music"),
+                                                      ClusterSpec.Group.from(0),
+                                                      new com.yahoo.component.Version(4, 5, 6),
+                                                      false);
+        provisioner.allocations().get(contentCluster).remove(0);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, createStateMonitor(), vipStatus);
         assertFalse(vipStatus.isInRotation());
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
@@ -58,7 +70,7 @@ public class ConfigServerBootstrapTest {
     }
 
     @Test
-    public void testBootStrapWhenRedeploymentFails() throws Exception {
+    public void testBootstrapWhenRedeploymentFails() throws Exception {
         ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
         DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig);
         tester.deployApp("myApp", "4.5.6", Instant.now());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
index 5d9a5f0fadc..ce53dc3f2fb 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
@@ -50,7 +50,8 @@ public class ModelContextImplTest {
                         null,
                         false,
                         Zone.defaultZone(),
-                        rotations),
+                        rotations,
+                        false),
                 Optional.empty(),
                 new Version(6), 
                 new Version(6));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index a4f5679aa39..b15356a172e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -83,6 +83,10 @@ public class DeployTester {
         this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC());
     }
 
+    public DeployTester(String appPath, ConfigserverConfig configserverConfig, HostProvisioner provisioner) {
+        this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC(), provisioner);
+    }
+
     public DeployTester(String appPath, ConfigserverConfig configserverConfig, Clock clock) {
         this(appPath, Collections.singletonList(createModelFactory(clock)), configserverConfig, clock);
     }
@@ -95,8 +99,12 @@ public class DeployTester {
         this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone());
     }
 
+    public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) {
+        this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner);
+    }
+
     public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) {
-        this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone(), createProvisioner());
+        this(appPath, modelFactories, configserverConfig, clock, zone, createProvisioner());
     }
 
     public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) {