Don't fail on out of capacity on bootstrap

10 files changed, 93 insertions, 43 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
index d3e91f8866c..b259f6cf3fb 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
@@ -25,6 +25,7 @@ public class DeployProperties {
     private final String athenzDnsSuffix;
     private final boolean hostedVespa;
     private final Version vespaVersion;
+    private final boolean isBootstrap;
 
     private DeployProperties(boolean multitenant,
                              ApplicationId applicationId,
@@ -33,7 +34,8 @@ public class DeployProperties {
                              boolean hostedVespa,
                              URI ztsUrl,
                              String athenzDnsSuffix,
-                             Version vespaVersion) {
+                             Version vespaVersion,
+                             boolean isBootstrap) {
         this.loadBalancerName = loadBalancerName;
         this.ztsUrl = ztsUrl;
         this.athenzDnsSuffix = athenzDnsSuffix;
@@ -42,9 +44,9 @@ public class DeployProperties {
         this.applicationId = applicationId;
         this.serverSpecs.addAll(configServerSpecs);
         this.hostedVespa = hostedVespa;
+        this.isBootstrap = isBootstrap;
     }
 
-
     public boolean multitenant() {
         return multitenant;
     }
@@ -78,6 +80,9 @@ public class DeployProperties {
         return vespaVersion;
     }
 
+    /** Returns whether this deployment happens during bootstrap *prepare* (not set on activate) */
+    public boolean isBootstrap() { return isBootstrap; }
+
     public static class Builder {
 
         private ApplicationId applicationId = ApplicationId.defaultId();
@@ -88,6 +93,7 @@ public class DeployProperties {
         private String athenzDnsSuffix;
         private boolean hostedVespa = false;
         private Version vespaVersion = Version.fromIntValues(1, 0, 0);
+        private boolean isBootstrap = false;
 
         public Builder applicationId(ApplicationId applicationId) {
             this.applicationId = applicationId;
@@ -129,8 +135,13 @@ public class DeployProperties {
             return this;
         }
 
+        public Builder isBootstrap(boolean isBootstrap) {
+            this.isBootstrap = isBootstrap;
+            return this;
+        }
+
         public DeployProperties build() {
-            return new DeployProperties(multitenant, applicationId, configServerSpecs, loadBalancerName, hostedVespa, ztsUrl, athenzDnsSuffix, vespaVersion);
+            return new DeployProperties(multitenant, applicationId, configServerSpecs, loadBalancerName, hostedVespa, ztsUrl, athenzDnsSuffix, vespaVersion, isBootstrap);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index e6b89d7f390..c744c509b9a 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -24,7 +24,8 @@ import java.util.Optional;
 import java.util.Set;
 
 /**
- * In memory host provisioner. NB! ATM cannot be reused after allocate has been called.
+ * In memory host provisioner for testing only.
+ * NB! ATM cannot be reused after allocate has been called.
  *
  * @author hmusum
  * @author bratseth
@@ -97,6 +98,9 @@ public class InMemoryProvisioner implements HostProvisioner {
         return hosts;
     }
 
+    /** Returns the current allocations of this as a mutable map */
+    public Map<ClusterSpec, List<HostSpec>> allocations() { return allocations; }
+
     @Override
     public HostSpec allocateHost(String alias) {
         if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias);
@@ -129,14 +133,16 @@ public class InMemoryProvisioner implements HostProvisioner {
             allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))),
                                                 flavor,
                                                 capacity,
-                                                startIndexForClusters));
+                                                startIndexForClusters,
+                                                requestedCapacity.canFail()));
         }
         else {
             for (int i = 0; i < groups; i++) {
                 allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))),
                                                     flavor,
                                                     capacity / groups,
-                                                    allocation.size()));
+                                                    allocation.size(),
+                                                    requestedCapacity.canFail()));
             }
         }
         for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) {
@@ -155,13 +161,18 @@ public class InMemoryProvisioner implements HostProvisioner {
                             host.version());
     }
 
-    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex) {
+    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex, boolean canFail) {
         List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
         allocations.put(clusterGroup, allocation);
 
         int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex);
         while (allocation.size() < nodesInGroup) {
-            if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("Insufficient capacity of flavor '" + flavor + "'");
+            if (freeNodes.get(flavor).isEmpty()) {
+                if (canFail)
+                    throw new IllegalArgumentException("Insufficient capacity of flavor '" + flavor + "'");
+                else
+                    break;
+            }
             Host newHost = freeNodes.removeValue(flavor, 0);
             ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
             allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.of(membership), newHost.version()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
index a6d24f33b5d..e88153342f9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
@@ -153,6 +153,7 @@ public class VespaModelFactory implements ModelFactory {
                 .multitenant(properties.multitenant())
                 .hostedVespa(properties.hostedVespa())
                 .vespaVersion(getVersion())
+                .isBootstrap(properties.isBootstrap())
                 .build();
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index 44de53991f4..ff8d3211d47 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -45,17 +45,16 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
     protected void doBuildAdmin(Admin admin, Element w3cAdminElement) {
         ModelElement adminElement = new ModelElement(w3cAdminElement);
         admin.addConfigservers(getConfigServersFromSpec(admin));
-        Version version = context.getDeployState().getWantedNodeVespaVersion();
-        
+
         // Note: These two elements only exists in admin version 4.0
         // This build handles admin version 3.0 by ignoring its content (as the content is not useful)
         Optional<NodesSpecification> requestedSlobroks = 
-                NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("slobroks"), version);
+                NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("slobroks"), context);
         Optional<NodesSpecification> requestedLogservers = 
-                NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers"), version);
+                NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers"), context);
 
-        assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, version)), admin);
-        assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1, version)), admin);
+        assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, context)), admin);
+        assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1, context)), admin);
 
         addLogForwarders(adminElement.getChild("logforwarding"), admin);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index f7ed7241133..94359e8672e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.model.builder.xml.dom;
 
 import com.yahoo.component.Version;
+import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
@@ -34,34 +35,38 @@ public class NodesSpecification {
      * at the discretion of the component fulfilling it
      */
     private final boolean required;
-    
-    /** The flavor the nodes should have, or empty to use the default */        
-    private final Optional<String> flavor;
 
+    private final boolean canFail;
+    
     private final boolean exclusive;
 
+    /** The flavor the nodes should have, or empty to use the default */
+    private final Optional<String> flavor;
+
     /** The identifier of the custom docker image layer to use (not supported yet) */
     private final Optional<String> dockerImage;
 
-    private NodesSpecification(boolean dedicated, int count, int groups, Version version, boolean required,
-                               boolean exclusive,
+    private NodesSpecification(boolean dedicated, int count, int groups, Version version,
+                               boolean required, boolean canFail, boolean exclusive,
                                Optional<String> flavor, Optional<String> dockerImage) {
         this.dedicated = dedicated;
         this.count = count;
         this.groups = groups;
         this.version = version;
         this.required = required;
+        this.canFail = canFail;
         this.exclusive = exclusive;
         this.flavor = flavor;
         this.dockerImage = dockerImage;
     }
 
-    private NodesSpecification(boolean dedicated, Version version, ModelElement nodesElement) {
+    private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) {
         this(dedicated,
              nodesElement.requiredIntegerAttribute("count"),
              nodesElement.getIntegerAttribute("groups", 1),
              version,
              nodesElement.getBooleanAttribute("required", false),
+             canFail,
              nodesElement.getBooleanAttribute("exclusive", false),
              Optional.ofNullable(nodesElement.getStringAttribute("flavor")),
              Optional.ofNullable(nodesElement.getStringAttribute("docker-image")));
@@ -70,8 +75,11 @@ public class NodesSpecification {
     /**
      * Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element
      */
-    public static NodesSpecification from(ModelElement nodesElement, Version version) {
-        return new NodesSpecification(true, version, nodesElement);
+    public static NodesSpecification from(ModelElement nodesElement, ConfigModelContext context) {
+        return new NodesSpecification(true,
+                                      ! context.getDeployState().getProperties().isBootstrap(),
+                                      context.getDeployState().getWantedNodeVespaVersion(),
+                                      nodesElement);
     }
 
     /**
@@ -79,11 +87,11 @@ public class NodesSpecification {
      * contained in the given parent element, or empty if the parent element is null, or the nodes elements
      * is not present.
      */
-    public static Optional<NodesSpecification> fromParent(ModelElement parentElement, Version version) {
+    public static Optional<NodesSpecification> fromParent(ModelElement parentElement, ConfigModelContext context) {
         if (parentElement == null) return Optional.empty();
         ModelElement nodesElement = parentElement.getChild("nodes");
         if (nodesElement == null) return Optional.empty();
-        return Optional.of(from(nodesElement, version));
+        return Optional.of(from(nodesElement, context));
     }
 
     /**
@@ -91,18 +99,28 @@ public class NodesSpecification {
      * contained in the given parent element, or empty if the parent element is null, or the nodes elements
      * is not present.
      */
-    public static Optional<NodesSpecification> optionalDedicatedFromParent(ModelElement parentElement, Version version) {
+    public static Optional<NodesSpecification> optionalDedicatedFromParent(ModelElement parentElement,
+                                                                           ConfigModelContext context) {
         if (parentElement == null) return Optional.empty();
         ModelElement nodesElement = parentElement.getChild("nodes");
         if (nodesElement == null) return Optional.empty();
         return Optional.of(new NodesSpecification(nodesElement.getBooleanAttribute("dedicated", false),
-                                                  version, nodesElement));
+                                                  ! context.getDeployState().getProperties().isBootstrap(),
+                                                  context.getDeployState().getWantedNodeVespaVersion(),
+                                                  nodesElement));
     }
 
     /** Returns a requirement from <code>count</code> nondedicated nodes in one group */
-    public static NodesSpecification nonDedicated(int count, Version version) {
-        return new NodesSpecification(false, count, 1, version, false, false,
-                                      Optional.empty(), Optional.empty());
+    public static NodesSpecification nonDedicated(int count, ConfigModelContext context) {
+        return new NodesSpecification(false,
+                                      count,
+                                      1,
+                                      context.getDeployState().getWantedNodeVespaVersion(),
+                                      false,
+                                      ! context.getDeployState().getProperties().isBootstrap(),
+                                      false,
+                                      Optional.empty(),
+                                      Optional.empty());
     }
 
     /**
@@ -124,9 +142,12 @@ public class NodesSpecification {
     /** Returns the number of host groups this specifies. Default is 1 */
     public int groups() { return groups; }
 
-    public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger) {
+    public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem,
+                                                          ClusterSpec.Type clusterType,
+                                                          ClusterSpec.Id clusterId,
+                                                          DeployLogger logger) {
         ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive);
-        return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor, required), groups, logger);
+        return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor, required, canFail), groups, logger);
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 2572b0d772b..a007c4765c0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -151,7 +151,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
     private void addClusterContent(ContainerCluster cluster, Element spec, ConfigModelContext context) {
         DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);
-
         addConfiguredComponents(cluster, spec);
         addSecretStore(cluster, spec);
         addHandlers(cluster, spec);
@@ -514,7 +513,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                                                               ClusterSpec.Id.from(cluster.getName()),
                                                               context.getDeployState().getWantedNodeVespaVersion(),
                                                               false);
-                return cluster.getHostSystem().allocateHosts(clusterSpec, Capacity.fromNodeCount(1), 1, logger).keySet().iterator().next();
+                Capacity capacity = Capacity.fromNodeCount(1,
+                                                           Optional.empty(),
+                                                           false,
+                                                           ! context.getDeployState().getProperties().isBootstrap());
+                return cluster.getHostSystem().allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next();
             }
         } else {
             return cluster.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC);
@@ -522,8 +525,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     }
 
     private List<Container> createNodesFromNodeCount(ContainerCluster cluster, Element nodesElement, ConfigModelContext context) {
-        NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), 
-                                                                        context.getDeployState().getWantedNodeVespaVersion());
+        NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context);
         Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(),
                                                                                   ClusterSpec.Type.container,
                                                                                   ClusterSpec.Id.from(cluster.getName()), 
@@ -559,8 +561,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         cluster.setHostClusterId(referenceId);
 
         Map<HostResource, ClusterMembership> hosts = 
-                StorageGroup.provisionHosts(NodesSpecification.from(new ModelElement(referencedNodesElement), 
-                                                                    context.getDeployState().getWantedNodeVespaVersion()), 
+                StorageGroup.provisionHosts(NodesSpecification.from(new ModelElement(referencedNodesElement), context),
                                             referenceId, 
                                             cluster.getRoot().getHostSystem(),
                                             context.getDeployLogger());
@@ -582,9 +583,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         
         NodesSpecification nodesSpec;
         if (contentNodesElementOrNull == null)
-            nodesSpec = NodesSpecification.nonDedicated(1, context.getDeployState().getWantedNodeVespaVersion());
+            nodesSpec = NodesSpecification.nonDedicated(1, context);
         else
-            nodesSpec = NodesSpecification.from(new ModelElement(contentNodesElementOrNull), context.getDeployState().getWantedNodeVespaVersion());
+            nodesSpec = NodesSpecification.from(new ModelElement(contentNodesElementOrNull), context);
 
         Map<HostResource, ClusterMembership> hosts =
                 StorageGroup.provisionHosts(nodesSpec,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index edf3f7c840d..1b5baefe4e4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -409,9 +409,9 @@ public class StorageGroup {
 
             Optional<NodesSpecification> nodeRequirement;
             if (nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ) // request these nodes
-                nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context.getDeployState().getWantedNodeVespaVersion()));
+                nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context));
             else if (! nodesElement.isPresent() && subGroups.isEmpty() && owner.getRoot().getDeployState().isHosted()) // request one node
-                nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1, context.getDeployState().getWantedNodeVespaVersion()));
+                nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1, context));
             else // Nodes or groups explicitly listed, and/opr not hosted - resolve in GroupBuilder
                 nodeRequirement = Optional.empty();
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 68ae4d2b242..154f719ff10 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -292,8 +292,8 @@ public class ContentCluster extends AbstractConfigProducer implements
             else if (admin.multitenant()) {
                 String clusterName = contentClusterName + "-controllers";
                 NodesSpecification nodesSpecification =
-                    NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers"), context.getDeployState().getWantedNodeVespaVersion())
-                                      .orElse(NodesSpecification.nonDedicated(3, context.getDeployState().getWantedNodeVespaVersion()));
+                    NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers"), context)
+                                      .orElse(NodesSpecification.nonDedicated(3, context));
                 Collection<HostResource> hosts = nodesSpecification.isDedicated() ?
                                                  getControllerHosts(nodesSpecification, admin, clusterName, context) :
                                                  drawControllerHosts(nodesSpecification.count(), rootGroup, containers);
diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
index c3fffa96076..6821b7e3b4b 100644
--- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
+++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
@@ -127,6 +127,9 @@ public class MockModelContext implements ModelContext {
             public Set<Rotation> rotations() {
                 return new HashSet<>();
             }
+
+            @Override
+            public boolean isBootstrap() { return false; }
         };
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index 4d221af45a0..2261affb65b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -209,6 +209,9 @@ public class VespaModelFactoryTest {
                     public String athenzDnsSuffix() {
                         return null;
                     }
+
+                    @Override
+                    public boolean isBootstrap() { return false; }
                 };
             }
         };