Merge pull request #12717 from vespa-engine/bratseth/autoscaling-ranges-1

Bratseth/autoscaling ranges 1 .

8 files changed, 57 insertions, 32 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
index f909f3864da..201b69c1aae 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner {
     }
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) {
         throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
     }
 
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) {
+        throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
+    }
+
     private HostSpec host2HostSpec(Host host) {
         return new HostSpec(host.hostname(), host.aliases());
     }
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 6047b6a9818..2a21af606ef 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -113,23 +113,25 @@ public class InMemoryProvisioner implements HostProvisioner {
     }
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
-        if (cluster.group().isPresent() && groups > 1)
+        return prepare(cluster, requestedCapacity.withGroups(groups), logger);
+    }
+
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, ProvisionLogger logger) {
+        if (cluster.group().isPresent() && requestedCapacity.minResources().groups() > 1)
             throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
-        if (requestedCapacity.nodeCount() % groups != 0)
-            throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
-                                               groups + " groups, but the node count is not divisible into this number of groups");
 
         int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() 
-                       ? requestedCapacity.nodeCount() 
-                       : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
-        if (groups > capacity)
-            groups = capacity;
+                       ? requestedCapacity.minResources().nodes()
+                       : Math.min(requestedCapacity.minResources().nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
+        int groups = requestedCapacity.minResources().groups() > capacity ? capacity : requestedCapacity.minResources().groups();
 
         List<HostSpec> allocation = new ArrayList<>();
         if (groups == 1) {
             allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))),
-                                                requestedCapacity.nodeResources(),
+                                                requestedCapacity.minResources().nodeResources(),
                                                 capacity,
                                                 startIndexForClusters,
                                                 requestedCapacity.canFail()));
@@ -137,7 +139,7 @@ public class InMemoryProvisioner implements HostProvisioner {
         else {
             for (int i = 0; i < groups; i++) {
                 allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))),
-                                                    requestedCapacity.nodeResources(),
+                                                    requestedCapacity.minResources().nodeResources(),
                                                     capacity / groups,
                                                     allocation.size(),
                                                     requestedCapacity.canFail()));
@@ -162,7 +164,7 @@ public class InMemoryProvisioner implements HostProvisioner {
                             host.dockerImageRepo());
     }
 
-    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources,
+    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources,
                                              int nodesInGroup, int startIndex, boolean canFail) {
         List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
         allocations.put(clusterGroup, allocation);
@@ -170,8 +172,8 @@ public class InMemoryProvisioner implements HostProvisioner {
         // Check if the current allocations are compatible with the new request
         for (int i = allocation.size() - 1; i >= 0; i--) {
             Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources);
-            if (currentResources.isEmpty() || requestedResources.isEmpty()) continue;
-            if (!currentResources.get().compatibleWith(requestedResources.get())) {
+            if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue;
+            if (!currentResources.get().compatibleWith(requestedResources)) {
                 HostSpec removed = allocation.remove(i);
                 freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool
             }
@@ -182,7 +184,7 @@ public class InMemoryProvisioner implements HostProvisioner {
             // Find the smallest host that can fit the requested requested
             Optional<NodeResources> hostResources = freeNodes.keySet().stream()
                     .sorted(new MemoryDiskCpu())
-                    .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get()))
+                    .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources))
                     .findFirst();
             if (hostResources.isEmpty()) {
                 if (canFail)
@@ -195,8 +197,9 @@ public class InMemoryProvisioner implements HostProvisioner {
             if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get());
             ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
             allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(),
-                    hostResources.map(Flavor::new), Optional.of(membership),
-                    newHost.version(), Optional.empty(), requestedResources));
+                                        hostResources.map(Flavor::new), Optional.of(membership),
+                                        newHost.version(), Optional.empty(),
+                                        requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources)));
         }
         nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
 
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
index 7d9b78719ea..8945223447f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -42,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner {
     }
 
     @Override
-    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1
+    @Deprecated // TODO: Remove after April 2020
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+        return prepare(cluster, capacity.withGroups(groups), logger);
+    }
+
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+        // TODO: This should fail if capacity requested is more than 1
         List<HostSpec> hosts = new ArrayList<>();
         hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++)));
         return hosts;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
index 3765e683b18..557a61ec211 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -111,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> {
         }
     }
 
-    public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) {
-        List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger));
+    public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) {
+        List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger));
         // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership.
         Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>();
         for (HostSpec spec : allocatedHosts) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index 384d891003e..20d101b07d9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.text.XML;
@@ -201,7 +202,9 @@ public class NodesSpecification {
                 .combinedId(combinedId.map(ClusterSpec.Id::from))
                 .dockerImageRepo(dockerImageRepo)
                 .build();
-        return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger);
+        return hostSystem.allocateHosts(cluster, Capacity.from(new ClusterResources(count, groups, resources.orElse(NodeResources.unspecified)),
+                                                               required, canFail),
+                                        logger);
     }
 
     private static Optional<NodeResources> getResources(ModelElement nodesElement) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 764b42c0ea2..31c8724d634 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements
                         .slobrokId(serviceSlobrokId())).
                 filedistributor(filedistributorConfig());
         if (clusterName != null) {
-            builder.discriminator(clusterName+"."+name);
+            builder.discriminator(clusterName + "." + name);
         } else {
             builder.discriminator(name);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index cd292da6fa1..06707786136 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -23,9 +23,11 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
@@ -665,11 +667,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                                                          .vespaVersion(deployState.getWantedNodeVespaVersion())
                                                          .dockerImageRepo(deployState.getWantedDockerImageRepo())
                                                          .build();
-                    Capacity capacity = Capacity.fromCount(1,
-                                                           Optional.empty(),
-                                                           false,
-                                                           ! deployState.getProperties().isBootstrap());
-                    HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, 1, log).keySet().iterator().next();
+                    Capacity capacity = Capacity.from(new ClusterResources(1, 1, NodeResources.unspecified),
+                                                      false,
+                                                      ! deployState.getProperties().isBootstrap());
+                    HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, log).keySet().iterator().next();
                     return singleHostContainerCluster(cluster, host, context);
                 }
             }
@@ -680,11 +681,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                                                  .dockerImageRepo(deployState.getWantedDockerImageRepo())
                                                  .build();
             int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1;
-            Capacity capacity = Capacity.fromCount(nodeCount,
-                                                   Optional.empty(),
-                                                   false,
-                                                   !deployState.getProperties().isBootstrap());
-            var hosts = hostSystem.allocateHosts(clusterSpec, capacity, 1, log);
+            Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified),
+                                              false,
+                                              !deployState.getProperties().isBootstrap());
+            var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log);
             return createNodesFromHosts(log, hosts, cluster);
         }
         return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context);
@@ -714,7 +714,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                 .build();
         Map<HostResource, ClusterMembership> hosts = 
                 cluster.getRoot().hostSystem().allocateHosts(clusterSpec,
-                                                             Capacity.fromRequiredNodeType(type), 1, log);
+                                                             Capacity.fromRequiredNodeType(type), log);
         return createNodesFromHosts(context.getDeployLogger(), hosts, cluster);
     }
     
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index 4b7f727ff63..cf1ae637cf9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -109,7 +109,13 @@ public class VespaModelFactoryTest {
             }
 
             @Override
+            @Deprecated // TODO: Remove after April 2020
             public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+                return prepare(cluster, capacity.withGroups(groups), logger);
+            }
+
+            @Override
+            public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
                 return List.of(new HostSpec(hostName,
                                             List.of(),
                                             ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),