Validate real resources available during allocation

7 files changed, 55 insertions, 36 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
index 0fa04032146..def3b18d14c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
@@ -138,7 +138,7 @@ public class AllocatableClusterResources {
                                                              Limits limits,
                                                              NodeRepository nodeRepository) {
         NodeResources cappedNodeResources = limits.cap(resources.nodeResources());
-        cappedNodeResources = new NodeResourceLimits(nodeRepository.zone()).enlargeToLegal(cappedNodeResources, clusterType);
+        cappedNodeResources = new NodeResourceLimits(nodeRepository).enlargeToLegal(cappedNodeResources, clusterType);
 
         if (nodeRepository.zone().getCloud().allowHostSharing()) {
             // return the requested resources, or empty if they cannot fit on existing hosts
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 7e3b6f5bd94..ee46d58ff9f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.util.Locale;
 
@@ -25,9 +26,9 @@ public class CapacityPolicies {
     /* Deployments must match 1-to-1 the advertised resources of a physical host */
     private final boolean isUsingAdvertisedResources;
 
-    public CapacityPolicies(Zone zone) {
-        this.zone = zone;
-        this.nodeResourceLimits = new NodeResourceLimits(zone);
+    public CapacityPolicies(NodeRepository nodeRepository) {
+        this.zone = nodeRepository.zone();
+        this.nodeResourceLimits = new NodeResourceLimits(nodeRepository);
         this.isUsingAdvertisedResources = zone.getCloud().dynamicProvisioning();
     }
 
@@ -48,7 +49,7 @@ public class CapacityPolicies {
     public NodeResources decideNodeResources(NodeResources target, Capacity capacity, ClusterSpec cluster) {
         if (target.isUnspecified())
             target = defaultNodeResources(cluster.type());
-        nodeResourceLimits.ensureSufficient(target, cluster);
+        nodeResourceLimits.ensureWithinAdvertisedLimits(target, cluster);
 
         if (capacity.isRequired()) return target;
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
index 78f851fbc9e..7a87dabeed2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java
@@ -47,15 +47,15 @@ public class FlavorConfigBuilder {
         FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder();
         for (String flavorName : flavors) {
             if (flavorName.equals("docker"))
-                flavorConfigBuilder.addFlavor(flavorName, 1., 3., 2., 1.5, Flavor.Type.DOCKER_CONTAINER);
+                flavorConfigBuilder.addFlavor(flavorName, 1., 30., 2., 1.5, Flavor.Type.DOCKER_CONTAINER);
             else if (flavorName.equals("docker2"))
-                flavorConfigBuilder.addFlavor(flavorName, 2.,  4., 4., 0.5, Flavor.Type.DOCKER_CONTAINER);
+                flavorConfigBuilder.addFlavor(flavorName, 2.,  40., 4., 0.5, Flavor.Type.DOCKER_CONTAINER);
             else if (flavorName.equals("host"))
-                flavorConfigBuilder.addFlavor(flavorName, 7., 10., 12., 5, Flavor.Type.BARE_METAL);
+                flavorConfigBuilder.addFlavor(flavorName, 7., 100., 12., 5, Flavor.Type.BARE_METAL);
             else if (flavorName.equals("devhost"))
-                flavorConfigBuilder.addFlavor(flavorName, 4.,  8., 10, 10, Flavor.Type.BARE_METAL);
+                flavorConfigBuilder.addFlavor(flavorName, 4.,  80., 10, 10, Flavor.Type.BARE_METAL);
             else
-                flavorConfigBuilder.addFlavor(flavorName, 1.,  3., 2., 3, Flavor.Type.BARE_METAL);
+                flavorConfigBuilder.addFlavor(flavorName, 1.,  30., 2., 3, Flavor.Type.BARE_METAL);
         }
         return new NodeFlavors(flavorConfigBuilder.build());
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index f8774073ce8..698f40c2b24 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -83,8 +83,7 @@ public class GroupPreparer {
                 prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo);
                 // Allocate from the prioritized list
                 NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes,
-                                                               highestIndex,  nodeRepository.flavors(),
-                                                               nodeRepository.zone(), nodeRepository.clock());
+                                                               highestIndex,  nodeRepository);
                 allocation.offer(prioritizer.prioritize());
 
                 if (dynamicProvisioningEnabled) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index 71e8259665b..8e2d3c64390 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -14,6 +14,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.lang.MutableInteger;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 
@@ -74,20 +75,18 @@ class NodeAllocation {
     /** The next membership index to assign to a new node */
     private final MutableInteger highestIndex;
 
-    private final NodeFlavors flavors;
-    private final Zone zone;
-    private final Clock clock;
+    private final NodeRepository nodeRepository;
+    private final NodeResourceLimits nodeResourceLimits;
 
     NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
-                   MutableInteger highestIndex, NodeFlavors flavors, Zone zone, Clock clock) {
+                   MutableInteger highestIndex, NodeRepository nodeRepository) {
         this.allNodes = allNodes;
         this.application = application;
         this.cluster = cluster;
         this.requestedNodes = requestedNodes;
         this.highestIndex = highestIndex;
-        this.flavors = flavors;
-        this.zone = zone;
-        this.clock = clock;
+        this.nodeRepository = nodeRepository;
+        nodeResourceLimits = new NodeResourceLimits(nodeRepository);
     }
 
     /**
@@ -104,6 +103,8 @@ class NodeAllocation {
         List<Node> accepted = new ArrayList<>();
         for (PrioritizableNode node : nodesPrioritized) {
             Node offered = node.node;
+            if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) continue;
+
             if (offered.allocation().isPresent()) {
                 ClusterMembership membership = offered.allocation().get().membership();
                 if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application
@@ -145,7 +146,7 @@ class NodeAllocation {
                 node.node = offered.allocate(application,
                                              ClusterMembership.from(cluster, highestIndex.add(1)),
                                              requestedNodes.resources().orElse(node.node.flavor().resources()),
-                                             clock.instant());
+                                             nodeRepository.clock().instant());
                 accepted.add(acceptNode(node, false, false));
             }
         }
@@ -159,7 +160,9 @@ class NodeAllocation {
     }
 
     private boolean checkForClashingParentHost() {
-        return zone.system() == SystemName.main && zone.environment().isProduction() &&  ! application.instance().isTester();
+        return nodeRepository.zone().system() == SystemName.main &&
+               nodeRepository.zone().environment().isProduction() &&
+               ! application.instance().isTester();
     }
 
     private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<PrioritizableNode> accepted, Node offered) {
@@ -229,7 +232,7 @@ class NodeAllocation {
     }
 
     private boolean hasCompatibleFlavor(PrioritizableNode node) {
-        return requestedNodes.isCompatible(node.node.flavor(), flavors) || node.isResizable;
+        return requestedNodes.isCompatible(node.node.flavor(), nodeRepository.flavors()) || node.isResizable;
     }
 
     private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) {
@@ -255,7 +258,7 @@ class NodeAllocation {
         } else {
             ++wasRetiredJustNow;
             // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host
-            node = node.retire(clock.instant());
+            node = node.retire(nodeRepository.clock().instant());
         }
         if ( ! node.allocation().get().membership().cluster().equals(cluster)) {
             // group may be different
@@ -332,7 +335,7 @@ class NodeAllocation {
         if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0, prefer to retire higher indexes to minimize redistribution
             for (PrioritizableNode node : byDecreasingIndex(nodes)) {
                 if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) {
-                    node.node = node.node.retire(Agent.application, clock.instant());
+                    node.node = node.node.retire(Agent.application, nodeRepository.clock().instant());
                     surplusNodes.add(node.node); // offer this node to other groups
                     if (--deltaRetiredCount == 0) break;
                 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index df35cabab36..28002463586 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -66,7 +66,7 @@ public class NodeRepositoryProvisioner implements Provisioner {
                                      ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) {
         this.nodeRepository = nodeRepository;
         this.allocationOptimizer = new AllocationOptimizer(nodeRepository);
-        this.capacityPolicies = new CapacityPolicies(zone);
+        this.capacityPolicies = new CapacityPolicies(nodeRepository);
         this.zone = zone;
         this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService));
         this.preparer = new Preparer(nodeRepository,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
index eb43cb97144..b71cc396c0e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
@@ -5,6 +5,8 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.util.Locale;
 
@@ -15,28 +17,42 @@ import java.util.Locale;
  */
 public class NodeResourceLimits {
 
-    private final Zone zone;
+    private final NodeRepository nodeRepository;
 
-    public NodeResourceLimits(Zone zone) {
-        this.zone = zone;
+    public NodeResourceLimits(NodeRepository nodeRepository) {
+        this.nodeRepository = nodeRepository;
     }
 
-    public void ensureSufficient(NodeResources resources, ClusterSpec cluster) {
-        double minMemoryGb = minMemoryGb(cluster.type());
-        if (resources.memoryGb() >= minMemoryGb) return;
+    /** Validates the resources applications ask for */
+    public void ensureWithinAdvertisedLimits(NodeResources advertisedResources, ClusterSpec cluster) {
+        double minMemoryGb = minAdvertisedMemoryGb(cluster.type());
+        if (advertisedResources.memoryGb() >= minMemoryGb) return;
         throw new IllegalArgumentException(String.format(Locale.ENGLISH,
                                                          "Must specify at least %.2f Gb of memory for %s cluster '%s', was: %.2f Gb",
-                                                         minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb()));
+                                                         minMemoryGb, cluster.type().name(), cluster.id().value(), advertisedResources.memoryGb()));
     }
 
-    public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) {
-        return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb()));
+    /** Returns whether the real resources we'll end up with on a given tenant node are within limits */
+    public boolean isWithinRealLimits(Node candidateTenantNode, ClusterSpec cluster) {
+        NodeResources realResources = nodeRepository.resourcesCalculator().realResourcesOf(candidateTenantNode, nodeRepository);
+
+        if (realResources.memoryGb() < minRealMemoryGb(cluster.type())) return false;
+
+        return true;
+    }
+
+    public NodeResources enlargeToLegal(NodeResources advertisedResources, ClusterSpec.Type clusterType) {
+        return advertisedResources.withMemoryGb(Math.max(minAdvertisedMemoryGb(clusterType), advertisedResources.memoryGb()));
     }
 
-    private int minMemoryGb(ClusterSpec.Type clusterType) {
-        if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system
+    private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) {
+        if (nodeRepository.zone().system() == SystemName.dev) return 1; // Allow small containers in dev system
         if (clusterType == ClusterSpec.Type.admin) return 2;
         return 4;
     }
 
+    private double minRealMemoryGb(ClusterSpec.Type clusterType) {
+        return minAdvertisedMemoryGb(clusterType) - 0.7;
+    }
+
 }