Merge pull request #13402 from vespa-engine/bratseth/lowestRealResourcesAllocating

Bratseth/lowest real resources allocating

10 files changed, 80 insertions, 54 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
index 3a3d3e4ec2e..e9e91910281 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
@@ -53,12 +53,12 @@ public class Application {
      * Returns an application with the given cluster having the min and max resource limits of the given cluster.
      * If the cluster has a target which is not inside the new limits, the target is removed.
      */
-    public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) {
+    public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) {
         Cluster cluster = clusters.get(id);
         if (cluster == null)
-            cluster = new Cluster(id, min, max, Optional.empty(), Optional.empty());
+            cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty());
         else
-            cluster = cluster.withLimits(min, max);
+            cluster = cluster.withConfiguration(exclusive, min, max);
         return with(cluster);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
index 847ec1290f6..3aae47a9088 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
@@ -19,16 +19,19 @@ import java.util.Optional;
 public class Cluster {
 
     private final ClusterSpec.Id id;
+    private final boolean exclusive;
     private final ClusterResources min, max;
     private final Optional<ClusterResources> suggested;
     private final Optional<ClusterResources> target;
 
     public Cluster(ClusterSpec.Id id,
+                   boolean exclusive,
                    ClusterResources minResources,
                    ClusterResources maxResources,
                    Optional<ClusterResources> suggestedResources,
                    Optional<ClusterResources> targetResources) {
         this.id = Objects.requireNonNull(id);
+        this.exclusive = exclusive;
         this.min = Objects.requireNonNull(minResources);
         this.max = Objects.requireNonNull(maxResources);
         this.suggested = Objects.requireNonNull(suggestedResources);
@@ -47,6 +50,9 @@ public class Cluster {
     /** Returns the configured maximal resources in this cluster */
     public ClusterResources maxResources() { return max; }
 
+    /** Returns whether the nodes allocated to this cluster must be on host exclusively dedicated to this application */
+    public boolean exclusive() { return exclusive; }
+
     /**
      * Returns the computed resources (between min and max, inclusive) this cluster should
      * have allocated at the moment (whether or not it actually has it),
@@ -60,16 +66,16 @@ public class Cluster {
      */
     public Optional<ClusterResources> suggestedResources() { return suggested; }
 
-    public Cluster withLimits(ClusterResources min, ClusterResources max) {
-        return new Cluster(id, min, max, suggested, target);
+    public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) {
+        return new Cluster(id, exclusive, min, max, suggested, target);
     }
 
     public Cluster withSuggested(Optional<ClusterResources> suggested) {
-        return new Cluster(id, min, max, suggested, target);
+        return new Cluster(id, exclusive, min, max, suggested, target);
     }
 
     public Cluster withTarget(Optional<ClusterResources> target) {
-        return new Cluster(id, min, max, suggested, target);
+        return new Cluster(id, exclusive, min, max, suggested, target);
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
index 1637d99a07a..b5a5d1e06b4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
@@ -33,19 +33,19 @@ public class AllocatableClusterResources {
 
     /** Fake allocatable resources from requested capacity */
     public AllocatableClusterResources(ClusterResources requested, ClusterSpec.Type clusterType) {
-        this.advertisedResources = requested.nodeResources();
-        this.realResources = requested.nodeResources(); // we don't know
         this.nodes = requested.nodes();
         this.groups = requested.groups();
+        this.realResources = requested.nodeResources(); // we don't know
+        this.advertisedResources = requested.nodeResources();
         this.clusterType = clusterType;
         this.fulfilment = 1;
     }
 
     public AllocatableClusterResources(List<Node> nodes, NodeRepository nodeRepository) {
-        this.advertisedResources = nodes.get(0).flavor().resources();
-        this.realResources = nodeRepository.resourcesCalculator().realResourcesOf(nodes.get(0), nodeRepository);
         this.nodes = nodes.size();
         this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count();
+        this.realResources = nodeRepository.resourcesCalculator().realResourcesOf(nodes.get(0), nodeRepository);
+        this.advertisedResources = nodes.get(0).flavor().resources();
         this.clusterType = nodes.get(0).allocation().get().membership().cluster().type();
         this.fulfilment = 1;
     }
@@ -54,23 +54,10 @@ public class AllocatableClusterResources {
                                        NodeResources advertisedResources,
                                        NodeResources idealResources,
                                        ClusterSpec.Type clusterType) {
-        this.realResources = realResources.nodeResources();
-        this.advertisedResources = advertisedResources;
         this.nodes = realResources.nodes();
         this.groups = realResources.groups();
-        this.clusterType = clusterType;
-        this.fulfilment = fulfilment(realResources.nodeResources(), idealResources);
-    }
-
-    public AllocatableClusterResources(ClusterResources realResources,
-                                       Flavor flavor,
-                                       NodeResources idealResources,
-                                       ClusterSpec.Type clusterType,
-                                       HostResourcesCalculator calculator) {
         this.realResources = realResources.nodeResources();
-        this.advertisedResources = calculator.advertisedResourcesOf(flavor);
-        this.nodes = realResources.nodes();
-        this.groups = realResources.groups();
+        this.advertisedResources = advertisedResources;
         this.clusterType = clusterType;
         this.fulfilment = fulfilment(realResources.nodeResources(), idealResources);
     }
@@ -127,7 +114,7 @@ public class AllocatableClusterResources {
     public String toString() {
         return nodes + " nodes " +
                ( groups > 1 ? "(in " + groups + " groups) " : "" ) +
-               "with " + realResources() +
+               "with " + advertisedResources() +
                " at cost $" + cost() +
                (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : "");
     }
@@ -135,31 +122,39 @@ public class AllocatableClusterResources {
     /**
      * Returns the best matching allocatable node resources given ideal node resources,
      * or empty if none available within the limits.
+     *
+     * @param resources the real resources that should ideally be allocated
+     * @param exclusive whether resources should be allocated on entire hosts
+     *        (in which case the allocated resources will be all the real resources of the host
+     *         and limits are required to encompass the full resources of candidate host flavors)
      */
     public static Optional<AllocatableClusterResources> from(ClusterResources resources,
+                                                             boolean exclusive,
                                                              ClusterSpec.Type clusterType,
                                                              Limits limits,
                                                              NodeRepository nodeRepository) {
         NodeResources cappedNodeResources = limits.cap(resources.nodeResources());
         cappedNodeResources = new NodeResourceLimits(nodeRepository).enlargeToLegal(cappedNodeResources, clusterType);
 
-        if (nodeRepository.zone().getCloud().allowHostSharing()) {
-            // return the requested resources, or empty if they cannot fit on existing hosts
+        if ( !exclusive && nodeRepository.zone().getCloud().allowHostSharing()) { // Check if any flavor can fit these hosts
+            // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources
+            NodeResources realResources = cappedNodeResources;
+            NodeResources advertisedResources = nodeRepository.resourcesCalculator().realToRequest(realResources);
             for (Flavor flavor : nodeRepository.flavors().getFlavors()) {
-                if (flavor.resources().satisfies(cappedNodeResources))
-                    return Optional.of(new AllocatableClusterResources(resources.with(cappedNodeResources),
-                                                                       cappedNodeResources,
+                if (flavor.resources().satisfies(advertisedResources))
+                    return Optional.of(new AllocatableClusterResources(resources.with(realResources),
+                                                                       advertisedResources,
                                                                        resources.nodeResources(),
                                                                        clusterType));
             }
             return Optional.empty();
         }
-        else {
-            // return the cheapest flavor satisfying the target resources, if any
+        else { // Return the cheapest flavor satisfying the requested resources, if any
             Optional<AllocatableClusterResources> best = Optional.empty();
             for (Flavor flavor : nodeRepository.flavors().getFlavors()) {
+                // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources
                 NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor);
-                NodeResources realResources = flavor.resources();
+                NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources);
 
                 // Adjust where we don't need exact match to the flavor
                 if (flavor.resources().storageType() == NodeResources.StorageType.remote) {
@@ -172,7 +167,6 @@ public class AllocatableClusterResources {
                 }
 
                 if ( ! between(limits.min().nodeResources(), limits.max().nodeResources(), advertisedResources)) continue;
-
                 var candidate = new AllocatableClusterResources(resources.with(realResources),
                                                                 advertisedResources,
                                                                 resources.nodeResources(),
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
index 475f2feaebd..d2589b15421 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
@@ -38,7 +38,8 @@ public class AllocationOptimizer {
      */
     public Optional<AllocatableClusterResources> findBestAllocation(ResourceTarget target,
                                                                     AllocatableClusterResources current,
-                                                                    Limits limits) {
+                                                                    Limits limits,
+                                                                    boolean exclusive) {
         if (limits.isEmpty())
             limits = Limits.of(new ClusterResources(minimumNodes,    1, NodeResources.unspecified()),
                                new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()));
@@ -57,7 +58,7 @@ public class AllocationOptimizer {
                                                              groups,
                                                              nodeResourcesWith(nodesAdjustedForRedundancy, groupsAdjustedForRedundancy, limits, current, target));
 
-                var allocatableResources = AllocatableClusterResources.from(next, current.clusterType(), limits, nodeRepository);
+                var allocatableResources = AllocatableClusterResources.from(next, exclusive, current.clusterType(), limits, nodeRepository);
                 if (allocatableResources.isEmpty()) continue;
                 if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get()))
                     bestAllocation = allocatableResources;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index 8930bf34f4a..27731159e9f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -53,7 +53,7 @@ public class Autoscaler {
      * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
      */
     public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) {
-        return autoscale(clusterNodes, Limits.empty())
+        return autoscale(clusterNodes, Limits.empty(), cluster.exclusive())
                        .map(AllocatableClusterResources::toAdvertisedClusterResources);
 
     }
@@ -66,11 +66,11 @@ public class Autoscaler {
      */
     public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) {
         if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut
-        return autoscale(clusterNodes, Limits.of(cluster))
+        return autoscale(clusterNodes, Limits.of(cluster), cluster.exclusive())
                        .map(AllocatableClusterResources::toAdvertisedClusterResources);
     }
 
-    private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits) {
+    private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits, boolean exclusive) {
         if (unstable(clusterNodes)) return Optional.empty();
 
         ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type();
@@ -82,7 +82,7 @@ public class Autoscaler {
         var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation);
 
         Optional<AllocatableClusterResources> bestAllocation =
-                allocationOptimizer.findBestAllocation(target, currentAllocation, limits);
+                allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive);
         if (bestAllocation.isEmpty()) return Optional.empty();
         if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty();
         return bestAllocation;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
index 9359d8251f7..f7c2a51436a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
@@ -11,7 +11,7 @@ public class ResourceTarget {
 
     private final boolean adjustForRedundancy;
 
-    /** The target resources per node, assuming the node assignment in current */
+    /** The target resources per node, assuming the node assignment where this was decided */
     private final double cpu, memory, disk;
 
     private ResourceTarget(double cpu, double memory, double disk, boolean adjustForRedundancy) {
@@ -33,6 +33,13 @@ public class ResourceTarget {
     /** Returns the target disk per node, in terms of the current allocation */
     public double nodeDisk() { return disk; }
 
+    @Override
+    public String toString() {
+        return "target " +
+               (adjustForRedundancy ? "(with redundancy adjustment) " : "") +
+               "[vcpu " + cpu + ", memoryGb " + memory + ", diskGb " + disk + "]";
+    }
+
     private static double nodeUsage(Resource resource, double load, AllocatableClusterResources current) {
         return load * resource.valueFrom(current.realResources());
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
index b1453a9729a..3464e9dd881 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
@@ -35,6 +35,7 @@ public class ApplicationSerializer {
 
     private static final String idKey = "id";
     private static final String clustersKey = "clusters";
+    private static final String exclusiveKey = "exclusive";
     private static final String minResourcesKey = "min";
     private static final String maxResourcesKey = "max";
     private static final String suggestedResourcesKey = "suggested";
@@ -80,6 +81,7 @@ public class ApplicationSerializer {
     }
 
     private static void toSlime(Cluster cluster, Cursor clusterObject) {
+        clusterObject.setBool(exclusiveKey, cluster.exclusive());
         toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey));
         toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey));
         cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey)));
@@ -88,6 +90,7 @@ public class ApplicationSerializer {
 
     private static Cluster clusterFromSlime(String id, Inspector clusterObject) {
         return new Cluster(ClusterSpec.Id.from(id),
+                           clusterObject.field(exclusiveKey).asBool(),
                            clusterResourcesFromSlime(clusterObject.field(minResourcesKey)),
                            clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)),
                            optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)),
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
index da54d89b4e5..e4b1e0fcbc0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
@@ -34,14 +34,17 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider {
     private static class IdentityHostResourcesCalculator implements HostResourcesCalculator {
 
         @Override
-        public NodeResources realResourcesOf(Node node, NodeRepository repository) {
-            return node.flavor().resources();
-        }
+        public NodeResources realResourcesOf(Node node, NodeRepository repository) { return node.flavor().resources(); }
 
         @Override
-        public NodeResources advertisedResourcesOf(Flavor flavor) {
-            return flavor.resources();
-        }
+        public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); }
+
+        @Override
+        public NodeResources requestToReal(NodeResources resources) { return resources; }
+
+        @Override
+        public NodeResources realToRequest(NodeResources resources) { return resources; }
 
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
index 096e58e963e..a071fb25da2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
@@ -16,10 +16,22 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
  */
 public interface HostResourcesCalculator {
 
-    /** Nodes use advertised resources. This returns the real resources for the node. */
+    /** Returns the real resources available on a node */
     NodeResources realResourcesOf(Node node, NodeRepository nodeRepository);
 
-    /** Flavors use real resources. This returns the advertised resources of the flavor. */
+    /** Returns the advertised resources of a flavor */
     NodeResources advertisedResourcesOf(Flavor flavor);
 
+    /**
+     * Used with exclusive hosts:
+     * Returns the lowest possible real resources we'll get if requesting the given advertised resources
+     */
+    NodeResources requestToReal(NodeResources advertisedResources);
+
+    /**
+     * Used with shared hosts:
+     * Returns the advertised resources we need to request to be sure to get at least the given real resources.
+     */
+    NodeResources realToRequest(NodeResources realResources);
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index ccf08fdef48..433a69a063e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -144,7 +144,7 @@ public class NodeRepositoryProvisioner implements Provisioner {
     private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) {
         try (Mutex lock = nodeRepository.lock(applicationId)) {
             Application application = nodeRepository.applications().get(applicationId).orElse(new Application(applicationId));
-            application = application.withClusterLimits(clusterSpec.id(), requested.minResources(), requested.maxResources());
+            application = application.withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested.minResources(), requested.maxResources());
             nodeRepository.applications().put(application, lock);
             return application.clusters().get(clusterSpec.id()).targetResources()
                     .orElseGet(() -> currentResources(applicationId, clusterSpec, requested));
@@ -163,15 +163,15 @@ public class NodeRepositoryProvisioner implements Provisioner {
         AllocatableClusterResources currentResources =
                 nodes.isEmpty() ? new AllocatableClusterResources(requested.minResources(), clusterSpec.type()) // new deployment: Use min
                                 : new AllocatableClusterResources(nodes, nodeRepository);
-        return within(Limits.of(requested), currentResources);
+        return within(Limits.of(requested), clusterSpec.isExclusive(), currentResources);
     }
 
     /** Make the minimal adjustments needed to the current resources to stay within the limits */
-    private ClusterResources within(Limits limits, AllocatableClusterResources current) {
+    private ClusterResources within(Limits limits, boolean exclusive, AllocatableClusterResources current) {
         if (limits.isEmpty()) return current.toAdvertisedClusterResources();
         if (limits.min().equals(limits.max())) return limits.min();
 
-        return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits)
+        return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits, exclusive)
                                   .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits))
                                   .toAdvertisedClusterResources();
     }