Allocate all groups in one go

With many groups and dynamic allocation allocating group by
group is too slow.

22 files changed, 346 insertions, 215 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
index 9e8388b6442..36f77a179ca 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
@@ -44,7 +44,7 @@ public class ClusterMembership {
 
         this.cluster = ClusterSpec.specification(ClusterSpec.Type.valueOf(components[0]),
                                                  ClusterSpec.Id.from(components[1]))
-                                  .group(ClusterSpec.Group.from(Integer.parseInt(components[2])))
+                                  .group(components[2].isEmpty() ? null : ClusterSpec.Group.from(Integer.parseInt(components[2])))
                                   .vespaVersion(vespaVersion)
                                   .exclusive(exclusive)
                                   .combinedId(combinedId.map(ClusterSpec.Id::from))
@@ -67,7 +67,7 @@ public class ClusterMembership {
     protected String toStringValue() {
         return cluster.type().name() +
                "/" + cluster.id().value() +
-               (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") +
+               (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "/") +
                "/" + index +
                ( cluster.isExclusive() ? "/exclusive" : "") +
                ( retired ? "/retired" : "") +
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
index ccc24e60edf..4a3045c9cdd 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
@@ -102,19 +102,18 @@ public final class ClusterSpec {
 
     /** Creates a ClusterSpec when requesting a cluster */
     public static Builder request(Type type, Id id) {
-        return new Builder(type, id, false);
+        return new Builder(type, id);
     }
 
     /** Creates a ClusterSpec for an existing cluster, group id and Vespa version needs to be set */
     public static Builder specification(Type type, Id id) {
-        return new Builder(type, id, true);
+        return new Builder(type, id);
     }
 
     public static class Builder {
 
         private final Type type;
         private final Id id;
-        private final boolean specification;
 
         private Optional<Group> groupId = Optional.empty();
         private Optional<DockerImage> dockerImageRepo = Optional.empty();
@@ -124,19 +123,13 @@ public final class ClusterSpec {
         private ZoneEndpoint zoneEndpoint = ZoneEndpoint.defaultEndpoint;
         private boolean stateful;
 
-        private Builder(Type type, Id id, boolean specification) {
+        private Builder(Type type, Id id) {
             this.type = type;
             this.id = id;
-            this.specification = specification;
             this.stateful = type.isContent(); // Default to true for content clusters
         }
 
         public ClusterSpec build() {
-            if (specification) {
-                if (groupId.isEmpty()) throw new IllegalArgumentException("groupId is required to be set when creating a ClusterSpec with specification()");
-                if (vespaVersion == null) throw new IllegalArgumentException("vespaVersion is required to be set when creating a ClusterSpec with specification()");
-            } else
-                if (groupId.isPresent()) throw new IllegalArgumentException("groupId is not allowed to be set when creating a ClusterSpec with request()");
             return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful);
         }
 
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java
index 507d95c1d7b..64d028db7b0 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java
@@ -16,6 +16,11 @@ public class NodeAllocationException extends RuntimeException {
         this.retryable = retryable;
     }
 
+    public NodeAllocationException(String message, Throwable cause, boolean retryable) {
+        super(message, cause);
+        this.retryable = retryable;
+    }
+
     public boolean retryable() {
         return retryable;
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java
index 9bc18533ddf..e760e36f90b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java
@@ -24,7 +24,7 @@ public final class LockedNodeList extends NodeList {
         this.lock = Objects.requireNonNull(lock, "lock must be non-null");
     }
 
-    /** Returns a new LockedNodeList with the for the same lock. */
+    /** Returns a new LockedNodeList with the same lock. */
     public LockedNodeList childList(List<Node> nodes) {
         return new LockedNodeList(nodes, lock);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
index 2a0b4f02b20..331759127e4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
@@ -268,11 +268,9 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
                 // build() requires a version, even though it is not (should not be) used
                 .vespaVersion(Vtag.currentVersion)
                 .build();
-        NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), nodeResources, false, true,
+        NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), 1, nodeResources, false, true,
                                           nodeRepository().zone().cloud().account(), Duration.ZERO);
-        int wantedGroups = 1;
-
-        NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec, wantedGroups,
+        NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec,
                 true, nodeRepository().nameResolver(), nodeRepository().nodes(), nodeRepository().resourcesCalculator(),
                 nodeRepository().spareCount(), nodeSpec.cloudAccount().isExclave(nodeRepository().zone()));
         List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
index c25f33bc8c2..462f43ee7af 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.applications.Application;
 import com.yahoo.vespa.hosted.provision.applications.ScalingEvent;
-import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 
@@ -71,8 +70,9 @@ class Activator {
         NodeList allNodes = nodeRepository.nodes().list();
         NodeList applicationNodes = allNodes.owner(application);
 
-        NodeList reserved = updatePortsFrom(hosts, applicationNodes.state(Node.State.reserved)
-                                                                   .matching(node -> hostnames.contains(node.hostname())));
+        NodeList reserved = applicationNodes.state(Node.State.reserved).matching(node -> hostnames.contains(node.hostname()));
+        reserved = updatePortsFrom(hosts, reserved);
+        reserved = updateGroupFrom(hosts, reserved);
         nodeRepository.nodes().reserve(reserved.asList()); // Re-reserve nodes to avoid reservation expiry
 
         NodeList oldActive = applicationNodes.state(Node.State.active); // All nodes active now
@@ -240,6 +240,18 @@ class Activator {
         return NodeList.copyOf(updated);
     }
 
+    /**
+     * Reserved nodes are stored before they are assigned a group.
+     */
+    private NodeList updateGroupFrom(Collection<HostSpec> hosts, NodeList nodes) {
+        List<Node> updated = new ArrayList<>();
+        for (Node node : nodes) {
+            var membership = getHost(node.hostname(), hosts).membership().get();
+            updated.add(node.with(node.allocation().get().with(membership)));
+        }
+        return NodeList.copyOf(updated);
+    }
+
     private HostSpec getHost(String hostname, Collection<HostSpec> fromHosts) {
         for (HostSpec host : fromHosts)
             if (host.hostname().equals(hostname))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java
new file mode 100644
index 00000000000..44f371be293
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java
@@ -0,0 +1,163 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+
+import java.time.Clock;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Knows how to assign a group index to a number of nodes (some of which have an index already),
+ * such that the nodes are placed in the desired groups with minimal group movement.
+ *
+ * @author bratseth
+ */
+class GroupIndices {
+
+    private final NodeSpec requested;
+    private final NodeList allNodes;
+    private final Clock clock;
+
+    GroupIndices(NodeSpec requested, NodeList allNodes, Clock clock) {
+        if (requested.groups() > 1 && requested.count().isEmpty())
+            throw new IllegalArgumentException("Unlimited nodes cannot be grouped");
+        this.requested = requested;
+        this.allNodes = allNodes;
+        this.clock = clock;
+    }
+
+    Collection<NodeCandidate> assignTo(Collection<NodeCandidate> nodes) {
+        int[] countInGroup = countInEachGroup(nodes);
+        nodes = byUnretiringPriority(nodes).stream().map(node -> unretireNodeInExpandedGroup(node, countInGroup)).toList();
+        nodes = nodes.stream().map(node -> assignGroupToNewNode(node, countInGroup)).toList();
+        nodes = byUnretiringPriority(nodes).stream().map(node -> moveNodeInSurplusGroup(node, countInGroup)).toList();
+        nodes = byRetiringPriority(nodes).stream().map(node -> retireSurplusNodeInGroup(node, countInGroup)).toList();
+        nodes = nodes.stream().filter(node -> ! shouldRemove(node)).toList();
+        return nodes;
+    }
+
+    /** Prefer to retire nodes we want the least */
+    private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) {
+        return candidates.stream().sorted(Comparator.reverseOrder()).toList();
+    }
+
+    /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */
+    private List<NodeCandidate> byUnretiringPriority(Collection<NodeCandidate> candidates) {
+        return candidates.stream()
+                         .sorted(Comparator.comparing(NodeCandidate::wantToRetire)
+                                           .thenComparing(n -> n.allocation().get().membership().index()))
+                         .toList();
+    }
+
+    private int[] countInEachGroup(Collection<NodeCandidate> nodes) {
+        int[] countInGroup = new int[requested.groups()];
+        for (var node : nodes) {
+            if (node.allocation().get().membership().retired()) continue;
+            var currentGroup = node.allocation().get().membership().cluster().group();
+            if (currentGroup.isEmpty()) continue;
+            if (currentGroup.get().index() >= requested.groups()) continue;
+            countInGroup[currentGroup.get().index()]++;
+        }
+        return countInGroup;
+    }
+
+    /** Assign a group to new or to be reactivated nodes. */
+    private NodeCandidate assignGroupToNewNode(NodeCandidate node, int[] countInGroup) {
+        if (node.state() == Node.State.active && node.allocation().get().membership().retired()) return node;
+        if (node.state() == Node.State.active && node.allocation().get().membership().cluster().group().isPresent()) return node;
+        return inFirstGroupWithDeficiency(node, countInGroup);
+    }
+
+    private NodeCandidate moveNodeInSurplusGroup(NodeCandidate node, int[] countInGroup) {
+        var currentGroup = node.allocation().get().membership().cluster().group();
+        if (currentGroup.isEmpty()) return node; // Shouldn't happen
+        if (currentGroup.get().index() < requested.groups()) return node;
+        return inFirstGroupWithDeficiency(node, countInGroup);
+    }
+
+    private NodeCandidate retireSurplusNodeInGroup(NodeCandidate node, int[] countInGroup) {
+        if (node.allocation().get().membership().retired()) return node;
+        var currentGroup = node.allocation().get().membership().cluster().group();
+        if (currentGroup.isEmpty()) return node;
+        if (currentGroup.get().index() >= requested.groups()) return node;
+        if (requested.count().isEmpty()) return node; // Can't retire
+        if (countInGroup[currentGroup.get().index()] <= requested.count().get() / requested.groups()) return node;
+        countInGroup[currentGroup.get().index()]--;
+        return node.withNode(node.toNode().retire(Agent.application, clock.instant()));
+    }
+
+    /** Unretire nodes that are already in the correct group when the group is deficient. */
+    private NodeCandidate unretireNodeInExpandedGroup(NodeCandidate node, int[] countInGroup) {
+        if ( ! node.allocation().get().membership().retired()) return node;
+        var currentGroup = node.allocation().get().membership().cluster().group();
+        if (currentGroup.isEmpty()) return node;
+        if (currentGroup.get().index() >= requested.groups()) return node;
+        if (node.preferToRetire() || node.wantToRetire()) return node;
+        if (requested.count().isPresent() && countInGroup[currentGroup.get().index()] >= requested.count().get() / requested.groups()) return node;
+        node = unretire(node);
+        if (node.allocation().get().membership().retired()) return node;
+        countInGroup[currentGroup.get().index()]++;
+        return node;
+    }
+
+    private NodeCandidate inFirstGroupWithDeficiency(NodeCandidate node, int[] countInGroup) {
+        for (int group = 0; group < requested.groups(); group++) {
+            if (requested.count().isEmpty() || countInGroup[group] < requested.count().get() / requested.groups()) {
+                return inGroup(group, node, countInGroup);
+            }
+        }
+        return node;
+    }
+
+    private boolean shouldRemove(NodeCandidate node) {
+        var currentGroup = node.allocation().get().membership().cluster().group();
+        if (currentGroup.isEmpty()) return true; // new and not assigned an index: Not needed
+        return currentGroup.get().index() >= requested.groups();
+    }
+
+    private NodeCandidate inGroup(int group, NodeCandidate node, int[] countInGroup) {
+        node = unretire(node);
+        if (node.allocation().get().membership().retired()) return node;
+        var membership = node.allocation().get().membership();
+        var currentGroup = membership.cluster().group();
+        countInGroup[group]++;
+        if ( ! currentGroup.isEmpty() && currentGroup.get().index() < requested.groups())
+            countInGroup[membership.cluster().group().get().index()]--;
+        return node.withNode(node.toNode().with(node.allocation().get().with(membership.with(membership.cluster().with(Optional.of(ClusterSpec.Group.from(group)))))));
+    }
+
+    /** Attempt to unretire the given node if it is retired. */
+    private NodeCandidate unretire(NodeCandidate node) {
+        if (node.retiredNow()) return node;
+        if ( ! node.allocation().get().membership().retired()) return node;
+        if ( ! hasCompatibleResources(node) ) return node;
+        var parent = node.parentHostname().flatMap(hostname -> allNodes.node(hostname));
+        if (parent.isPresent() && (parent.get().status().wantToRetire() || parent.get().status().preferToRetire())) return node;
+        node = node.withNode();
+        if ( ! requested.isCompatible(node.resources()))
+            node = node.withNode(resize(node.toNode()));
+        return node.withNode(node.toNode().unretire());
+    }
+
+    private Node resize(Node node) {
+        NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources();
+        return node.with(new Flavor(requested.resources().get()
+                                             .with(hostResources.diskSpeed())
+                                             .with(hostResources.storageType())
+                                             .with(hostResources.architecture())),
+                         Agent.application, clock.instant());
+    }
+
+    private boolean hasCompatibleResources(NodeCandidate candidate) {
+        return requested.isCompatible(candidate.resources()) || candidate.isResizable;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 0c4838abe4d..3c8f1b070e7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -61,14 +61,14 @@ public class GroupPreparer {
     // but it may not change the set of active nodes, as the active nodes must stay in sync with the
     // active config model which is changed on activate
     public PrepareResult prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
-                                 List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups,
+                                 List<Node> surplusActiveNodes, NodeIndices indices,
                                  LockedNodeList allNodes) {
         log.log(Level.FINE, () -> "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " +
                                   requestedNodes.resources().orElse(NodeResources.unspecified()));
         // Try preparing in memory without global unallocated lock. Most of the time there should be no changes,
         // and we can return nodes previously allocated.
         NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes,
-                                                           indices::probeNext, wantedGroups, allNodes);
+                                                           indices::probeNext, allNodes);
         if (probeAllocation.fulfilledAndNoChanges()) {
             List<Node> acceptedNodes = probeAllocation.finalNodes();
             surplusActiveNodes.removeAll(acceptedNodes);
@@ -77,7 +77,7 @@ public class GroupPreparer {
         } else {
             // There were some changes, so re-do the allocation with locks
             indices.resetProbe();
-            List<Node> prepared = prepareWithLocks(application, cluster, requestedNodes, surplusActiveNodes, indices, wantedGroups);
+            List<Node> prepared = prepareWithLocks(application, cluster, requestedNodes, surplusActiveNodes, indices);
             return new PrepareResult(prepared, createUnlockedNodeList());
         }
     }
@@ -87,12 +87,12 @@ public class GroupPreparer {
 
     /// Note that this will write to the node repo.
     private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
-                                        List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups) {
+                                        List<Node> surplusActiveNodes, NodeIndices indices) {
         try (Mutex lock = nodeRepository.applications().lock(application);
              Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) {
             LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock);
             NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes,
-                                                          indices::next, wantedGroups, allNodes);
+                                                          indices::next, allNodes);
             NodeType hostType = allocation.nodeType().hostType();
             if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) {
                 HostSharing sharing = hostSharing(cluster, hostType);
@@ -134,15 +134,13 @@ public class GroupPreparer {
                 // Non-dynamically provisioned zone with a deficit because we just now retired some nodes.
                 // Try again, but without retiring
                 indices.resetProbe();
-                List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), surplusActiveNodes, indices, wantedGroups);
+                List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), surplusActiveNodes, indices);
                 log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity");
                 return accepted;
             }
 
             if (! allocation.fulfilled() && requestedNodes.canFail())
-                throw new NodeAllocationException((cluster.group().isPresent() ? "Node allocation failure on " + cluster.group().get()
-                                                                               : "") + allocation.allocationFailureDetails(),
-                                                  true);
+                throw new NodeAllocationException(allocation.allocationFailureDetails(), true);
 
             // Carry out and return allocation
             nodeRepository.nodes().reserve(allocation.reservableNodes());
@@ -154,7 +152,7 @@ public class GroupPreparer {
     }
 
     private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
-                                             List<Node> surplusActiveNodes, Supplier<Integer> nextIndex, int wantedGroups,
+                                             List<Node> surplusActiveNodes, Supplier<Integer> nextIndex,
                                              LockedNodeList allNodes) {
 
         NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, nextIndex, nodeRepository);
@@ -162,7 +160,6 @@ public class GroupPreparer {
                                                           application,
                                                           cluster,
                                                           requestedNodes,
-                                                          wantedGroups,
                                                           nodeRepository.zone().cloud().dynamicProvisioning(),
                                                           nodeRepository.nameResolver(),
                                                           nodeRepository.nodes(),
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index a2e0e59e329..7715dce3a6a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -120,7 +119,6 @@ class NodeAllocation {
                 ClusterMembership membership = allocation.membership();
                 if ( ! allocation.owner().equals(application)) continue; // wrong application
                 if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type
-                if ((! candidate.isSurplus || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group, and we can't or have no reason to change it
                 if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal
                 if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing
                 if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure)
@@ -175,6 +173,7 @@ class NodeAllocation {
         if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest;
         if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity;
         if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor;
+        if (candidate.violatesSpares) return Retirement.violatesSpares;
         return Retirement.none;
     }
 
@@ -243,12 +242,10 @@ class NodeAllocation {
      */
     private boolean acceptIncompatible(NodeCandidate candidate) {
         if (candidate.state() != Node.State.active) return false;
-        if (! candidate.allocation().get().membership().cluster().group().equals(cluster.group())) return false;
         if (candidate.allocation().get().membership().retired()) return true; // don't second-guess if already retired
 
         if ( ! requestedNodes.considerRetiring()) // the node is active and we are not allowed to remove gracefully, so keep
             return true;
-
         return cluster.isStateful() ||
                (cluster.type() == ClusterSpec.Type.container && !hasCompatibleResources(candidate));
     }
@@ -259,7 +256,6 @@ class NodeAllocation {
 
     private Node acceptNode(NodeCandidate candidate, Retirement retirement, boolean resizeable) {
         Node node = candidate.toNode();
-
         if (node.allocation().isPresent()) // Record the currently requested resources
             node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources())));
 
@@ -268,10 +264,11 @@ class NodeAllocation {
             // We want to allocate new nodes rather than unretiring with resize, so count without those
             // for the purpose of deciding when to stop accepting nodes (saturation)
             if (node.allocation().isEmpty()
-                || ! ( requestedNodes.needsResize(node) &&
-                       (node.allocation().get().membership().retired() || ! requestedNodes.considerRetiring()))) {
+                || (canBeUsedInGroupWithDeficiency(node) &&
+                   ! ( requestedNodes.needsResize(node) && (node.allocation().get().membership().retired() || ! requestedNodes.considerRetiring())))) {
                 acceptedAndCompatible++;
             }
+
             if (hasCompatibleResources(candidate))
                 acceptedAndCompatibleOrResizable++;
 
@@ -289,15 +286,28 @@ class NodeAllocation {
             node = node.retire(nodeRepository.clock().instant());
         }
         if ( ! node.allocation().get().membership().cluster().equals(cluster)) {
-            // group may be different
-            node = setCluster(cluster, node);
+            // Cluster has the updated settings but do not set a group
+            node = setCluster(cluster.with(node.allocation().get().membership().cluster().group()), node);
         }
-        candidate = candidate.withNode(node);
+        candidate = candidate.withNode(node, retirement != Retirement.none && retirement != Retirement.alreadyRetired );
         indexes.add(node.allocation().get().membership().index());
         nodes.put(node.hostname(), candidate);
         return node;
     }
 
+    private boolean canBeUsedInGroupWithDeficiency(Node node) {
+        if (requestedNodes.count().isEmpty()) return true;
+        if (node.allocation().isEmpty()) return true;
+        var group = node.allocation().get().membership().cluster().group();
+        if (group.isEmpty()) return true;
+        long nodesInGroup = nodes.values().stream().filter(n -> groupOf(n).equals(group)).count();
+        return nodesInGroup < requestedNodes.count().get() / requestedNodes.groups();
+    }
+
+    private Optional<ClusterSpec.Group> groupOf(NodeCandidate candidate) {
+        return candidate.allocation().flatMap(a -> a.membership().cluster().group());
+    }
+
     private Node resize(Node node) {
         NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources();
         return node.with(new Flavor(requestedNodes.resources().get()
@@ -391,52 +401,19 @@ class NodeAllocation {
         return requestedNodes.type();
     }
 
-    /**
-     * Make the number of <i>non-retired</i> nodes in the list equal to the requested number
-     * of nodes, and retire the rest of the list. Only retire currently active nodes.
-     * Prefer to retire nodes of the wrong flavor.
-     * Make as few changes to the retired set as possible.
-     *
-     * @return the final list of nodes
-     */
     List<Node> finalNodes() {
-        int wantToRetireCount = (int) matching(NodeCandidate::wantToRetire).count();
-        int currentRetiredCount = (int) matching(node -> node.allocation().get().membership().retired()).count();
-        int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), wantToRetireCount, currentRetiredCount);
-
-        if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0
-            for (NodeCandidate candidate : byRetiringPriority(nodes.values())) {
-                if ( ! candidate.allocation().get().membership().retired() && candidate.state() == Node.State.active) {
-                    candidate = candidate.withNode();
-                    candidate = candidate.withNode(candidate.toNode().retire(Agent.application, nodeRepository.clock().instant()));
-                    nodes.put(candidate.toNode().hostname(), candidate);
-                    if (--deltaRetiredCount == 0) break;
-                }
-            }
-        }
-        else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0
-            for (NodeCandidate candidate : byUnretiringPriority(nodes.values())) {
-                if (candidate.allocation().get().membership().retired() && hasCompatibleResources(candidate) ) {
-                    candidate = candidate.withNode();
-                    if (candidate.isResizable)
-                        candidate = candidate.withNode(resize(candidate.toNode()));
-                    candidate = candidate.withNode(candidate.toNode().unretire());
-                    nodes.put(candidate.toNode().hostname(), candidate);
-                    if (++deltaRetiredCount == 0) break;
-                }
-            }
-        }
-        
+        // Set whether the node is exclusive
         for (NodeCandidate candidate : nodes.values()) {
-            // Set whether the node is exclusive
             candidate = candidate.withNode();
             Allocation allocation = candidate.allocation().get();
             candidate = candidate.withNode(candidate.toNode().with(allocation.with(allocation.membership()
-                                 .with(allocation.membership().cluster().exclusive(cluster.isExclusive())))));
+                                                                                             .with(allocation.membership().cluster().exclusive(cluster.isExclusive())))));
             nodes.put(candidate.toNode().hostname(), candidate);
         }
 
-        return nodes.values().stream().map(NodeCandidate::toNode).toList();
+        GroupIndices groupIndices = new GroupIndices(requestedNodes, allNodes, nodeRepository.clock());
+        Collection<NodeCandidate> finalNodes = groupIndices.assignTo(nodes.values());
+        return finalNodes.stream().map(NodeCandidate::toNode).toList();
     }
 
     List<Node> reservableNodes() {
@@ -461,19 +438,6 @@ class NodeAllocation {
         return allNodes.nodeType(nodeType()).size();
     }
 
-    /** Prefer to retire nodes we want the least */
-    private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) {
-        return candidates.stream().sorted(Comparator.reverseOrder()).toList();
-    }
-
-    /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */
-    private List<NodeCandidate> byUnretiringPriority(Collection<NodeCandidate> candidates) {
-        return candidates.stream()
-                         .sorted(Comparator.comparing(NodeCandidate::wantToRetire)
-                                           .thenComparing(n -> n.allocation().get().membership().index()))
-                         .toList();
-    }
-
     String allocationFailureDetails() {
         List<String> reasons = new ArrayList<>();
         if (rejectedDueToExclusivity > 0)
@@ -486,7 +450,7 @@ class NodeAllocation {
             reasons.add("insufficient real resources on hosts");
 
         if (reasons.isEmpty()) return "";
-        return ": Not enough suitable nodes available due to " + String.join(", ", reasons);
+        return "Not enough suitable nodes available due to " + String.join(", ", reasons);
     }
 
     private static Integer parseIndex(String hostname) {
@@ -510,6 +474,7 @@ class NodeAllocation {
         violatesExclusivity("node violates host exclusivity"),
         violatesHostFlavor("node violates host flavor"),
         violatesHostFlavorGeneration("node violates host flavor generation"),
+        violatesSpares("node is assigned to a host we want to use as a spare"),
         none("");
 
         private final String description;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java
index 8462e23fbfd..adc04c491e2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java
@@ -81,6 +81,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
 
     public abstract boolean preferToRetire();
 
+    /** Returns true if we have decided to retire this node as part of this deployment */
+    public boolean retiredNow() { return false; }
+
     public abstract boolean wantToFail();
 
     public abstract Flavor flavor();
@@ -217,7 +220,12 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
 
     /** Returns a copy of this with node set to given value */
     NodeCandidate withNode(Node node) {
-        return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable);
+        return withNode(node, retiredNow());
+    }
+
+    /** Returns a copy of this with node set to given value */
+    NodeCandidate withNode(Node node, boolean retiredNow) {
+        return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable);
     }
 
     /** Returns the switch priority, based on switch exclusivity, of this compared to other */
@@ -260,7 +268,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
                                             boolean isSurplus,
                                             boolean isNew,
                                             boolean isResizeable) {
-        return new ConcreteNodeCandidate(node, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable);
+        return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable);
     }
 
     public static NodeCandidate createNewChild(NodeResources resources,
@@ -274,26 +282,33 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
     }
 
     public static NodeCandidate createNewExclusiveChild(Node node, Node parent) {
-        return new ConcreteNodeCandidate(node, node.resources(), Optional.of(parent), false, true, false, true, false);
+        return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, true, false);
     }
 
     public static NodeCandidate createStandalone(Node node, boolean isSurplus, boolean isNew) {
-        return new ConcreteNodeCandidate(node, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false);
+        return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false);
     }
 
     /** A candidate backed by a node */
     static class ConcreteNodeCandidate extends NodeCandidate {
 
         private final Node node;
+        private final boolean retiredNow;
 
-        ConcreteNodeCandidate(Node node, NodeResources freeParentCapacity, Optional<Node> parent,
+        ConcreteNodeCandidate(Node node,
+                              boolean retiredNow,
+                              NodeResources freeParentCapacity, Optional<Node> parent,
                               boolean violatesSpares, boolean exclusiveSwitch,
                               boolean isSurplus, boolean isNew, boolean isResizeable) {
             super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizeable);
+            this.retiredNow = retiredNow;
             this.node = Objects.requireNonNull(node, "Node cannot be null");
         }
 
         @Override
+        public boolean retiredNow() { return retiredNow; }
+
+        @Override
         public NodeResources resources() { return node.resources(); }
 
         @Override
@@ -322,7 +337,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
 
         @Override
         public NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) {
-            return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at),
+            return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), retiredNow,
                                              freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable);
         }
 
@@ -332,7 +347,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
 
         @Override
         public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) {
-            return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch,
+            return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch,
                                              isSurplus, isNew, isResizable);
         }
 
@@ -439,7 +454,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat
                                      NodeType.tenant)
                             .cloudAccount(parent.get().cloudAccount())
                             .build();
-            return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable);
+            return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable);
 
         }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
index 4f21c8dcd50..9f00e5fdbba 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
@@ -46,7 +46,7 @@ public class NodePrioritizer {
     private final boolean enclave;
 
     public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec,
-                           int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes,
+                           boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes,
                            HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean enclave) {
         this.allNodes = allNodes;
         this.calculator = hostResourcesCalculator;
@@ -70,12 +70,9 @@ public class NodePrioritizer {
                         .stream())
                 .distinct()
                 .count();
-        this.topologyChange = currentGroups != wantedGroups;
+        this.topologyChange = currentGroups != requestedNodes.groups();
 
-        this.currentClusterSize = (int) nonRetiredNodesInCluster.state(Node.State.active).stream()
-                .map(node -> node.allocation().flatMap(alloc -> alloc.membership().cluster().group()))
-                .filter(clusterSpec.group()::equals)
-                .count();
+        this.currentClusterSize = (int) nonRetiredNodesInCluster.state(Node.State.active).stream().count();
 
         // In dynamically provisioned zones, we can always take spare hosts since we can provision new on-demand,
         // NodeCandidate::compareTo will ensure that they will not be used until there is no room elsewhere.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index ad91bdef478..c29c51ccbd5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -96,19 +96,17 @@ public class NodeRepositoryProvisioner implements Provisioner {
             validate(actual, target, cluster, application);
             logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger);
 
-            groups = target.groups();
             resources = getNodeResources(cluster, target.nodeResources(), application);
-            nodeSpec = NodeSpec.from(target.nodes(), resources, cluster.isExclusive(), actual.canFail(),
+            nodeSpec = NodeSpec.from(target.nodes(), target.groups(), resources, cluster.isExclusive(), actual.canFail(),
                                      requested.cloudAccount().orElse(nodeRepository.zone().cloud().account()),
                                      requested.clusterInfo().hostTTL());
         }
         else {
-            groups = 1; // type request with multiple groups is not supported
             cluster = cluster.withExclusivity(true);
             resources = getNodeResources(cluster, requested.minResources().nodeResources(), application);
             nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account());
         }
-        return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups),
+        return asSortedHosts(preparer.prepare(application, cluster, nodeSpec),
                              requireCompatibleResources(resources, cluster));
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
index f32928a9ec4..f4b2c4ceee0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java
@@ -35,21 +35,20 @@ public interface NodeSpec {
         return fulfilledDeficitCount(count) == 0;
     }
 
+    /** Returns the total number of nodes this is requesting, or empty if not specified */
+    Optional<Integer> count();
+
+    int groups();
+
     /** Returns whether this should throw an exception if the requested nodes are not fully available */
     boolean canFail();
 
     /** Returns whether we should retire nodes at all when fulfilling this spec */
     boolean considerRetiring();
 
-    /** Returns the ideal number of nodes that should be retired to fulfill this spec */
-    int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount);
-
     /** Returns number of additional nodes needed for this spec to be fulfilled given the current node count */
     int fulfilledDeficitCount(int count);
 
-    /** Returns a specification of a fraction of all the nodes of this. It is assumed the argument is a valid divisor. */
-    NodeSpec fraction(int divisor);
-
     /** Returns the resources requested by this or empty if none are explicitly requested */
     Optional<NodeResources> resources();
 
@@ -77,9 +76,9 @@ public interface NodeSpec {
         return false;
     }
 
-    static NodeSpec from(int nodeCount, NodeResources resources, boolean exclusive, boolean canFail,
+    static NodeSpec from(int nodeCount, int groupCount, NodeResources resources, boolean exclusive, boolean canFail,
                          CloudAccount cloudAccount, Duration hostTTL) {
-        return new CountNodeSpec(nodeCount, resources, exclusive, canFail, canFail, cloudAccount, hostTTL);
+        return new CountNodeSpec(nodeCount, groupCount, resources, exclusive, canFail, canFail, cloudAccount, hostTTL);
     }
 
     static NodeSpec from(NodeType type, CloudAccount cloudAccount) {
@@ -90,6 +89,7 @@ public interface NodeSpec {
     class CountNodeSpec implements NodeSpec {
 
         private final int count;
+        private final int groups;
         private final NodeResources requestedNodeResources;
         private final boolean exclusive;
         private final boolean canFail;
@@ -97,9 +97,10 @@ public interface NodeSpec {
         private final CloudAccount cloudAccount;
         private final Duration hostTTL;
 
-        private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail,
+        private CountNodeSpec(int count, int groups, NodeResources resources, boolean exclusive, boolean canFail,
                               boolean considerRetiring, CloudAccount cloudAccount, Duration hostTTL) {
             this.count = count;
+            this.groups = groups;
             this.requestedNodeResources = Objects.requireNonNull(resources, "Resources must be specified");
             this.exclusive = exclusive;
             this.canFail = canFail;
@@ -112,6 +113,12 @@ public interface NodeSpec {
         }
 
         @Override
+        public Optional<Integer> count() { return Optional.of(count); }
+
+        @Override
+        public int groups() { return groups; }
+
+        @Override
         public Optional<NodeResources> resources() {
             return Optional.of(requestedNodeResources);
         }
@@ -136,22 +143,12 @@ public interface NodeSpec {
         }
 
         @Override
-        public int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount) {
-            return acceptedCount - this.count - currentRetiredCount;
-        }
-
-        @Override
         public int fulfilledDeficitCount(int count) {
             return Math.max(this.count - count, 0);
         }
 
-        @Override
-        public NodeSpec fraction(int divisor) {
-            return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail, considerRetiring, cloudAccount, hostTTL);
-        }
-
         public NodeSpec withoutRetiring() {
-            return new CountNodeSpec(count, requestedNodeResources, exclusive, canFail, false, cloudAccount, hostTTL);
+            return new CountNodeSpec(count, groups, requestedNodeResources, exclusive, canFail, false, cloudAccount, hostTTL);
         }
 
         @Override
@@ -163,7 +160,6 @@ public interface NodeSpec {
         public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources,
                                  ClusterSpec.Type type, boolean hasTopologyChange, int currentClusterSize) {
             if (exclusive) return false; // exclusive resources must match the host
-
             // Never allow in-place resize when also changing topology or decreasing cluster size
             if (hasTopologyChange || count < currentClusterSize) return false;
 
@@ -192,7 +188,10 @@ public interface NodeSpec {
         public Duration hostTTL() { return hostTTL; }
 
         @Override
-        public String toString() { return "request for " + count + " nodes with " + requestedNodeResources; }
+        public String toString() {
+            return "request for " + count + " nodes" +
+                   ( groups > 1 ? " (in " + groups + " groups)" : "") +
+                   " with " + requestedNodeResources; }
 
     }
 
@@ -211,6 +210,12 @@ public interface NodeSpec {
         }
 
         @Override
+        public Optional<Integer> count() { return Optional.empty(); }
+
+        @Override
+        public int groups() { return 1; }
+
+        @Override
         public NodeType type() { return type; }
 
         @Override
@@ -226,20 +231,12 @@ public interface NodeSpec {
         public boolean considerRetiring() { return true; }
 
         @Override
-        public int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount) {
-            return wantToRetireCount - currentRetiredCount;
-        }
-
-        @Override
         public int fulfilledDeficitCount(int count) {
             // If no wanted count is specified for this node type, then any count fulfills the deficit
             return Math.max(0, WANTED_NODE_COUNT.getOrDefault(type, 0) - count);
         }
 
         @Override
-        public NodeSpec fraction(int divisor) { return this; }
-
-        @Override
         public Optional<NodeResources> resources() {
             return Optional.empty();
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
index 42b9e53dd8a..25efcabfe8e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.yolean.Exceptions;
 
+import java.time.Clock;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.ListIterator;
@@ -33,17 +33,15 @@ class Preparer {
     }
 
     /** Prepare all required resources for the given application and cluster */
-    public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) {
+    public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) {
         try {
-            var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups);
+            var nodes = prepareNodes(application, cluster, requestedNodes);
             prepareLoadBalancer(application, cluster, requestedNodes);
             return nodes;
         }
         catch (NodeAllocationException e) {
-            e.printStackTrace();
             throw new NodeAllocationException("Could not satisfy " + requestedNodes +
-                                              ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") +
-                                              " in " + application + " " + cluster + ": " + Exceptions.toMessageString(e),
+                                              " in " + application + " " + cluster, e,
                                               e.retryable());
         }
     }
@@ -56,34 +54,29 @@ class Preparer {
      // Note: This operation may make persisted changes to the set of reserved and inactive nodes,
      // but it may not change the set of active nodes, as the active nodes must stay in sync with the
      // active config model which is changed on activate
-    private List<Node> prepareNodes(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
-                                    int wantedGroups) {
+    private List<Node> prepareNodes(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) {
         LockedNodeList allNodes = groupPreparer.createUnlockedNodeList();
-        NodeList appNodes = allNodes.owner(application);
-        List<Node> surplusNodes = findNodesInRemovableGroups(appNodes, cluster, wantedGroups);
+        NodeList clusterNodes = allNodes.owner(application);
+        List<Node> surplusNodes = findNodesInRemovableGroups(clusterNodes, requestedNodes.groups());
 
-        List<Integer> usedIndices = appNodes.cluster(cluster.id()).mapToList(node -> node.allocation().get().membership().index());
+        List<Integer> usedIndices = clusterNodes.mapToList(node -> node.allocation().get().membership().index());
         NodeIndices indices = new NodeIndices(usedIndices);
         List<Node> acceptedNodes = new ArrayList<>();
 
-        for (int groupIndex = 0; groupIndex < wantedGroups; groupIndex++) {
-            ClusterSpec clusterGroup = cluster.with(Optional.of(ClusterSpec.Group.from(groupIndex)));
-            GroupPreparer.PrepareResult result = groupPreparer.prepare(application, clusterGroup,
-                                                                       requestedNodes.fraction(wantedGroups),
-                                                                       surplusNodes, indices, wantedGroups,
-                                                                       allNodes);
-            allNodes = result.allNodes(); // Might have changed
-            List<Node> accepted = result.prepared();
-            if (requestedNodes.rejectNonActiveParent()) {
-                NodeList activeHosts = allNodes.state(Node.State.active).parents().nodeType(requestedNodes.type().hostType());
-                accepted = accepted.stream()
-                                   .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent())
-                                   .toList();
-            }
-
-            replace(acceptedNodes, accepted);
+        GroupPreparer.PrepareResult result = groupPreparer.prepare(application, cluster,
+                                                                   requestedNodes,
+                                                                   surplusNodes, indices,
+                                                                   allNodes);
+        List<Node> accepted = result.prepared();
+        if (requestedNodes.rejectNonActiveParent()) {
+            NodeList activeHosts = result.allNodes().state(Node.State.active).parents().nodeType(requestedNodes.type().hostType());
+            accepted = accepted.stream()
+                               .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent())
+                               .toList();
         }
-        moveToActiveGroup(surplusNodes, wantedGroups, cluster.group());
+
+        replace(acceptedNodes, accepted);
+        moveToActiveGroup(surplusNodes, requestedNodes.groups(), cluster.group());
         acceptedNodes.removeAll(surplusNodes);
         return acceptedNodes;
     }
@@ -97,18 +90,16 @@ class Preparer {
      * Returns a list of the nodes which are
      * in groups with index number above or equal the group count
      */
-    private List<Node> findNodesInRemovableGroups(NodeList appNodes, ClusterSpec requestedCluster, int wantedGroups) {
+    private List<Node> findNodesInRemovableGroups(NodeList clusterNodes, int wantedGroups) {
         List<Node> surplusNodes = new ArrayList<>();
-        for (Node node : appNodes.state(Node.State.active)) {
+        for (Node node : clusterNodes.state(Node.State.active)) {
             ClusterSpec nodeCluster = node.allocation().get().membership().cluster();
-            if ( ! nodeCluster.id().equals(requestedCluster.id())) continue;
-            if ( ! nodeCluster.type().equals(requestedCluster.type())) continue;
             if (nodeCluster.group().get().index() >= wantedGroups)
                 surplusNodes.add(node);
         }
         return surplusNodes;
     }
-    
+
     /** Move nodes from unwanted groups to wanted groups to avoid lingering groups consisting of retired nodes */
     private void moveToActiveGroup(List<Node> surplusNodes, int wantedGroups, Optional<ClusterSpec.Group> targetGroup) {
         for (ListIterator<Node> i = surplusNodes.listIterator(); i.hasNext(); ) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java
index 85338bdb2b4..deaee02f992 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeListMicroBenchmarkTest.java
@@ -58,7 +58,6 @@ public class NodeListMicroBenchmarkTest {
             nodeList.childrenOf(nodes.get(indexes.get(i)));
         }
         Duration duration = Duration.between(start, Instant.now());
-        System.out.println("Calling NodeList.childrenOf took " + duration + " (" + duration.toNanos() / iterations / 1000 + " microseconds per invocation)");
     }
 
     private List<Node> createHosts() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
index 478b201d71b..8e9417008b9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
@@ -94,7 +94,6 @@ public class DynamicAllocationTest {
             hostsWithChildren.add(node.parentHostname().get());
         }
         assertEquals(4 - spareCount, hostsWithChildren.size());
-
     }
 
     /**
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
index c99728b714b..5539bb0cb6e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
@@ -334,7 +334,7 @@ public class DynamicProvisioningTest {
                 .flagSource(flagSource)
                 .build();
 
-        ApplicationId app = ProvisioningTester.applicationId();
+        ApplicationId app = ProvisioningTester.applicationId("a1");
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build();
         Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any)));
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
index 0bb6dc61d1b..54f0507831d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
@@ -57,7 +57,7 @@ public class InPlaceResizeProvisionTest {
     private final ProvisioningTester tester = new ProvisioningTester.Builder()
             .flagSource(flagSource)
             .zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
-    private final ApplicationId app = ProvisioningTester.applicationId();
+    private final ApplicationId app = ProvisioningTester.applicationId("a1");
 
     @Test
     public void single_group_same_cluster_size_resource_increase() {
@@ -167,8 +167,6 @@ public class InPlaceResizeProvisionTest {
         assertEquals(0, listCluster(content1).retired().size());
     }
 
-
-    /** In this scenario there should be no resizing */
     @Test
     public void increase_size_decrease_resources() {
         addParentHosts(14, largeResources.with(fast));
@@ -198,15 +196,15 @@ public class InPlaceResizeProvisionTest {
         assertSizeAndResources(listCluster(content1).retired(), 4, resources);
         assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources);
 
-        // ... same with setting a node to want to retire
-        Node nodeToWantoToRetire = listCluster(content1).not().retired().asList().get(0);
-        try (NodeMutex lock = tester.nodeRepository().nodes().lockAndGetRequired(nodeToWantoToRetire)) {
+        // Here we'll unretire and resize one of the previously retired nodes as there is no rule against it
+        Node nodeToWantToRetire = listCluster(content1).not().retired().asList().get(0);
+        try (NodeMutex lock = tester.nodeRepository().nodes().lockAndGetRequired(nodeToWantToRetire)) {
             tester.nodeRepository().nodes().write(lock.node().withWantToRetire(true, Agent.system,
                     tester.clock().instant()), lock);
         }
         new PrepareHelper(tester, app).prepare(content1, 8, 1, halvedResources).activate();
-        assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire)));
-        assertEquals(5, listCluster(content1).retired().size());
+        assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantToRetire)));
+        assertEquals(4, listCluster(content1).retired().size());
         assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java
index 32db213c445..c82b29c7d65 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java
@@ -24,17 +24,17 @@ public class NodeCandidateTest {
     @Test
     public void testOrdering() {
         List<NodeCandidate> expected = List.of(
-                new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, true, false, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false),
-                new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false)
+                new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, true, false, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), false, new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false),
+                new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false)
         );
         assertOrder(expected);
     }
@@ -148,7 +148,7 @@ public class NodeCandidateTest {
         Node parent = Node.create(hostname + "parent", hostname, new Flavor(totalHostResources), Node.State.ready, NodeType.host)
                           .ipConfig(IP.Config.of(Set.of("::1"), Set.of("::2")))
                           .build();
-        return new NodeCandidate.ConcreteNodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent),
+        return new NodeCandidate.ConcreteNodeCandidate(node, false, totalHostResources.subtract(allocatedHostResources), Optional.of(parent),
                                                        false, exclusiveSwitch, false, true, false);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index a76b576e430..cb4644f179f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -241,7 +241,7 @@ public class ProvisioningTest {
     public void application_deployment_variable_application_size() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
-        ApplicationId application1 = ProvisioningTester.applicationId();
+        ApplicationId application1 = ProvisioningTester.applicationId("a1");
 
         tester.makeReadyHosts(30, defaultResources);
         tester.activateTenantHosts();
@@ -821,7 +821,7 @@ public class ProvisioningTest {
     public void highest_node_indexes_are_retired_first() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
-        ApplicationId application1 = ProvisioningTester.applicationId();
+        ApplicationId application1 = ProvisioningTester.applicationId("a1");
 
         tester.makeReadyHosts(14, defaultResources).activateTenantHosts();
 
@@ -833,17 +833,19 @@ public class ProvisioningTest {
         SystemState state2 = prepare(application1, 2, 2, 2, 2, defaultResources, tester);
         tester.activate(application1, state2.allHosts);
 
-        // content0
-        assertFalse(state2.hostByMembership("content0", 0, 0).membership().get().retired());
-        assertFalse(state2.hostByMembership("content0", 0, 1).membership().get().retired());
-        assertTrue( state2.hostByMembership("content0", 0, 2).membership().get().retired());
-        assertTrue( state2.hostByMembership("content0", 0, 3).membership().get().retired());
-
-        // content1
-        assertFalse(state2.hostByMembership("content1", 0, 0).membership().get().retired());
-        assertFalse(state2.hostByMembership("content1", 0, 1).membership().get().retired());
-        assertTrue( state2.hostByMembership("content1", 0, 2).membership().get().retired());
-        assertTrue( state2.hostByMembership("content1", 0, 3).membership().get().retired());
+        List<Integer> unretiredInContent0Indices = state2.content0.stream().filter(h -> ! h.membership().get().retired()).map(h -> h.membership().get().index()).toList();
+        for (var host : state2.content0) {
+            if ( ! host.membership().get().retired()) continue;
+            for (int unretiredIndex : unretiredInContent0Indices)
+                assertTrue(host.membership().get().index() > unretiredIndex);
+        }
+
+        List<Integer> unretiredInContent1Indices = state2.content1.stream().filter(h -> ! h.membership().get().retired()).map(h -> h.membership().get().index()).toList();
+        for (var host : state2.content1) {
+            if ( ! host.membership().get().retired()) continue;
+            for (int unretiredIndex : unretiredInContent1Indices)
+                assertTrue(host.membership().get().index() > unretiredIndex);
+        }
     }
 
     @Test
@@ -857,7 +859,7 @@ public class ProvisioningTest {
 
         tester.deploy(application, spec, Capacity.from(new ClusterResources(6, 1, defaultResources)));
 
-        // Pick out a random application node and make it's parent larger, this will make it the spare host
+        // Pick out a random application node and make its parent larger, this will make it the spare host
         NodeList nodes = tester.nodeRepository().nodes().list();
         Node randomNode = nodes.owner(application).shuffle(new Random()).first().get();
         tester.nodeRepository().nodes().write(nodes.parentOf(randomNode).get()
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
index a6a988052e6..6ec189d98c3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
@@ -23,6 +23,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.yolean.Exceptions;
 import org.junit.Test;
 
 import java.util.HashSet;
@@ -442,9 +443,8 @@ public class VirtualNodeProvisioningTest {
                          "Could not satisfy request for 3 nodes with " +
                          "[vcpu: 2.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " +
                          "in tenant2.app2 container cluster 'my-container' 6.39: " +
-                         "Node allocation failure on group 0: " +
                          "Not enough suitable nodes available due to host exclusivity constraints",
-                         e.getMessage());
+                         Exceptions.toMessageString(e));
         }
 
         // Adding 3 nodes of another application for the same tenant works
@@ -469,8 +469,8 @@ public class VirtualNodeProvisioningTest {
             assertEquals("Could not satisfy request for 2 nodes with " +
                          "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any] " +
                          "in tenant.app1 content cluster 'my-content'" +
-                         " 6.42: Node allocation failure on group 0",
-                         e.getMessage());
+                         " 6.42",
+                         Exceptions.toMessageString(e));
         }
     }
 
diff --git a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java
index 4f3f048eb0c..a50ea2b97c1 100644
--- a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java
+++ b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java
@@ -27,6 +27,8 @@ public class Exceptions {
         for (; t != null; t = t.getCause()) {
             message = getMessage(t);
             if (message == null) continue;
+            message = message.trim();
+            if (message.isEmpty()) continue;
             if (message.equals(lastMessage)) continue;
             if (b.length() > 0) {
                 b.append(": ");