Prefer latest generation with fallback to older

4 files changed, 40 insertions, 37 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
index 5a53681001f..3f9e8ef4407 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
@@ -213,7 +213,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
             List<Node> hosts = new ArrayList<>();
             HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion,
                                                                     HostSharing.shared, Optional.empty(), Optional.empty(),
-                                                                    nodeRepository().zone().cloud().account());
+                                                                    nodeRepository().zone().cloud().account(), false);
             hostProvisioner.provisionHosts(request,
                                            provisionedHosts -> {
                                                hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 857db2ec132..3823f4906c2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -119,7 +119,8 @@ public class GroupPreparer {
                                                                             sharing,
                                                                             Optional.of(cluster.type()),
                                                                             Optional.of(cluster.id()),
-                                                                            requestedNodes.cloudAccount());
+                                                                            requestedNodes.cloudAccount(),
+                                                                            false);
                     hostProvisioner.get().provisionHosts(request, whenProvisioned);
                 } catch (NodeAllocationException e) {
                     // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java
index 072a556c3b3..f7b9c9016b1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java
@@ -15,20 +15,22 @@ import java.util.Optional;
 /**
  * A host provisioning request. This contains the details required to provision a host.
  *
- * @param indices          list of unique provision indices which will be used to generate the node hostnames
- *                         on the form of <code>[prefix][index].[domain]</code>
- * @param type             the host type to provision
- * @param resources        the resources needed per node - the provisioned host may be significantly larger
- * @param owner            id of the application that will own the provisioned host
- * @param osVersion        the OS version to use. If this version does not exist, implementations may choose a suitable
- *                         fallback version.
- * @param sharing          puts requirements on sharing or exclusivity of the host to be provisioned.
- * @param clusterType      the cluster we are provisioning for, or empty if we are provisioning hosts
- *                         to be shared by multiple cluster nodes
- * @param clusterId        the id of the cluster we are provisioning for, or empty if we are provisioning hosts
- *                         to be shared by multiple cluster nodes
- * @param cloudAccount     the cloud account to use
- *
+ * @param indices                 List of unique provision indices which will be used to generate the node hostnames
+ *                                on the form of <code>[prefix][index].[domain]</code>.
+ * @param type                    The host type to provision.
+ * @param resources               The resources needed per node - the provisioned host may be significantly larger.
+ * @param owner                   ID of the application that will own the provisioned host.
+ * @param osVersion               The OS version to use. If this version does not exist, implementations may choose a suitable
+ *                                fallback version.
+ * @param sharing                 Puts requirements on sharing or exclusivity of the host to be provisioned.
+ * @param clusterType             The cluster we are provisioning for, or empty if we are provisioning hosts
+ *                                to be shared by multiple cluster nodes.
+ * @param clusterId               The ID of the cluster we are provisioning for, or empty if we are provisioning hosts
+ *                                to be shared by multiple cluster nodes.
+ * @param cloudAccount            The cloud account to use.
+ * @param requireLatestGeneration Whether to require the latest generation when choosing a flavor. Latest generation will
+ *                                always be preferred, but setting this to true disallows falling back to an older
+ *                                generation.
  * @author mpolden
  */
 public record HostProvisionRequest(List<Integer> indices,
@@ -39,12 +41,13 @@ public record HostProvisionRequest(List<Integer> indices,
                                    HostProvisioner.HostSharing sharing,
                                    Optional<ClusterSpec.Type> clusterType,
                                    Optional<ClusterSpec.Id> clusterId,
-                                   CloudAccount cloudAccount) {
+                                   CloudAccount cloudAccount,
+                                   boolean requireLatestGeneration) {
 
     public HostProvisionRequest(List<Integer> indices, NodeType type, NodeResources resources,
                                 ApplicationId owner, Version osVersion, HostProvisioner.HostSharing sharing,
                                 Optional<ClusterSpec.Type> clusterType, Optional<ClusterSpec.Id> clusterId,
-                                CloudAccount cloudAccount) {
+                                CloudAccount cloudAccount, boolean requireLatestGeneration) {
         this.indices = List.copyOf(Objects.requireNonNull(indices));
         this.type = Objects.requireNonNull(type);
         this.resources = Objects.requireNonNull(resources);
@@ -54,6 +57,7 @@ public record HostProvisionRequest(List<Integer> indices,
         this.clusterType = Objects.requireNonNull(clusterType);
         this.clusterId = Objects.requireNonNull(clusterId);
         this.cloudAccount = Objects.requireNonNull(cloudAccount);
+        this.requireLatestGeneration = requireLatestGeneration;
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index 7f0d201b3e4..e6f2758ff7f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -31,6 +31,7 @@ import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Used to manage a list of nodes during the node reservation process to fulfill the nodespec.
@@ -73,7 +74,7 @@ class NodeAllocation {
     /** The number of nodes that just now was changed to retired */
     private int wasRetiredJustNow = 0;
 
-    /** The node indexes to verify uniqueness of each members index */
+    /** The node indexes to verify uniqueness of each member's index */
     private final Set<Integer> indexes = new HashSet<>();
 
     /** The next membership index to assign to a new node */
@@ -93,11 +94,11 @@ class NodeAllocation {
         this.nodeRepository = nodeRepository;
         this.nodeResourceLimits = new NodeResourceLimits(nodeRepository);
         this.requiredHostFlavor = Optional.of(PermanentFlags.HOST_FLAVOR.bindTo(nodeRepository.flagSource())
-                        .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
-                        .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name())
-                        .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value())
-                        .value())
-                .filter(s -> !s.isBlank());
+                                                                        .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
+                                                                        .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name())
+                                                                        .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value())
+                                                                        .value())
+                                          .filter(s -> !s.isBlank());
     }
 
     /**
@@ -379,8 +380,8 @@ class NodeAllocation {
      * @return the final list of nodes
      */
     List<Node> finalNodes() {
-        int wantToRetireCount = (int) nodes.values().stream().filter(NodeCandidate::wantToRetire).count();
-        int currentRetiredCount = (int) nodes.values().stream().filter(node -> node.allocation().get().membership().retired()).count();
+        int wantToRetireCount = (int) matching(NodeCandidate::wantToRetire).count();
+        int currentRetiredCount = (int) matching(node -> node.allocation().get().membership().retired()).count();
         int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), wantToRetireCount, currentRetiredCount);
 
         if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0
@@ -415,24 +416,21 @@ class NodeAllocation {
             nodes.put(candidate.toNode().hostname(), candidate);
         }
 
-        return nodes.values().stream().map(n -> n.toNode()).toList();
+        return nodes.values().stream().map(NodeCandidate::toNode).toList();
     }
 
     List<Node> reservableNodes() {
         // Include already reserved nodes to extend reservation period and to potentially update their cluster spec.
         EnumSet<Node.State> reservableStates = EnumSet.of(Node.State.inactive, Node.State.ready, Node.State.reserved);
-        return nodesFilter(n -> ! n.isNew && reservableStates.contains(n.state()));
+        return matching(n -> ! n.isNew && reservableStates.contains(n.state())).toList();
     }
 
     List<Node> newNodes() {
-        return nodesFilter(n -> n.isNew);
+        return matching(node -> node.isNew).toList();
     }
 
-    private List<Node> nodesFilter(Predicate<NodeCandidate> predicate) {
-        return nodes.values().stream()
-                .filter(predicate)
-                .map(n -> n.toNode())
-                .toList();
+    private Stream<Node> matching(Predicate<NodeCandidate> predicate) {
+        return nodes.values().stream().filter(predicate).map(NodeCandidate::toNode);
     }
 
     /** Returns the number of nodes accepted this far */
@@ -487,8 +485,8 @@ class NodeAllocation {
         outsideRealLimits("node real resources is outside limits"),
         violatesParentHostPolicy("node violates parent host policy"),
         incompatibleResources("node resources are incompatible"),
-        hardRequest("node is requested to retire"),
-        softRequest("node is requested to retire (soft)"),
+        hardRequest("node is requested and required to retire"),
+        softRequest("node is requested to retire"),
         violatesExclusivity("node violates host exclusivity"),
         violatesHostFlavor("node violates host flavor"),
         none("");
@@ -499,7 +497,7 @@ class NodeAllocation {
             this.description = description;
         }
 
-        /** Human readable description of this cause */
+        /** Human-readable description of this cause */
         public String description() {
             return description;
         }