diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-06-01 14:02:47 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-06-01 14:09:40 +0200 |
commit | 76b52f4ec5910cb81b8ac7ecd47e28d904ab7e18 (patch) | |
tree | 65766658349b1f8001bc2e8b91ba242a23d05152 /node-repository | |
parent | 83d07d212130654bdba77619d0e2e3307d44730b (diff) |
Prefer latest generation with fallback to older
Diffstat (limited to 'node-repository')
4 files changed, 40 insertions, 37 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 5a53681001f..3f9e8ef4407 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -213,7 +213,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { List<Node> hosts = new ArrayList<>(); HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, HostSharing.shared, Optional.empty(), Optional.empty(), - nodeRepository().zone().cloud().account()); + nodeRepository().zone().cloud().account(), false); hostProvisioner.provisionHosts(request, provisionedHosts -> { hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 857db2ec132..3823f4906c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -119,7 +119,8 @@ public class GroupPreparer { sharing, Optional.of(cluster.type()), Optional.of(cluster.id()), - requestedNodes.cloudAccount()); + requestedNodes.cloudAccount(), + false); hostProvisioner.get().provisionHosts(request, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java index 072a556c3b3..f7b9c9016b1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java @@ -15,20 +15,22 @@ import java.util.Optional; /** * A host provisioning request. This contains the details required to provision a host. * - * @param indices list of unique provision indices which will be used to generate the node hostnames - * on the form of <code>[prefix][index].[domain]</code> - * @param type the host type to provision - * @param resources the resources needed per node - the provisioned host may be significantly larger - * @param owner id of the application that will own the provisioned host - * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable - * fallback version. - * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. - * @param clusterType the cluster we are provisioning for, or empty if we are provisioning hosts - * to be shared by multiple cluster nodes - * @param clusterId the id of the cluster we are provisioning for, or empty if we are provisioning hosts - * to be shared by multiple cluster nodes - * @param cloudAccount the cloud account to use - * + * @param indices List of unique provision indices which will be used to generate the node hostnames + * on the form of <code>[prefix][index].[domain]</code>. + * @param type The host type to provision. + * @param resources The resources needed per node - the provisioned host may be significantly larger. + * @param owner ID of the application that will own the provisioned host. + * @param osVersion The OS version to use. If this version does not exist, implementations may choose a suitable + * fallback version. + * @param sharing Puts requirements on sharing or exclusivity of the host to be provisioned. + * @param clusterType The cluster we are provisioning for, or empty if we are provisioning hosts + * to be shared by multiple cluster nodes. + * @param clusterId The ID of the cluster we are provisioning for, or empty if we are provisioning hosts + * to be shared by multiple cluster nodes. + * @param cloudAccount The cloud account to use. + * @param requireLatestGeneration Whether to require the latest generation when choosing a flavor. Latest generation will + * always be preferred, but setting this to true disallows falling back to an older + * generation. * @author mpolden */ public record HostProvisionRequest(List<Integer> indices, @@ -39,12 +41,13 @@ public record HostProvisionRequest(List<Integer> indices, HostProvisioner.HostSharing sharing, Optional<ClusterSpec.Type> clusterType, Optional<ClusterSpec.Id> clusterId, - CloudAccount cloudAccount) { + CloudAccount cloudAccount, + boolean requireLatestGeneration) { public HostProvisionRequest(List<Integer> indices, NodeType type, NodeResources resources, ApplicationId owner, Version osVersion, HostProvisioner.HostSharing sharing, Optional<ClusterSpec.Type> clusterType, Optional<ClusterSpec.Id> clusterId, - CloudAccount cloudAccount) { + CloudAccount cloudAccount, boolean requireLatestGeneration) { this.indices = List.copyOf(Objects.requireNonNull(indices)); this.type = Objects.requireNonNull(type); this.resources = Objects.requireNonNull(resources); @@ -54,6 +57,7 @@ public record HostProvisionRequest(List<Integer> indices, this.clusterType = Objects.requireNonNull(clusterType); this.clusterId = Objects.requireNonNull(clusterId); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.requireLatestGeneration = requireLatestGeneration; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 7f0d201b3e4..e6f2758ff7f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -31,6 +31,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Used to manage a list of nodes during the node reservation process to fulfill the nodespec. @@ -73,7 +74,7 @@ class NodeAllocation { /** The number of nodes that just now was changed to retired */ private int wasRetiredJustNow = 0; - /** The node indexes to verify uniqueness of each members index */ + /** The node indexes to verify uniqueness of each member's index */ private final Set<Integer> indexes = new HashSet<>(); /** The next membership index to assign to a new node */ @@ -93,11 +94,11 @@ class NodeAllocation { this.nodeRepository = nodeRepository; this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.requiredHostFlavor = Optional.of(PermanentFlags.HOST_FLAVOR.bindTo(nodeRepository.flagSource()) - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name()) - .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) - .value()) - .filter(s -> !s.isBlank()); + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name()) + .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) + .value()) + .filter(s -> !s.isBlank()); } /** @@ -379,8 +380,8 @@ class NodeAllocation { * @return the final list of nodes */ List<Node> finalNodes() { - int wantToRetireCount = (int) nodes.values().stream().filter(NodeCandidate::wantToRetire).count(); - int currentRetiredCount = (int) nodes.values().stream().filter(node -> node.allocation().get().membership().retired()).count(); + int wantToRetireCount = (int) matching(NodeCandidate::wantToRetire).count(); + int currentRetiredCount = (int) matching(node -> node.allocation().get().membership().retired()).count(); int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), wantToRetireCount, currentRetiredCount); if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0 @@ -415,24 +416,21 @@ class NodeAllocation { nodes.put(candidate.toNode().hostname(), candidate); } - return nodes.values().stream().map(n -> n.toNode()).toList(); + return nodes.values().stream().map(NodeCandidate::toNode).toList(); } List<Node> reservableNodes() { // Include already reserved nodes to extend reservation period and to potentially update their cluster spec. EnumSet<Node.State> reservableStates = EnumSet.of(Node.State.inactive, Node.State.ready, Node.State.reserved); - return nodesFilter(n -> ! n.isNew && reservableStates.contains(n.state())); + return matching(n -> ! n.isNew && reservableStates.contains(n.state())).toList(); } List<Node> newNodes() { - return nodesFilter(n -> n.isNew); + return matching(node -> node.isNew).toList(); } - private List<Node> nodesFilter(Predicate<NodeCandidate> predicate) { - return nodes.values().stream() - .filter(predicate) - .map(n -> n.toNode()) - .toList(); + private Stream<Node> matching(Predicate<NodeCandidate> predicate) { + return nodes.values().stream().filter(predicate).map(NodeCandidate::toNode); } /** Returns the number of nodes accepted this far */ @@ -487,8 +485,8 @@ class NodeAllocation { outsideRealLimits("node real resources is outside limits"), violatesParentHostPolicy("node violates parent host policy"), incompatibleResources("node resources are incompatible"), - hardRequest("node is requested to retire"), - softRequest("node is requested to retire (soft)"), + hardRequest("node is requested and required to retire"), + softRequest("node is requested to retire"), violatesExclusivity("node violates host exclusivity"), violatesHostFlavor("node violates host flavor"), none(""); @@ -499,7 +497,7 @@ class NodeAllocation { this.description = description; } - /** Human readable description of this cause */ + /** Human-readable description of this cause */ public String description() { return description; } |