diff options
26 files changed, 201 insertions, 229 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 5fae9497f69..153b305dc01 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -90,6 +90,10 @@ public final class ClusterSpec { return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } + public ClusterSpec withExclusivity(boolean exclusive) { + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); + } + public ClusterSpec exclusive(boolean exclusive) { return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, loadBalancerSettings, stateful); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 0bf32e534b7..fb21b009a30 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -7,11 +7,15 @@ import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.JacksonFlag; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; @@ -62,6 +66,7 @@ public class NodeRepository extends AbstractComponent { private final MetricsDb metricsDb; private final Orchestrator orchestrator; private final int spareCount; + private final JacksonFlag<SharedHost> sharedHosts; /** * Creates a node repository from a zookeeper provider. @@ -134,6 +139,7 @@ public class NodeRepository extends AbstractComponent { this.metricsDb = metricsDb; this.orchestrator = orchestrator; this.spareCount = spareCount; + this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(flagSource()); nodes.rewrite(); } @@ -197,7 +203,8 @@ public class NodeRepository extends AbstractComponent { * perfectly. */ public boolean exclusiveAllocation(ClusterSpec clusterSpec) { - return clusterSpec.isExclusive() || ! zone().cloud().allowHostSharing(); + return clusterSpec.isExclusive() || + ( !zone().cloud().allowHostSharing() && !sharedHosts.value().isEnabled(clusterSpec.type().name())); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 389be5b6652..3d76c8e3f94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -164,10 +164,10 @@ public class AllocatableClusterResources { if (! exclusive) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive); - advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterSpec.type(), exclusive); // Ask for something legal + advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterSpec, exclusive); // Ask for something legal advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive); // What we'll really get - if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec.type())) + if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec)) return Optional.empty(); if (anySatisfies(realResources, availableRealHostResources)) return Optional.of(new AllocatableClusterResources(wantedResources.with(realResources), @@ -187,7 +187,7 @@ public class AllocatableClusterResources { // Adjust where we don't need exact match to the flavor if (flavor.resources().storageType() == NodeResources.StorageType.remote) { - double diskGb = systemLimits.enlargeToLegal(cappedWantedResources, clusterSpec.type(), exclusive).diskGb(); + double diskGb = systemLimits.enlargeToLegal(cappedWantedResources, clusterSpec, exclusive).diskGb(); advertisedResources = advertisedResources.withDiskGb(diskGb); realResources = realResources.withDiskGb(diskGb); } @@ -197,7 +197,7 @@ public class AllocatableClusterResources { } if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; - if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec.type())) continue; + if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec)) continue; var candidate = new AllocatableClusterResources(wantedResources.with(realResources), advertisedResources, wantedResources, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 36b32f0b099..cb5d8dd5042 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -63,7 +63,7 @@ public class Limits { public Limits fullySpecified(ClusterSpec clusterSpec, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(clusterSpec, applicationId, clusterSpec.isExclusive()); + var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(clusterSpec, applicationId); var specifiedMin = min.nodeResources().isUnspecified() ? min.with(defaultResources) : min; var specifiedMax = max.nodeResources().isUnspecified() ? max.with(defaultResources) : max; return new Limits(specifiedMin, specifiedMax); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java index fd9771103de..532e6747d9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java @@ -9,6 +9,7 @@ import java.util.Objects; * @author hakon */ public class Address { + private final String hostname; public Address(String hostname) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 8d6c6b4bb62..a1400626658 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -10,10 +10,8 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Map; import java.util.TreeMap; @@ -30,13 +28,13 @@ import static java.util.Objects.requireNonNull; */ public class CapacityPolicies { + private final NodeRepository nodeRepository; private final Zone zone; - private final JacksonFlag<SharedHost> sharedHosts; private final StringFlag adminClusterNodeArchitecture; public CapacityPolicies(NodeRepository nodeRepository) { + this.nodeRepository = nodeRepository; this.zone = nodeRepository.zone(); - this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()); this.adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(nodeRepository.flagSource()); } @@ -79,16 +77,15 @@ public class CapacityPolicies { return target; } - public NodeResources defaultNodeResources(ClusterSpec clusterSpec, ApplicationId applicationId, boolean exclusive) { + public NodeResources defaultNodeResources(ClusterSpec clusterSpec, ApplicationId applicationId) { if (clusterSpec.type() == ClusterSpec.Type.admin) { Architecture architecture = adminClusterArchitecture(applicationId); if (clusterSpec.id().value().equals("cluster-controllers")) { - return clusterControllerResources(clusterSpec, exclusive) - .with(architecture); + return clusterControllerResources(clusterSpec).with(architecture); } - return (requiresExclusiveHost(clusterSpec.type(), exclusive) + return (nodeRepository.exclusiveAllocation(clusterSpec) ? versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources())) : versioned(clusterSpec, Map.of(new Version(0), smallestSharedResources()))) .with(architecture); @@ -107,8 +104,8 @@ public class CapacityPolicies { } } - private NodeResources clusterControllerResources(ClusterSpec clusterSpec, boolean exclusive) { - if (requiresExclusiveHost(clusterSpec.type(), exclusive)) { + private NodeResources clusterControllerResources(ClusterSpec clusterSpec) { + if (nodeRepository.exclusiveAllocation(clusterSpec)) { return versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources())); } return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3))); @@ -118,11 +115,6 @@ public class CapacityPolicies { return Architecture.valueOf(adminClusterNodeArchitecture.with(APPLICATION_ID, instance.serializedForm()).value()); } - /** Returns whether an exclusive host is required for given cluster type and exclusivity requirement */ - private boolean requiresExclusiveHost(ClusterSpec.Type type, boolean exclusive) { - return ! zone.cloud().allowHostSharing() && (exclusive || !sharedHosts.value().isEnabled(type.name())); - } - /** Returns the resources for the newest version not newer than that requested in the cluster spec. */ static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), @@ -145,9 +137,10 @@ public class CapacityPolicies { } /** Returns whether the nodes requested can share physical host with other applications */ - public boolean decideExclusivity(Capacity capacity, boolean requestedExclusivity) { - if (capacity.cloudAccount().isPresent()) return true; // Implicit exclusive when using custom cloud account - return requestedExclusivity && (capacity.isRequired() || zone.environment() == Environment.prod); + public ClusterSpec decideExclusivity(Capacity capacity, ClusterSpec requestedCluster) { + if (capacity.cloudAccount().isPresent()) return requestedCluster.withExclusivity(true); // Implicit exclusive + boolean exclusive = requestedCluster.isExclusive() && (capacity.isRequired() || zone.environment() == Environment.prod); + return requestedCluster.withExclusivity(exclusive); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index ad2973ff435..5e01ba5b0a6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -74,9 +74,9 @@ public class GroupPreparer { public PrepareResult prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups, NodesAndHosts<LockedNodeList> allNodesAndHosts) { - log.log(Level.FINE, "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " + requestedNodes.resources().orElse(NodeResources.unspecified())); - // Try preparing in memory without global unallocated lock. Most of the time there should be no changes and we - // can return nodes previously allocated. + log.log(Level.FINE, () -> "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " + requestedNodes.resources().orElse(NodeResources.unspecified())); + // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, + // and we can return nodes previously allocated. NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, indices::probeNext, wantedGroups, allNodesAndHosts); if (probeAllocation.fulfilledAndNoChanges()) { @@ -105,7 +105,7 @@ public class GroupPreparer { indices::next, wantedGroups, allNodesAndHosts); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { - HostSharing sharing = hostSharing(requestedNodes, hostType); + HostSharing sharing = hostSharing(cluster, hostType); Version osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); NodeAllocation.HostDeficit deficit = allocation.hostDeficit().get(); @@ -125,7 +125,8 @@ public class GroupPreparer { try { hostProvisioner.get().provisionHosts( allocation.provisionIndices(deficit.count()), hostType, deficit.resources(), application, - osVersion, sharing, Optional.of(cluster.type()), requestedNodes.cloudAccount(), provisionedHostsConsumer); + osVersion, sharing, Optional.of(cluster.type()), requestedNodes.cloudAccount(), + provisionedHostsConsumer); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been @@ -173,12 +174,11 @@ public class GroupPreparer { (hostType == NodeType.host || hostType.isConfigServerHostLike()); } - private static HostSharing hostSharing(NodeSpec spec, NodeType hostType) { - HostSharing sharing = spec.isExclusive() ? HostSharing.exclusive : HostSharing.any; - if (!hostType.isSharable() && sharing != HostSharing.any) { - throw new IllegalArgumentException(hostType + " does not support sharing requirement"); - } - return sharing; + private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { + if ( hostType.isSharable()) + return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + else + return HostSharing.any; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 95d10557e3a..38fa1abf8e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.List; import java.util.Optional; @@ -47,7 +48,8 @@ public interface HostProvisioner { * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable * fallback version. * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. - * @param clusterType provision host exclusively for this cluster type + * @param clusterType the cluster we are provisioning for, or empty if we are provisioning hosts + * to be shared by multiple cluster nodes * @param cloudAccount the cloud account to use * @param provisionedHostConsumer consumer of {@link ProvisionedHost}s describing the provisioned nodes, * the {@link Node} returned from {@link ProvisionedHost#generateHost()} must be diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 178b42096e6..8d350e304a2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -34,8 +34,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Used to manage a list of nodes during the node reservation process - * in order to fulfill the nodespec. + * Used to manage a list of nodes during the node reservation process to fulfill the nodespec. * * @author bratseth */ @@ -128,9 +127,8 @@ class NodeAllocation { if ((! saturated() && hasCompatibleResources(candidate) && requestedNodes.acceptable(candidate)) || acceptToRetire) { candidate = candidate.withNode(); - if (candidate.isValid()) { + if (candidate.isValid()) acceptNode(candidate, shouldRetire(candidate, candidates), resizeable); - } } } else if (! saturated() && hasCompatibleResources(candidate)) { @@ -221,7 +219,7 @@ class NodeAllocation { /** * Returns whether this node should be accepted into the cluster even if it is not currently desired - * (already enough nodes, or wrong flavor). + * (already enough nodes, or wrong resources, etc.). * Such nodes will be marked retired during finalization of the list of accepted nodes. * The conditions for this are: * @@ -263,8 +261,9 @@ class NodeAllocation { || ! ( requestedNodes.needsResize(node) && node.allocation().get().membership().retired())) acceptedWithoutResizingRetired++; - if (resizeable && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) + if (resizeable && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) { node = resize(node); + } if (node.state() != Node.State.active) // reactivated node - wipe state that deactivated it node = node.unretire().removable(false); @@ -316,7 +315,7 @@ class NodeAllocation { * Returns {@link HostDeficit} describing the host deficit for the given {@link NodeSpec}. * * @return empty if the requested spec is already fulfilled. Otherwise returns {@link HostDeficit} containing the - * flavor and host count required to cover the deficit. + * flavor and host count required to cover the deficit. */ Optional<HostDeficit> hostDeficit() { if (nodeType().isHost()) { @@ -521,6 +520,11 @@ class NodeAllocation { return count; } + @Override + public String toString() { + return "deficit of " + count + " nodes with " + resources; + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index fe4eb5d68c9..c1d65e0df4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -146,12 +146,13 @@ public class NodePrioritizer { if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requestedNodes.resources().get())) continue; if ( ! allNodesAndHosts.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; + candidates.add(NodeCandidate.createNewChild(requestedNodes.resources().get(), - capacity.availableCapacityOf(host), - host, - spareHosts.contains(host), - allNodesAndHosts.nodes(), - nameResolver)); + capacity.availableCapacityOf(host), + host, + spareHosts.contains(host), + allNodesAndHosts.nodes(), + nameResolver)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 0ed4f4ee9b0..3256ae7c73a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -95,29 +95,30 @@ public class NodeRepositoryProvisioner implements Provisioner { NodeResources resources; NodeSpec nodeSpec; if (requested.type() == NodeType.tenant) { - boolean exclusive = capacityPolicies.decideExclusivity(requested, cluster.isExclusive()); - Capacity actual = capacityPolicies.applyOn(requested, application, exclusive); + cluster = capacityPolicies.decideExclusivity(requested, cluster); + Capacity actual = capacityPolicies.applyOn(requested, application, cluster.isExclusive()); ClusterResources target = decideTargetResources(application, cluster, actual); ensureRedundancy(target.nodes(), cluster, actual.canFail(), application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); groups = target.groups(); - resources = getNodeResources(cluster, target.nodeResources(), application, exclusive); - nodeSpec = NodeSpec.from(target.nodes(), resources, exclusive, actual.canFail(), + resources = getNodeResources(cluster, target.nodeResources(), application); + nodeSpec = NodeSpec.from(target.nodes(), resources, cluster.isExclusive(), actual.canFail(), requested.cloudAccount().orElse(nodeRepository.zone().cloud().account())); } else { groups = 1; // type request with multiple groups is not supported - resources = getNodeResources(cluster, requested.minResources().nodeResources(), application, true); + cluster = cluster.withExclusivity(true); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account()); } return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), requireCompatibleResources(resources, cluster)); } - private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId, boolean exclusive) { + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { return nodeResources.isUnspecified() - ? capacityPolicies.defaultNodeResources(cluster, applicationId, exclusive) + ? capacityPolicies.defaultNodeResources(cluster, applicationId) : nodeResources; } @@ -178,8 +179,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { var initial = requested.minResources(); if (initial.nodeResources().isUnspecified()) - initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec, applicationId, - capacityPolicies.decideExclusivity(requested, clusterSpec.isExclusive()))); + initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec, applicationId)); return initial; } @@ -268,8 +268,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); - boolean exclusiveHosts = spec.isExclusive() || ! nodeRepository.zone().cloud().allowHostSharing(); - if (exclusiveHosts) + if (nodeRepository.exclusiveAllocation(spec)) message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits)); return new IllegalArgumentException(message.toString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 81dd852e2a1..66895867623 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -28,10 +28,10 @@ public class NodeResourceLimits { public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ClusterSpec cluster) { if (requested.isUnspecified()) return; - if (requested.vcpu() < minAdvertisedVcpu(cluster.type())) - illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster.type())); - if (requested.memoryGb() < minAdvertisedMemoryGb(cluster.type())) - illegal(type, "memoryGb", "Gb", cluster, requested.memoryGb(), minAdvertisedMemoryGb(cluster.type())); + if (requested.vcpu() < minAdvertisedVcpu(cluster)) + illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster)); + if (requested.memoryGb() < minAdvertisedMemoryGb(cluster)) + illegal(type, "memoryGb", "Gb", cluster, requested.memoryGb(), minAdvertisedMemoryGb(cluster)); if (requested.diskGb() < minAdvertisedDiskGb(requested, cluster.isExclusive())) illegal(type, "diskGb", "Gb", cluster, requested.diskGb(), minAdvertisedDiskGb(requested, cluster.isExclusive())); } @@ -40,36 +40,36 @@ public class NodeResourceLimits { public boolean isWithinRealLimits(NodeCandidate candidateNode, ClusterSpec cluster) { if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository), - cluster.type()); + cluster); } /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ - public boolean isWithinRealLimits(NodeResources realResources, ClusterSpec.Type clusterType) { + public boolean isWithinRealLimits(NodeResources realResources, ClusterSpec cluster) { if (realResources.isUnspecified()) return true; - if (realResources.vcpu() < minRealVcpu(clusterType)) return false; - if (realResources.memoryGb() < minRealMemoryGb(clusterType)) return false; + if (realResources.vcpu() < minRealVcpu(cluster)) return false; + if (realResources.memoryGb() < minRealMemoryGb(cluster)) return false; if (realResources.diskGb() < minRealDiskGb()) return false; return true; } - public NodeResources enlargeToLegal(NodeResources requested, ClusterSpec.Type clusterType, boolean exclusive) { + public NodeResources enlargeToLegal(NodeResources requested, ClusterSpec cluster, boolean exclusive) { if (requested.isUnspecified()) return requested; - return requested.withVcpu(Math.max(minAdvertisedVcpu(clusterType), requested.vcpu())) - .withMemoryGb(Math.max(minAdvertisedMemoryGb(clusterType), requested.memoryGb())) + return requested.withVcpu(Math.max(minAdvertisedVcpu(cluster), requested.vcpu())) + .withMemoryGb(Math.max(minAdvertisedMemoryGb(cluster), requested.memoryGb())) .withDiskGb(Math.max(minAdvertisedDiskGb(requested, exclusive), requested.diskGb())); } - private double minAdvertisedVcpu(ClusterSpec.Type clusterType) { - if (zone().environment() == Environment.dev && zone().cloud().allowHostSharing()) return 0.1; - if (clusterType.isContent() && zone().environment().isProduction()) return 1.0; - if (clusterType == ClusterSpec.Type.admin) return 0.1; + private double minAdvertisedVcpu(ClusterSpec cluster) { + if (zone().environment() == Environment.dev && ! nodeRepository.exclusiveAllocation(cluster)) return 0.1; + if (cluster.type().isContent() && zone().environment().isProduction()) return 1.0; + if (cluster.type() == ClusterSpec.Type.admin) return 0.1; return 0.5; } - private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) { - if (clusterType == ClusterSpec.Type.admin) return 1; + private double minAdvertisedMemoryGb(ClusterSpec cluster) { + if (cluster.type() == ClusterSpec.Type.admin) return 1; return 4; } @@ -85,10 +85,10 @@ public class NodeResourceLimits { return 4; } - private double minRealVcpu(ClusterSpec.Type clusterType) { return minAdvertisedVcpu(clusterType); } + private double minRealVcpu(ClusterSpec cluster) { return minAdvertisedVcpu(cluster); } - private double minRealMemoryGb(ClusterSpec.Type clusterType) { - return minAdvertisedMemoryGb(clusterType) - 1.7; + private double minRealMemoryGb(ClusterSpec cluster) { + return minAdvertisedMemoryGb(cluster) - 1.7; } private double minRealDiskGb() { return 6; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 1525dbe1008..59c089943ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -154,6 +154,8 @@ public interface NodeSpec { @Override public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, ClusterSpec.Type type, boolean hasTopologyChange, int currentClusterSize) { + if (exclusive) return false; // exclusive resources must match the host + // Never allow in-place resize when also changing topology or decreasing cluster size if (hasTopologyChange || count < currentClusterSize) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 5a8c5221c47..15ee064b59f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -46,21 +46,19 @@ public class MockHostProvisioner implements HostProvisioner { private int deprovisionedHosts = 0; private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); private Optional<Flavor> hostFlavor = Optional.empty(); - private Cloud cloud; - public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb, Cloud cloud) { + public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) { this.flavors = List.copyOf(flavors); this.nameResolver = nameResolver; this.memoryTaxGb = memoryTaxGb; - this.cloud = cloud; } - public MockHostProvisioner(List<Flavor> flavors, Cloud cloud) { - this(flavors, 0, cloud); + public MockHostProvisioner(List<Flavor> flavors) { + this(flavors, 0); } - public MockHostProvisioner(List<Flavor> flavors, int memoryTaxGb, Cloud cloud) { - this(flavors, new MockNameResolver().mockAnyLookup(), memoryTaxGb, cloud); + public MockHostProvisioner(List<Flavor> flavors, int memoryTaxGb) { + this(flavors, new MockNameResolver().mockAnyLookup(), memoryTaxGb); } @Override @@ -68,10 +66,9 @@ public class MockHostProvisioner implements HostProvisioner { ApplicationId applicationId, Version osVersion, HostSharing sharing, Optional<ClusterSpec.Type> clusterType, CloudAccount cloudAccount, Consumer<List<ProvisionedHost>> provisionedHostsConsumer) { - boolean exclusive = sharing == HostSharing.exclusive || ! cloud.allowHostSharing(); Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream() - .filter(f -> exclusive ? compatible(f, resources) - : f.resources().satisfies(resources)) + .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources) + : f.resources().satisfies(resources)) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true))); List<ProvisionedHost> hosts = new ArrayList<>(); @@ -190,7 +187,7 @@ public class MockHostProvisioner implements HostProvisioner { .collect(Collectors.toList()); } - private Node withIpAssigned(Node node) { + public Node withIpAssigned(Node node) { if (!node.type().isHost()) { return node.with(node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname()))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index 0764d4527d5..dbc74f32f6b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -64,9 +64,9 @@ public class MockNameResolver implements NameResolver { return records.get(name); } if (mockAnyLookup) { - Set<String> ipAddresses = Set.of(randomIpAddress()); - records.put(name, ipAddresses); - return ipAddresses; + records.computeIfAbsent(name, (k) -> new HashSet<>()) + .add(randomIpAddress()); + return records.get(name); } throw new RuntimeException(new UnknownHostException("Could not resolve: " + name)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 4bc947cf095..fc837ee54b4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -60,7 +60,6 @@ public class AutoscalingTest { assertTrue(fixture.autoscale().target().isEmpty()); } - /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_no_autoscaling_with_no_measurements_exclusive() { var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); @@ -88,14 +87,11 @@ public class AutoscalingTest { .initialResources(Optional.empty()) .hostSharingFlag() .build(); - // TODO: Not actually at min since flags are inconsistently handled fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", - 7, 1, 2.0, 16.0, 384.0, + 7, 1, 2.0, 10.0, 384.0, fixture.currentResources().advertisedResources()); } - - /** When scaling up, disregard underutilized dimensions (memory here) */ @Test public void test_only_autoscaling_up_quickly() { @@ -291,7 +287,7 @@ public class AutoscalingTest { .build(); NodeResources defaultResources = - new CapacityPolicies(fixture.tester().nodeRepository()).defaultNodeResources(fixture.clusterSpec, fixture.applicationId, false); + new CapacityPolicies(fixture.tester().nodeRepository()).defaultNodeResources(fixture.clusterSpec, fixture.applicationId); fixture.tester().assertResources("Min number of nodes and default resources", 2, 1, defaultResources, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 7969864c063..de5d3e8544e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -44,26 +44,19 @@ class AutoscalingTester { private final HostResourcesCalculator hostResourcesCalculator; private final CapacityPolicies capacityPolicies; - public AutoscalingTester(Zone zone, - HostResourcesCalculator resourcesCalculator, - List<Flavor> hostFlavors, - InMemoryFlagSource flagSource, - int hostCount) { + public AutoscalingTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { this(zone, hostFlavors, resourcesCalculator, flagSource); for (Flavor flavor : hostFlavors) provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8); provisioningTester.activateTenantHosts(); } - private AutoscalingTester(Zone zone, - List<Flavor> flavors, - HostResourcesCalculator resourcesCalculator, - InMemoryFlagSource flagSource) { + private AutoscalingTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { provisioningTester = new ProvisioningTester.Builder().zone(zone) .flavors(flavors) - .flagSource(flagSource) .resourcesCalculator(resourcesCalculator) - .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors, zone.cloud()) : null) + .flagSource(flagSource) + .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors) : null) .build(); hostResourcesCalculator = resourcesCalculator; @@ -151,7 +144,7 @@ class AutoscalingTester { } public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster.isExclusive())); + capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(cluster.id(), false, capacity); try (Mutex lock = nodeRepository().applications().lock(applicationId)) { @@ -257,8 +250,8 @@ class AutoscalingTester { private class MockHostProvisioner extends com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner { - public MockHostProvisioner(List<Flavor> flavors, Cloud cloud) { - super(flavors, cloud); + public MockHostProvisioner(List<Flavor> flavors) { + super(flavors); } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 0c146f525a2..ff04083ebde 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -235,14 +235,14 @@ public class Fixture { return this; } - public Fixture.Builder hostSharingFlag() { - var resources = new HostResources(8.0, 32.0, 100.0, 10.0, "fast", "local", null, 6, "x86_64"); - flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(resources), null), SharedHost.class); + public Fixture.Builder hostCount(int hostCount) { + this.hostCount = hostCount; return this; } - public Fixture.Builder hostCount(int hostCount) { - this.hostCount = hostCount; + public Fixture.Builder hostSharingFlag() { + var resources = new HostResources(8.0, 32.0, 100.0, 10.0, "fast", "local", null, 6, "x86_64"); + flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(resources), null), SharedHost.class); return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirerTest.java index 8bdb0fb2daf..ac20b9164f8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirerTest.java @@ -39,7 +39,7 @@ public class DirtyExpirerTest { private void assertAllocationAfterExpiry(boolean dynamicProvisioning) { Zone zone = new Zone(Cloud.builder().dynamicProvisioning(dynamicProvisioning).build(), SystemName.main, Environment.prod, RegionName.from("us-east")); ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) - .hostProvisioner(dynamicProvisioning ? new MockHostProvisioner(List.of(), zone.cloud()) : null) + .hostProvisioner(dynamicProvisioning ? new MockHostProvisioner(List.of()) : null) .build(); Node node = Node.create("id", "node1.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.dirty, NodeType.tenant) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacerTest.java index 599303bb098..16ee28bd5e7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacerTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertEquals; public class DiskReplacerTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final MockHostProvisioner hostProvisioner = new MockHostProvisioner(List.of(), Cloud.defaultCloud()); + private final MockHostProvisioner hostProvisioner = new MockHostProvisioner(List.of()); private final DiskReplacer diskReplacer = new DiskReplacer(tester.nodeRepository(), Duration.ofDays(1), new TestMetric(), hostProvisioner); @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 2cc20f50bbd..b26d9f677db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -626,7 +626,7 @@ public class HostCapacityMaintainerTest { Zone zone = new Zone(cloud, SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); - this.hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), nameResolver, 0, cloud); + this.hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), nameResolver, 0); this.provisioningTester = new ProvisioningTester.Builder().zone(zone) .flavors(flavors.getFlavors()) .nameResolver(nameResolver) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java index 606e379371a..5e507d447ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java @@ -42,7 +42,7 @@ public class HostResumeProvisionerTest { SystemName.defaultSystem(), Environment.dev, RegionName.defaultName()); - private final MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors, nameResolver, 0, zone.cloud()); + private final MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors, nameResolver, 0); private final ProvisioningTester tester = new ProvisioningTester.Builder() .zone(zone) .hostProvisioner(hostProvisioner) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java index c7e06676a2e..387a2cf5a4b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java @@ -35,7 +35,7 @@ public class HostRetirerTest { .build(), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); - MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), zone.cloud()); + MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors.getFlavors()); ProvisioningTester tester = new ProvisioningTester.Builder().hostProvisioner(hostProvisioner) .flavors(flavors.getFlavors()) .zone(zone) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index e0301d0c329..623ce47b611 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -1,16 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; -import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.Architecture; import com.yahoo.config.provision.NodeResources.DiskSpeed; @@ -23,18 +22,16 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Instant; +import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.Set; -import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -45,13 +42,6 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; /** * @author freva @@ -59,79 +49,63 @@ import static org.mockito.Mockito.when; */ public class DynamicProvisioningTest { - private static final Zone zone = new Zone( - Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(), - SystemName.main, - Environment.prod, - RegionName.from("us-east")); private final MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup(); - private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); - private final ProvisioningTester tester = new ProvisioningTester.Builder() - .zone(zone).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); @Test public void dynamically_provision_with_empty_node_repo() { + var tester = tester(true); assertEquals(0, tester.nodeRepository().nodes().list().size()); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = ProvisioningTester.applicationId("application1"); NodeResources resources = new NodeResources(1, 4, 10, 1); - - mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); - verify(hostProvisioner).provisionHosts(eq(List.of(100, 101, 102, 103)), eq(NodeType.host), eq(resources), eq(application1), - eq(Version.emptyVersion), eq(HostSharing.any), eq(Optional.of(ClusterSpec.Type.content)), eq(CloudAccount.empty), any()); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes assertEquals(8, tester.nodeRepository().nodes().list().size()); assertEquals(4, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); - assertEquals(Set.of("host-100-1", "host-101-1", "host-102-1", "host-103-1"), + assertEquals(Set.of("host100-1", "host101-1", "host102-1", "host103-1"), tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).hostnames()); // Deploy new application - ApplicationId application2 = ProvisioningTester.applicationId(); - prepareAndActivate(application2, clusterSpec("mycluster"), 4, 1, resources); + ApplicationId application2 = ProvisioningTester.applicationId("application2"); + prepareAndActivate(application2, clusterSpec("mycluster"), 4, 1, resources, tester); // Total of 12 nodes should now be in node-repo, 4 active hosts and 8 active nodes assertEquals(12, tester.nodeRepository().nodes().list().size()); assertEquals(4, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); - assertEquals(Set.of("host-100-1", "host-100-2", "host-101-1", "host-101-2", "host-102-1", "host-102-2", - "host-103-1", "host-103-2"), + assertEquals(Set.of("host100-1", "host100-2", "host101-1", "host101-2", "host102-1", "host102-2", "host103-1", "host103-2"), tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).hostnames()); // Deploy new exclusive application - ApplicationId application3 = ProvisioningTester.applicationId(); - mockHostProvisioner(hostProvisioner, "large", 3, application3); - prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, resources); - verify(hostProvisioner).provisionHosts(eq(List.of(104, 105, 106, 107)), eq(NodeType.host), eq(resources), eq(application3), - eq(Version.emptyVersion), eq(HostSharing.exclusive), eq(Optional.of(ClusterSpec.Type.content)), eq(CloudAccount.empty), any()); + ApplicationId application3 = ProvisioningTester.applicationId("application3"); + NodeResources exclusiveResources = new NodeResources(1, 10, 10, 1); + prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, exclusiveResources, tester); // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes assertEquals(20, tester.nodeRepository().nodes().list().size()); assertEquals(8, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); assertEquals(12, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); - - verifyNoMoreInteractions(hostProvisioner); } @Test public void in_place_resize_not_allowed_on_exclusive_to_hosts() { - NodeResources initialResources = new NodeResources(2, 8, 10, 1); - NodeResources smallResources = new NodeResources(1, 4, 10, 1); + var tester = tester(true); + + NodeResources initialResources = new NodeResources(4, 80, 100, 1); + NodeResources smallResources = new NodeResources(1, 20, 50, 1); ApplicationId application1 = ProvisioningTester.applicationId(); - mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, initialResources); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, initialResources, tester); ApplicationId application2 = ProvisioningTester.applicationId(); - mockHostProvisioner(hostProvisioner, "large", 3, application2); // Provision exclusive hosts - prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, initialResources); + prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, initialResources, tester); // Total of 16 nodes should now be in node-repo, 8 active hosts and 8 active nodes assertEquals(16, tester.nodeRepository().nodes().list().size()); assertEquals(8, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallResources); - prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, smallResources); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallResources, tester); + prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, smallResources, tester); // 24 nodes: 4 shared hosts with 4 app1 nodes + 8 exclusive hosts with 8 nodes of app2, 4 of which are retired NodeList nodes = tester.nodeRepository().nodes().list(); @@ -143,50 +117,51 @@ public class DynamicProvisioningTest { @Test public void avoids_allocating_to_empty_hosts() { + var tester = tester(false); tester.makeReadyHosts(6, new NodeResources(12, 12, 200, 12)); tester.activateTenantHosts(); NodeResources resources = new NodeResources(1, 4, 10, 4); ApplicationId application1 = ProvisioningTester.applicationId(); - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); ApplicationId application2 = ProvisioningTester.applicationId(); - prepareAndActivate(application2, clusterSpec("mycluster"), 3, 1, resources); + prepareAndActivate(application2, clusterSpec("mycluster"), 3, 1, resources, tester); ApplicationId application3 = ProvisioningTester.applicationId(); - prepareAndActivate(application3, clusterSpec("mycluster"), 3, 1, resources); + prepareAndActivate(application3, clusterSpec("mycluster"), 3, 1, resources, tester); assertEquals(4, tester.nodeRepository().nodes().list().nodeType(NodeType.tenant).stream().map(Node::parentHostname).distinct().count()); ApplicationId application4 = ProvisioningTester.applicationId(); - prepareAndActivate(application4, clusterSpec("mycluster"), 3, 1, resources); + prepareAndActivate(application4, clusterSpec("mycluster"), 3, 1, resources, tester); assertEquals(5, tester.nodeRepository().nodes().list().nodeType(NodeType.tenant).stream().map(Node::parentHostname).distinct().count()); } @Test public void retires_on_exclusivity_violation() { + var tester = tester(true); ApplicationId application1 = ProvisioningTester.applicationId(); - NodeResources resources = new NodeResources(1, 4, 10, 1); - - mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); + NodeResources resources = new NodeResources(4, 80, 100, 1); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); NodeList initialNodes = tester.nodeRepository().nodes().list().owner(application1); assertEquals(4, initialNodes.size()); // Redeploy same application with exclusive=true - mockHostProvisioner(hostProvisioner, "large", 3, application1); - prepareAndActivate(application1, clusterSpec("mycluster", true), 4, 1, resources); + NodeResources smallerExclusiveResources = new NodeResources(1, 20, 50, 1); + prepareAndActivate(application1, clusterSpec("mycluster", true), 4, 1, smallerExclusiveResources, tester); assertEquals(8, tester.nodeRepository().nodes().list().owner(application1).size()); assertEquals(initialNodes, tester.nodeRepository().nodes().list().owner(application1).retired()); // Redeploy without exclusive again is no-op - prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); + prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallerExclusiveResources, tester); assertEquals(8, tester.nodeRepository().nodes().list().owner(application1).size()); assertEquals(initialNodes, tester.nodeRepository().nodes().list().owner(application1).retired()); } @Test public void node_indices_are_unique_even_when_a_node_is_left_in_reserved_state() { + var tester = tester(false); NodeResources resources = new NodeResources(10, 10, 10, 10); ApplicationId app = ProvisioningTester.applicationId(); @@ -224,9 +199,9 @@ public class DynamicProvisioningTest { List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 17, 200, 10, fast, remote))); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) - .hostProvisioner(new MockHostProvisioner(flavors, memoryTax, zone.cloud())) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) .resourcesCalculator(memoryTax, 0) .build(); @@ -268,8 +243,8 @@ public class DynamicProvisioningTest { InMemoryFlagSource flagSource = new InMemoryFlagSource(); List<Flavor> flavors = List.of(new Flavor("x86", new NodeResources(1, 4, 50, 0.1, fast, local, Architecture.x86_64)), new Flavor("arm", new NodeResources(1, 4, 50, 0.1, fast, local, Architecture.arm64))); - MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors, zone.cloud()); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) .hostProvisioner(hostProvisioner) .resourcesCalculator(0, 0) @@ -312,9 +287,9 @@ public class DynamicProvisioningTest { new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote))); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) - .hostProvisioner(new MockHostProvisioner(flavors, memoryTax, zone.cloud())) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) .resourcesCalculator(memoryTax, 0) .build(); @@ -387,9 +362,9 @@ public class DynamicProvisioningTest { new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote)), new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, local))); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) - .hostProvisioner(new MockHostProvisioner(flavors, memoryTax, zone.cloud())) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) .resourcesCalculator(memoryTax, 0) .build(); @@ -422,9 +397,9 @@ public class DynamicProvisioningTest { new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote))); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) - .hostProvisioner(new MockHostProvisioner(flavors, memoryTax, zone.cloud())) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) .resourcesCalculator(memoryTax, localDiskTax) .build(); @@ -445,9 +420,9 @@ public class DynamicProvisioningTest { public void gpu_host() { List<Flavor> flavors = List.of(new Flavor("gpu", new NodeResources(4, 16, 125, 10, fast, local, Architecture.x86_64, new NodeResources.GpuResources(1, 16)))); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone(false)) .flavors(flavors) - .hostProvisioner(new MockHostProvisioner(flavors, zone.cloud())) + .hostProvisioner(new MockHostProvisioner(flavors)) .nameResolver(nameResolver) .build(); NodeResources resources = new NodeResources(4, 16, 125, 0.3, @@ -458,16 +433,43 @@ public class DynamicProvisioningTest { 2, 1, resources); } - private void prepareAndActivate(ApplicationId application, ClusterSpec clusterSpec, int nodes, int groups, NodeResources resources) { + private Zone zone(boolean sharing) { + return new Zone( + Cloud.builder().dynamicProvisioning(true).allowHostSharing(sharing).build(), + SystemName.main, + Environment.prod, + RegionName.from("us-east")); + } + + private ProvisioningTester tester(boolean sharing) { + var hostProvisioner = new MockHostProvisioner(new NodeFlavors(ProvisioningTester.createConfig()).getFlavors(), nameResolver, 0); + return new ProvisioningTester.Builder().zone(zone(sharing)).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); + } + + private void prepareAndActivate(ApplicationId application, ClusterSpec clusterSpec, int nodes, int groups, NodeResources resources, + ProvisioningTester tester) { List<HostSpec> prepared = tester.prepare(application, clusterSpec, nodes, groups, resources); NodeList provisionedHosts = tester.nodeRepository().nodes().list(Node.State.provisioned).nodeType(NodeType.host); if (!provisionedHosts.isEmpty()) { - tester.move(Node.State.ready, provisionedHosts.asList()); + List<Node> hosts = provisionedHosts.asList() + .stream() + .map(h -> ((MockHostProvisioner)tester.hostProvisioner()).withIpAssigned(h)) + .toList(); + tester.move(Node.State.ready, hosts); tester.activateTenantHosts(); } + assignIpAddresses(prepared, tester.nodeRepository()); tester.activate(application, prepared); } + private void assignIpAddresses(Collection<HostSpec> hosts, NodeRepository nodeRepository) { + for (var host : hosts) { + try (var nodeLock = nodeRepository.nodes().lockAndGetRequired(host.hostname())) { + var node = nodeLock.node(); + nodeRepository.nodes().write(node.with(node.ipConfig().withPrimary(nodeRepository.nameResolver().resolveAll(node.hostname()))), nodeLock); + } + } + } private static ClusterSpec clusterSpec(String clusterId) { return clusterSpec(clusterId, false); } @@ -482,39 +484,8 @@ public class DynamicProvisioningTest { } private static ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk, - DiskSpeed diskSpeed, StorageType storageType) { + DiskSpeed diskSpeed, StorageType storageType) { return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1, diskSpeed, storageType)); } - private void mockHostProvisioner(HostProvisioner hostProvisioner, String hostFlavorName, int numIps, ApplicationId exclusiveTo) { - doAnswer(invocation -> { - Flavor hostFlavor = tester.nodeRepository().flavors().getFlavorOrThrow(hostFlavorName); - List<Integer> provisionIndexes = invocation.getArgument(0); - NodeResources nodeResources = invocation.getArgument(2); - Consumer<List<ProvisionedHost>> provisionedHostConsumer = invocation.getArgument(8); - - List<ProvisionedHost> provisionedHosts = provisionIndexes.stream() - .map(hostIndex -> { - String hostHostname = "host-" + hostIndex; - String hostIp = "::" + hostIndex + ":0"; - nameResolver.addRecord(hostHostname, hostIp); - Set<String> pool = IntStream.range(1, numIps + 1).mapToObj(i -> { - String ip = "::" + hostIndex + ":" + i; - nameResolver.addRecord(hostHostname + "-" + i, ip); - return ip; - }).collect(Collectors.toSet()); - - Node parent = Node.create(hostHostname, new IP.Config(Set.of(hostIp), pool), hostHostname, hostFlavor, NodeType.host) - .exclusiveToApplicationId(exclusiveTo).build(); - Node child = Node.reserve(Set.of("::" + hostIndex + ":1"), hostHostname + "-1", hostHostname, nodeResources, NodeType.tenant).build(); - ProvisionedHost provisionedHost = mock(ProvisionedHost.class); - when(provisionedHost.generateHost()).thenReturn(parent); - when(provisionedHost.generateNode()).thenReturn(child); - return provisionedHost; - }).toList(); - provisionedHostConsumer.accept(provisionedHosts); - return null; - }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any(), any(), any(), any()); - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 5679201c77b..a19f986a177 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -161,8 +161,7 @@ public class LoadBalancerProvisionerTest { clusterRequest(ClusterSpec.Type.container, containerCluster), clusterRequest(ClusterSpec.Type.content, contentCluster))); List<LoadBalancer> activeLoadBalancers = lbApp1.get().stream() - .filter(lb -> lb.state() == LoadBalancer.State.active) - .collect(Collectors.toList()); + .filter(lb -> lb.state() == LoadBalancer.State.active).toList(); assertEquals(1, activeLoadBalancers.size()); assertEquals(Set.of(), activeLoadBalancers.get(0).instance().get().reals()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 2ea8d95bc9d..bf15e4bbe1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -80,6 +80,7 @@ public class ProvisioningTester { private final NodeFlavors nodeFlavors; private final ManualClock clock; private final NodeRepository nodeRepository; + private final HostProvisioner hostProvisioner; private final NodeRepositoryProvisioner provisioner; private final CapacityPolicies capacityPolicies; private final ProvisionLogger provisionLogger; @@ -102,6 +103,7 @@ public class ProvisioningTester { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); + this.hostProvisioner = hostProvisioner; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner, resourcesCalculator); this.nodeRepository = new NodeRepository(nodeFlavors, provisionServiceProvider, @@ -144,6 +146,7 @@ public class ProvisioningTester { public Orchestrator orchestrator() { return nodeRepository.orchestrator(); } public ManualClock clock() { return clock; } public NodeRepositoryProvisioner provisioner() { return provisioner; } + public HostProvisioner hostProvisioner() { return hostProvisioner; } public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return nodeRepository.nodes().list(inState).owner(id); } |