From 300fc3045f97eb9287160d0b7d973d38ccb8df0b Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Tue, 7 Nov 2023 09:27:35 +0100 Subject: Revert "Evaluate flags referenced multiple times at the start of prepare" --- .../com/yahoo/vespa/flags/custom/SharedHost.java | 3 - .../vespa/hosted/provision/NodeRepository.java | 27 ++++++- .../provision/autoscale/AllocatableResources.java | 31 ++++---- .../provision/autoscale/AllocationOptimizer.java | 15 ++-- .../hosted/provision/autoscale/Autoscaler.java | 12 +-- .../hosted/provision/autoscale/ClusterModel.java | 50 +++++++----- .../vespa/hosted/provision/autoscale/Limits.java | 9 +-- .../maintenance/HostCapacityMaintainer.java | 92 ++++++++++++---------- .../provision/maintenance/HostFlavorUpgrader.java | 6 +- .../provision/provisioning/AllocationParams.java | 89 --------------------- .../provision/provisioning/CapacityPolicies.java | 46 ++++++----- .../provision/provisioning/HostProvisioner.java | 6 +- .../provision/provisioning/NodeAllocation.java | 16 ++-- .../provision/provisioning/NodeCandidate.java | 23 +++--- .../provision/provisioning/NodePrioritizer.java | 40 ++++++---- .../provisioning/NodeRepositoryProvisioner.java | 70 ++++++++-------- .../provision/provisioning/NodeResourceLimits.java | 26 +++--- .../hosted/provision/provisioning/Preparer.java | 68 +++++++++------- .../provision/restapi/ApplicationSerializer.java | 11 +-- .../provision/restapi/NodesV2ApiHandler.java | 2 - .../provision/testutils/MockHostProvisioner.java | 9 +-- .../provision/autoscale/AutoscalingTest.java | 2 +- .../provision/autoscale/ClusterModelTest.java | 8 +- .../vespa/hosted/provision/autoscale/Fixture.java | 6 +- .../provisioning/DynamicProvisioningTester.java | 2 +- 25 files changed, 302 insertions(+), 367 deletions(-) delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationParams.java diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java index 0c0a510d96e..66356d979a4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java @@ -36,9 +36,6 @@ public class SharedHost { this.resources = resourcesOrNull == null ? List.of() : List.copyOf(resourcesOrNull); } - @JsonIgnore - public SharedHost() { this(null); } - @JsonGetter("resources") public List getResourcesOrNull() { return resources.isEmpty() ? null : resources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 25919d6a81d..dfbe41e31d7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -12,6 +12,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.JacksonFlag; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; @@ -26,7 +29,6 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.JobControlFlags; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -65,6 +67,7 @@ public class NodeRepository extends AbstractComponent { private final MetricsDb metricsDb; private final Orchestrator orchestrator; private final int spareCount; + private final JacksonFlag sharedHosts; /** * Creates a node repository from a zookeeper provider. @@ -138,6 +141,7 @@ public class NodeRepository extends AbstractComponent { this.metricsDb = metricsDb; this.orchestrator = orchestrator; this.spareCount = spareCount; + this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(flagSource()); nodes.rewrite(); } @@ -197,6 +201,27 @@ public class NodeRepository extends AbstractComponent { /** The number of nodes we should ensure has free capacity for node failures whenever possible */ public int spareCount() { return spareCount; } + /** Returns whether nodes must be allocated to hosts that are exclusive to the cluster type. */ + public boolean exclusiveClusterType(ClusterSpec cluster) { + return sharedHosts.value().hasClusterType(cluster.type().name()); + } + + /** + * Returns whether nodes are allocated exclusively in this instance given this cluster spec. + * Exclusive allocation requires that the wanted node resources matches the advertised resources of the node + * perfectly. + */ + public boolean exclusiveAllocation(ClusterSpec clusterSpec) { + return clusterSpec.isExclusive() || + ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || + ( !zone().cloud().allowHostSharing() && !sharedHosts.value().supportsClusterType(clusterSpec.type().name())); + } + + /** Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ + public boolean exclusiveProvisioning(ClusterSpec clusterSpec) { + return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + } + /** * Returns ACLs for the children of the given host. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 4f116630af8..544436dc902 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.time.Duration; import java.util.List; @@ -34,12 +33,14 @@ public class AllocatableResources { private final double fulfilment; /** Fake allocatable resources from requested capacity */ - public AllocatableResources(AllocationParams params, ClusterResources requested) { + public AllocatableResources(ClusterResources requested, + ClusterSpec clusterSpec, + NodeRepository nodeRepository) { this.nodes = requested.nodes(); this.groups = requested.groups(); - this.realResources = params.nodeRepository().resourcesCalculator().requestToReal(requested.nodeResources(), params.exclusiveAllocation(), false); + this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec), false); this.advertisedResources = requested.nodeResources(); - this.clusterSpec = params.cluster(); + this.clusterSpec = clusterSpec; this.fulfilment = 1; } @@ -166,8 +167,7 @@ public class AllocatableResources { .withBandwidthGbps(sum.bandwidthGbps() / nodes.size()); } - public static Optional from(AllocationParams params, - ClusterResources wantedResources, + public static Optional from(ClusterResources wantedResources, ApplicationId applicationId, ClusterSpec clusterSpec, Limits applicationLimits, @@ -175,11 +175,10 @@ public class AllocatableResources { ClusterModel model, NodeRepository nodeRepository) { var systemLimits = nodeRepository.nodeResourceLimits(); - boolean exclusive = params.exclusiveAllocation(); + boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); if (! exclusive) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources - var allocatableResources = calculateAllocatableResources(params, - wantedResources, + var allocatableResources = calculateAllocatableResources(wantedResources, nodeRepository, applicationId, clusterSpec, @@ -190,9 +189,8 @@ public class AllocatableResources { var worstCaseRealResources = nodeRepository.resourcesCalculator().requestToReal(allocatableResources.advertisedResources, exclusive, false); - if ( ! systemLimits.isWithinRealLimits(params, worstCaseRealResources, applicationId, clusterSpec)) { - allocatableResources = calculateAllocatableResources(params, - wantedResources, + if ( ! systemLimits.isWithinRealLimits(worstCaseRealResources, applicationId, clusterSpec)) { + allocatableResources = calculateAllocatableResources(wantedResources, nodeRepository, applicationId, clusterSpec, @@ -201,7 +199,7 @@ public class AllocatableResources { false); } - if ( ! systemLimits.isWithinRealLimits(params, allocatableResources.realResources, applicationId, clusterSpec)) + if ( ! systemLimits.isWithinRealLimits(allocatableResources.realResources, applicationId, clusterSpec)) return Optional.empty(); if ( ! anySatisfies(allocatableResources.realResources, availableRealHostResources)) return Optional.empty(); @@ -230,7 +228,7 @@ public class AllocatableResources { } if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; - if ( ! systemLimits.isWithinRealLimits(params, realResources, applicationId, clusterSpec)) continue; + if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec)) continue; var candidate = new AllocatableResources(wantedResources.with(realResources), advertisedResources, @@ -253,8 +251,7 @@ public class AllocatableResources { } } - private static AllocatableResources calculateAllocatableResources(AllocationParams params, - ClusterResources wantedResources, + private static AllocatableResources calculateAllocatableResources(ClusterResources wantedResources, NodeRepository nodeRepository, ApplicationId applicationId, ClusterSpec clusterSpec, @@ -266,7 +263,7 @@ public class AllocatableResources { advertisedResources = systemLimits.enlargeToLegal(advertisedResources, applicationId, clusterSpec, exclusive, true); // Ask for something legal advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); // What we'll really get - if ( ! systemLimits.isWithinRealLimits(params, realResources, applicationId, clusterSpec) + if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec) && advertisedResources.storageType() == NodeResources.StorageType.any) { // Since local disk reserves some of the storage, try to constrain to remote disk advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index f8cfcaa1ce3..ff30f9d6163 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.util.Optional; @@ -35,8 +34,7 @@ public class AllocationOptimizer { * @return the best allocation, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits */ - public Optional findBestAllocation(AllocationParams params, - Load loadAdjustment, + public Optional findBestAllocation(Load loadAdjustment, ClusterModel model, Limits limits) { if (limits.isEmpty()) @@ -44,12 +42,12 @@ public class AllocationOptimizer { new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), IntRange.empty()); else - limits = atLeast(minimumNodes, limits).fullySpecified(params); + limits = atLeast(minimumNodes, limits).fullySpecified(model.current().clusterSpec(), nodeRepository, model.application().id()); Optional bestAllocation = Optional.empty(); var availableRealHostResources = nodeRepository.zone().cloud().dynamicProvisioning() ? nodeRepository.flavors().getFlavors().stream().map(flavor -> flavor.resources()).toList() : nodeRepository.nodes().list().hosts().stream().map(host -> host.flavor().resources()) - .map(hostResources -> maxResourcesOf(params, hostResources, model)) + .map(hostResources -> maxResourcesOf(hostResources, model)) .toList(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { @@ -59,8 +57,7 @@ public class AllocationOptimizer { groups, nodeResourcesWith(nodes, groups, limits, loadAdjustment, model)); - var allocatableResources = AllocatableResources.from(params, - resources, + var allocatableResources = AllocatableResources.from(resources, model.application().id(), model.current().clusterSpec(), limits, @@ -76,8 +73,8 @@ public class AllocationOptimizer { } /** Returns the max resources of a host one node may allocate. */ - private NodeResources maxResourcesOf(AllocationParams params, NodeResources hostResources, ClusterModel model) { - if (params.exclusiveAllocation()) return hostResources; + private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel model) { + if (nodeRepository.exclusiveAllocation(model.clusterSpec())) return hostResources; // static, shared hosts: Allocate at most half of the host cpu to simplify management return hostResources.withVcpu(hostResources.vcpu() / 2); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 59dfc167d6a..738abddc31a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling.Status; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.time.Duration; @@ -56,14 +54,12 @@ public class Autoscaler { } private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { - NodeList notRetired = clusterNodes.not().retired(); - ClusterSpec clusterSpec = notRetired.clusterSpec(); - AllocationParams params = AllocationParams.from(nodeRepository, application.id(), clusterSpec, clusterNodes.clusterSpec().vespaVersion()); - var model = new ClusterModel(params, + var model = new ClusterModel(nodeRepository, application, + clusterNodes.not().retired().clusterSpec(), cluster, clusterNodes, - new AllocatableResources(notRetired, nodeRepository), + new AllocatableResources(clusterNodes.not().retired(), nodeRepository), nodeRepository.metricsDb(), nodeRepository.clock()); if (model.isEmpty()) return Autoscaling.empty(); @@ -77,7 +73,7 @@ public class Autoscaler { var loadAdjustment = model.loadAdjustment(); // Ensure we only scale down if we'll have enough headroom to not scale up again given a small load increase - var target = allocationOptimizer.findBestAllocation(params, loadAdjustment, model, limits); + var target = allocationOptimizer.findBestAllocation(loadAdjustment, model, limits); if (target.isEmpty()) return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", model); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index de132d53d63..4c5ace3d51a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.time.Clock; import java.time.Duration; @@ -47,8 +46,9 @@ public class ClusterModel { // TODO: Measure this, and only take it into account with queries private static final double fixedCpuCostFraction = 0.1; - private final AllocationParams params; + private final NodeRepository nodeRepository; private final Application application; + private final ClusterSpec clusterSpec; private final Cluster cluster; private final AllocatableResources current; @@ -74,28 +74,31 @@ public class ClusterModel { private Double maxQueryGrowthRate = null; private OptionalDouble averageQueryRate = null; - public ClusterModel(AllocationParams params, + public ClusterModel(NodeRepository nodeRepository, Application application, + ClusterSpec clusterSpec, Cluster cluster, NodeList clusterNodes, AllocatableResources current, MetricsDb metricsDb, Clock clock) { - this.params = params; + this.nodeRepository = nodeRepository; this.application = application; + this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = clusterNodes; this.current = current; this.clock = clock; - this.scalingDuration = cluster.scalingDuration(params.cluster()); - this.allocationDuration = cluster.allocationDuration(params.cluster()); + this.scalingDuration = cluster.scalingDuration(clusterSpec); + this.allocationDuration = cluster.allocationDuration(clusterSpec); this.clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id()); this.nodeTimeseries = new ClusterNodesTimeseries(scalingDuration(), cluster, nodes, metricsDb); this.at = clock.instant(); } - ClusterModel(AllocationParams params, + ClusterModel(NodeRepository nodeRepository, Application application, + ClusterSpec clusterSpec, Cluster cluster, AllocatableResources current, Clock clock, @@ -103,8 +106,9 @@ public class ClusterModel { Duration allocationDuration, ClusterTimeseries clusterTimeseries, ClusterNodesTimeseries nodeTimeseries) { - this.params = params; + this.nodeRepository = nodeRepository; this.application = application; + this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = NodeList.of(); this.current = current; @@ -118,7 +122,7 @@ public class ClusterModel { } public Application application() { return application; } - public ClusterSpec clusterSpec() { return params.cluster(); } + public ClusterSpec clusterSpec() { return clusterSpec; } public AllocatableResources current() { return current; } private ClusterNodesTimeseries nodeTimeseries() { return nodeTimeseries; } private ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } @@ -140,7 +144,7 @@ public class ClusterModel { public Duration allocationDuration() { return allocationDuration; } public boolean isContent() { - return params.cluster().type().isContent(); + return clusterSpec.type().isContent(); } /** Returns the predicted duration of data redistribution in this cluster. */ @@ -165,7 +169,7 @@ public class ClusterModel { } public boolean isExclusive() { - return params.exclusiveAllocation(); + return nodeRepository.exclusiveAllocation(clusterSpec); } /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ @@ -273,7 +277,7 @@ public class ClusterModel { * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); double neededTotalVcpuPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + ( 1 - cpu.queryFraction()) * cpu.idealLoad() * - (params.cluster().type().isContainer() ? 1 : groupSize()); + (clusterSpec.type().isContainer() ? 1 : groupSize()); // Max 1: Only use bcp group info if it indicates that we need to scale *up* double cpuAdjustment = Math.max(1.0, neededTotalVcpuPerGroup / currentClusterTotalVcpuPerGroup); return ideal.withCpu(ideal.cpu() / cpuAdjustment); @@ -337,7 +341,7 @@ public class ClusterModel { /** Returns the headroom for growth during organic traffic growth as a multiple of current resources. */ private double growthRateHeadroom() { - if ( ! params.nodeRepository().zone().environment().isProduction()) return 1; + if ( ! nodeRepository.zone().environment().isProduction()) return 1; double growthRateHeadroom = 1 + maxQueryGrowthRate() * scalingDuration().toMinutes(); // Cap headroom at 10% above the historical observed peak if (queryFractionOfMax() != 0) @@ -351,7 +355,7 @@ public class ClusterModel { * as a multiple of current resources. */ private double trafficShiftHeadroom() { - if ( ! params.nodeRepository().zone().environment().isProduction()) return 1; + if ( ! nodeRepository.zone().environment().isProduction()) return 1; if (canRescaleWithinBcpDeadline()) return 1; double trafficShiftHeadroom; if (application.status().maxReadShare() == 0) // No traffic fraction data @@ -387,7 +391,7 @@ public class ClusterModel { OptionalDouble costPerQuery() { if (averageQueryRate().isEmpty() || averageQueryRate().getAsDouble() == 0.0) return OptionalDouble.empty(); // TODO: Query rate should generally be sampled at the time where we see the peak resource usage - int fanOut = params.cluster().type().isContainer() ? 1 : groupSize(); + int fanOut = clusterSpec.type().isContainer() ? 1 : groupSize(); return OptionalDouble.of(peakLoad().cpu() * cpu.queryFraction() * fanOut * nodes.not().retired().first().get().resources().vcpu() / averageQueryRate().getAsDouble() / groupCount()); } @@ -410,8 +414,8 @@ public class ClusterModel { private class MemoryModel { double idealLoad() { - if (params.cluster().type().isContainer()) return idealContainerMemoryLoad; - if (params.cluster().type() == ClusterSpec.Type.admin) return idealContainerMemoryLoad; // Not autoscaled, but ideal shown in console + if (clusterSpec.type().isContainer()) return idealContainerMemoryLoad; + if (clusterSpec.type() == ClusterSpec.Type.admin) return idealContainerMemoryLoad; // Not autoscaled, but ideal shown in console return idealContentMemoryLoad; } @@ -428,12 +432,16 @@ public class ClusterModel { double averageReal() { if (nodes.isEmpty()) { // we're estimating - var initialResources = new CapacityPolicies(params.nodeRepository()).specifyFully(params, cluster.minResources().nodeResources()); - return params.nodeRepository().resourcesCalculator().requestToReal(initialResources, params.exclusiveAllocation(), false).memoryGb(); + var initialResources = new CapacityPolicies(nodeRepository).specifyFully(cluster.minResources().nodeResources(), + clusterSpec, + application.id()); + return nodeRepository.resourcesCalculator().requestToReal(initialResources, + nodeRepository.exclusiveAllocation(clusterSpec), + false).memoryGb(); } else { return nodes.stream() - .mapToDouble(node -> params.nodeRepository().resourcesCalculator().realResourcesOf(node, params.nodeRepository()).memoryGb()) + .mapToDouble(node -> nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository).memoryGb()) .average() .getAsDouble(); } @@ -446,7 +454,7 @@ public class ClusterModel { double idealLoad() { // Stateless clusters are not expected to consume more disk over time - // if they do it is due to logs which will be rotated away right before the disk is full - return params.cluster().isStateful() ? idealContentDiskLoad : idealContainerDiskLoad; + return clusterSpec.isStateful() ? idealContentDiskLoad : idealContainerDiskLoad; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 827744a6e56..ab93e585c88 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.util.Objects; @@ -66,12 +65,12 @@ public class Limits { return resources; } - public Limits fullySpecified(AllocationParams params) { + public Limits fullySpecified(ClusterSpec clusterSpec, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var capacityPolicies = new CapacityPolicies(params.nodeRepository()); - return new Limits(capacityPolicies.specifyFully(params, min), - capacityPolicies.specifyFully(params, max), + var capacityPolicies = new CapacityPolicies(nodeRepository); + return new Limits(capacityPolicies.specifyFully(min, clusterSpec, applicationId), + capacityPolicies.specifyFully(max, clusterSpec, applicationId), groupSize); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 156be954c61..108f8d77837 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -13,7 +13,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; @@ -25,7 +28,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisionRequest; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; @@ -45,7 +47,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.hosted.provision.provisioning.NodeCandidate.ExclusivityViolation.YES; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.stream.Collectors.groupingBy; @@ -61,6 +62,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { private final HostProvisioner hostProvisioner; private final ListFlag preprovisionCapacityFlag; + private final BooleanFlag makeExclusiveFlag; private final ProvisioningThrottler throttler; HostCapacityMaintainer(NodeRepository nodeRepository, @@ -71,6 +73,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { super(nodeRepository, interval, metric); this.hostProvisioner = hostProvisioner; this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(flagSource); this.throttler = new ProvisioningThrottler(nodeRepository, metric); } @@ -156,13 +159,12 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } private List provision(NodeList nodeList) { - return provisionUntilNoDeficit(nodeList) - .stream() - .sorted(comparing(node -> node.history().events().stream() - .map(History.Event::at) - .min(naturalOrder()) - .orElse(Instant.MIN))) - .toList(); + return provisionUntilNoDeficit(nodeList).stream() + .sorted(comparing(node -> node.history().events().stream() + .map(History.Event::at) + .min(naturalOrder()) + .orElse(Instant.MIN))) + .toList(); } private static boolean canRemoveHost(Node host) { @@ -189,9 +191,12 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { * should be sufficient (avoid no stack trace). */ private List provisionUntilNoDeficit(NodeList nodeList) { - // Ensure flags for allocation params are evaluated before provision loop. Use a dummy cluster spec - var params = AllocationParams.from(nodeRepository(), ApplicationId.defaultId(), asClusterSpec(Optional.empty(), 0), Vtag.currentVersion); List preprovisionCapacity = preprovisionCapacityFlag.value(); + ApplicationId application = ApplicationId.defaultId(); + boolean makeExclusive = makeExclusiveFlag.with(FetchVector.Dimension.TENANT_ID, application.tenant().value()) + .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) + .with(FetchVector.Dimension.VESPA_VERSION, Vtag.currentVersion.toFullString()) + .value(); // Worst-case each ClusterCapacity in preprovisionCapacity will require an allocation. int maxProvisions = preprovisionCapacity.size(); @@ -199,7 +204,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { var nodesPlusProvisioned = new ArrayList<>(nodeList.asList()); for (int numProvisions = 0;; ++numProvisions) { var nodesPlusProvisionedPlusAllocated = new ArrayList<>(nodesPlusProvisioned); - Optional deficit = allocatePreprovisionCapacity(params, preprovisionCapacity, nodesPlusProvisionedPlusAllocated); + Optional deficit = allocatePreprovisionCapacity(application, preprovisionCapacity, nodesPlusProvisionedPlusAllocated, makeExclusive); if (deficit.isEmpty()) { return nodesPlusProvisionedPlusAllocated; } @@ -209,22 +214,21 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } ClusterCapacity clusterCapacityDeficit = deficit.get(); - nodesPlusProvisioned.addAll(provisionHosts(params, - clusterCapacityDeficit.count(), + nodesPlusProvisioned.addAll(provisionHosts(clusterCapacityDeficit.count(), toNodeResources(clusterCapacityDeficit), Optional.ofNullable(clusterCapacityDeficit.clusterType()), nodeList)); } } - private List provisionHosts(AllocationParams params, int count, NodeResources nodeResources, Optional clusterType, NodeList allNodes) { + private List provisionHosts(int count, NodeResources nodeResources, Optional clusterType, NodeList allNodes) { try { if (throttler.throttle(allNodes, Agent.HostCapacityMaintainer)) { throw new NodeAllocationException("Host provisioning is being throttled", true); } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List provisionIndices = nodeRepository().database().readProvisionIndices(count); - HostSharing sharingMode = params.exclusiveAllocation() ? HostSharing.exclusive : HostSharing.shared; + HostSharing sharingMode = nodeRepository().exclusiveAllocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, sharingMode, clusterType.map(ClusterSpec.Type::valueOf), Optional.empty(), @@ -232,16 +236,15 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { List hosts = new ArrayList<>(); Runnable waiter; try (var lock = nodeRepository().nodes().lockUnallocated()) { - waiter = hostProvisioner.provisionHosts(params.sharedHost(), - request, - resources -> true, - provisionedHosts -> { - hosts.addAll(provisionedHosts.stream() - .map(host -> host.generateHost(Duration.ZERO)) - .map(host -> host.withExclusiveToApplicationId(null)) - .toList()); - nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); - }); + waiter = hostProvisioner.provisionHosts(request, + resources -> true, + provisionedHosts -> { + hosts.addAll(provisionedHosts.stream() + .map(host -> host.generateHost(Duration.ZERO)) + .map(host -> host.withExclusiveToApplicationId(null)) + .toList()); + nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); + }); } waiter.run(); return hosts; @@ -260,18 +263,14 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { * they are added to {@code mutableNodes} * @return the part of a cluster capacity it was unable to allocate, if any */ - private Optional allocatePreprovisionCapacity(AllocationParams params, + private Optional allocatePreprovisionCapacity(ApplicationId application, List preprovisionCapacity, - ArrayList mutableNodes) { + ArrayList mutableNodes, + boolean makeExclusive) { for (int clusterIndex = 0; clusterIndex < preprovisionCapacity.size(); ++clusterIndex) { ClusterCapacity clusterCapacity = preprovisionCapacity.get(clusterIndex); - - params = params.with(asClusterSpec(Optional.ofNullable(clusterCapacity.clusterType()), clusterIndex)); - if (params.exclusiveProvisioning()) - throw new IllegalStateException("Preprovision cluster requires exclusive provisioning: " + clusterCapacity); - LockedNodeList allNodes = new LockedNodeList(mutableNodes, () -> {}); - List candidates = findCandidates(params, clusterCapacity, allNodes); + List candidates = findCandidates(application, clusterCapacity, clusterIndex, allNodes, makeExclusive); int deficit = Math.max(0, clusterCapacity.count() - candidates.size()); if (deficit > 0) { return Optional.of(clusterCapacity.withCount(deficit)); @@ -284,34 +283,45 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { return Optional.empty(); } - private List findCandidates(AllocationParams params, ClusterCapacity clusterCapacity, LockedNodeList allNodes) { + private List findCandidates(ApplicationId application, ClusterCapacity clusterCapacity, int clusterIndex, LockedNodeList allNodes, boolean makeExclusive) { NodeResources nodeResources = toNodeResources(clusterCapacity); // We'll allocate each ClusterCapacity as a unique cluster in a dummy application + ClusterSpec cluster = asSpec(Optional.ofNullable(clusterCapacity.clusterType()), clusterIndex); NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), 1, nodeResources, false, true, nodeRepository().zone().cloud().account(), Duration.ZERO); var allocationContext = IP.Allocation.Context.from(nodeRepository().zone().cloud().name(), nodeSpec.cloudAccount().isExclave(nodeRepository().zone()), nodeRepository().nameResolver()); - NodePrioritizer prioritizer = new NodePrioritizer(params, allNodes, nodeSpec, true, false, allocationContext, nodeRepository().nodes(), - nodeRepository().resourcesCalculator(), nodeRepository().spareCount()); + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, nodeSpec, + true, false, allocationContext, nodeRepository().nodes(), + nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), + nodeRepository().exclusiveAllocation(cluster), makeExclusive); List nodeCandidates = prioritizer.collect() .stream() - .filter(node -> node.violatesExclusivity(params, allNodes) != YES) + .filter(node -> node.violatesExclusivity(cluster, + application, + nodeRepository().exclusiveClusterType(cluster), + nodeRepository().exclusiveAllocation(cluster), + false, + nodeRepository().zone().cloud().allowHostSharing(), + allNodes, + makeExclusive) + != NodeCandidate.ExclusivityViolation.YES) .toList(); MutableInteger index = new MutableInteger(0); return nodeCandidates .stream() .limit(clusterCapacity.count()) .map(candidate -> candidate.toNode() - .allocate(params.application(), - ClusterMembership.from(params.cluster(), index.next()), + .allocate(application, + ClusterMembership.from(cluster, index.next()), nodeResources, nodeRepository().clock().instant())) .toList(); } - private static ClusterSpec asClusterSpec(Optional clusterType, int index) { + private static ClusterSpec asSpec(Optional clusterType, int index) { return ClusterSpec.request(clusterType.map(ClusterSpec.Type::from).orElse(ClusterSpec.Type.content), ClusterSpec.Id.from(String.valueOf(index))) .vespaVersion(Vtag.currentVersion) // Needed, but should not be used here. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java index e7b86c32618..b6897d5b1c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; @@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.time.Duration; import java.util.HashSet; @@ -73,9 +71,7 @@ public class HostFlavorUpgrader extends NodeRepositoryMaintainer { if (parent.isEmpty()) continue; if (exhaustedFlavors.contains(parent.get().flavor().name())) continue; Allocation allocation = node.allocation().get(); - ClusterSpec cluster = allocation.membership().cluster(); - var params = AllocationParams.from(nodeRepository(), allocation.owner(), cluster, cluster.vespaVersion()); - Predicate realHostResourcesWithinLimits = resources -> nodeRepository().nodeResourceLimits().isWithinRealLimits(params, resources, allocation.owner(), cluster); + Predicate realHostResourcesWithinLimits = resources -> nodeRepository().nodeResourceLimits().isWithinRealLimits(resources, allocation.owner(), allocation.membership().cluster()); if (!hostProvisioner.canUpgradeFlavor(parent.get(), node, realHostResourcesWithinLimits)) continue; if (parent.get().status().wantToUpgradeFlavor() && allocation.membership().retired()) continue; // Already upgrading diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationParams.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationParams.java deleted file mode 100644 index 59a1f7b025a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationParams.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.custom.SharedHost; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import static java.util.Objects.requireNonNull; - -/** - * Miscellaneous parameters for the preparation of an allocation of a cluster. - * - *

Ideal for feature flags that guards new code paths in various parts of the allocation code.

- * - * @param exclusiveClusterType whether nodes must be allocated to hosts that are exclusive to the cluster type - * @param exclusiveAllocation whether nodes are allocated exclusively in this instance given this cluster spec. - * Exclusive allocation requires that the wanted node resources matches the advertised - * resources of the node perfectly - * @param exclusiveProvisioning Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application - * @param sharedHost snapshot of shared-host flag - * @param makeExclusive snapshot of make-exclusive flag - * @author hakonhall - */ -public record AllocationParams(NodeRepository nodeRepository, - ApplicationId application, - ClusterSpec cluster, - boolean exclusiveClusterType, - boolean exclusiveAllocation, - boolean exclusiveProvisioning, - SharedHost sharedHost, - boolean makeExclusive) { - - public AllocationParams { - requireNonNull(nodeRepository, "nodeRepository cannot be null"); - requireNonNull(application, "application cannot be null"); - requireNonNull(cluster, "cluster cannot be null"); - requireNonNull(sharedHost, "sharedHost cannot be null"); - } - - /** The canonical way of constructing an instance: ensures consistencies between the various parameters. */ - public static AllocationParams from(NodeRepository nodeRepository, ApplicationId application, ClusterSpec cluster, Version version) { - return from(nodeRepository, - application, - cluster, - PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()).value(), - Flags.MAKE_EXCLUSIVE.bindTo(nodeRepository.flagSource()) - .with(FetchVector.Dimension.TENANT_ID, application.tenant().value()) - .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) - .with(FetchVector.Dimension.VESPA_VERSION, version.toFullString()) - .value()); - } - - /** - * Returns the same allocation parameters, but as-if it was built with the given cluster. Flags are NOT re-evaluated, - * but exclusivity may change. - */ - public AllocationParams with(ClusterSpec cluster) { return from(nodeRepository, application, cluster, sharedHost, makeExclusive); } - - private static AllocationParams from(NodeRepository nodeRepository, ApplicationId application, ClusterSpec cluster, SharedHost sharedHost, boolean makeExclusive) { - return new AllocationParams(nodeRepository, - application, - cluster, - exclusiveClusterType(cluster, sharedHost), - exclusiveAllocation(nodeRepository.zone(), cluster, sharedHost), - exclusiveProvisioning(nodeRepository.zone(), cluster), - sharedHost, - makeExclusive); - } - - private static boolean exclusiveClusterType(ClusterSpec cluster, SharedHost sharedHost) { - return sharedHost.hasClusterType(cluster.type().name()); - } - - private static boolean exclusiveAllocation(Zone zone, ClusterSpec cluster, SharedHost sharedHost) { - return cluster.isExclusive() || - ( cluster.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || - ( !zone.cloud().allowHostSharing() && !sharedHost.supportsClusterType(cluster.type().name())); - } - - private static boolean exclusiveProvisioning(Zone zone, ClusterSpec clusterSpec) { - return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 1d1be35aee8..1e9adea4e95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; @@ -87,47 +86,47 @@ public class CapacityPolicies { return target; } - public ClusterResources specifyFully(AllocationParams params, ClusterResources resources) { - return resources.with(specifyFully(params, resources.nodeResources())); + public ClusterResources specifyFully(ClusterResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { + return resources.with(specifyFully(resources.nodeResources(), clusterSpec, applicationId)); } - public NodeResources specifyFully(AllocationParams params, NodeResources resources) { - NodeResources amended = resources.withUnspecifiedFieldsFrom(defaultResources(params).with(DiskSpeed.any)); + public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { + NodeResources amended = resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); // TODO jonmv: remove this after all apps are 8.248.8 or above; architecture for admin nodes was not picked up before this. - if (params.cluster().vespaVersion().isBefore(Version.fromString("8.248.8"))) amended = amended.with(resources.architecture()); + if (clusterSpec.vespaVersion().isBefore(Version.fromString("8.248.8"))) amended = amended.with(resources.architecture()); return amended; } - private NodeResources defaultResources(AllocationParams params) { - if (params.cluster().type() == ClusterSpec.Type.admin) { - Architecture architecture = adminClusterArchitecture(params.application()); + private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { + if (clusterSpec.type() == ClusterSpec.Type.admin) { + Architecture architecture = adminClusterArchitecture(applicationId); - if (params.exclusiveAllocation()) { + if (nodeRepository.exclusiveAllocation(clusterSpec)) { return smallestExclusiveResources().with(architecture); } - if (params.cluster().id().value().equals("cluster-controllers")) { - return clusterControllerResources(params.cluster(), architecture).with(architecture); + if (clusterSpec.id().value().equals("cluster-controllers")) { + return clusterControllerResources(clusterSpec, architecture).with(architecture); } - if (params.cluster().id().value().equals("logserver")) { + if (clusterSpec.id().value().equals("logserver")) { return logserverResources(architecture).with(architecture); } - return versioned(params.cluster(), Map.of(new Version(0), smallestSharedResources())).with(architecture); + return versioned(clusterSpec, Map.of(new Version(0), smallestSharedResources())).with(architecture); } - if (params.cluster().type() == ClusterSpec.Type.content) { + if (clusterSpec.type() == ClusterSpec.Type.content) { // When changing defaults here update cloud.vespa.ai/en/reference/services return zone.cloud().dynamicProvisioning() - ? versioned(params.cluster(), Map.of(new Version(0), new NodeResources(2, 16, 300, 0.3))) - : versioned(params.cluster(), Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); + ? versioned(clusterSpec, Map.of(new Version(0), new NodeResources(2, 16, 300, 0.3))) + : versioned(clusterSpec, Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); } else { // When changing defaults here update cloud.vespa.ai/en/reference/services return zone.cloud().dynamicProvisioning() - ? versioned(params.cluster(), Map.of(new Version(0), new NodeResources(2.0, 8, 50, 0.3))) - : versioned(params.cluster(), Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); + ? versioned(clusterSpec, Map.of(new Version(0), new NodeResources(2.0, 8, 50, 0.3))) + : versioned(clusterSpec, Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); } } @@ -178,11 +177,10 @@ public class CapacityPolicies { } /** Returns whether the nodes requested can share physical host with other applications */ - public boolean decideExclusivity(Capacity capacity, ClusterSpec requestedCluster) { - if (capacity.type() != NodeType.tenant) return true; - if (capacity.cloudAccount().isPresent()) return true; - if (!requestedCluster.isExclusive()) return false; - return capacity.isRequired() || zone.environment() == Environment.prod; + public ClusterSpec decideExclusivity(Capacity capacity, ClusterSpec requestedCluster) { + if (capacity.cloudAccount().isPresent()) return requestedCluster.withExclusivity(true); // Implicit exclusive + boolean exclusive = requestedCluster.isExclusive() && (capacity.isRequired() || zone.environment() == Environment.prod); + return requestedCluster.withExclusivity(exclusive); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index ebb7081e366..38cbfa7fe5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.HostEvent; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node; import java.util.Collection; @@ -54,10 +53,7 @@ public interface HostProvisioner { * @return a runnable that waits for the provisioning request to finish. It can be run without holding any locks, * but may fail with an exception that should be propagated to the user initiating prepare() */ - Runnable provisionHosts(SharedHost sharedHost, - HostProvisionRequest request, - Predicate realHostResourcesWithinLimits, - Consumer> whenProvisioned) throws NodeAllocationException; + Runnable provisionHosts(HostProvisionRequest request, Predicate realHostResourcesWithinLimits, Consumer> whenProvisioned) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index e22c039196c..21340baf273 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -84,10 +84,10 @@ class NodeAllocation { private final NodeRepository nodeRepository; private final Optional requiredHostFlavor; - private final AllocationParams params; + private final boolean makeExclusive; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier nextIndex, NodeRepository nodeRepository, AllocationParams params) { + Supplier nextIndex, NodeRepository nodeRepository, boolean makeExclusive) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; @@ -100,7 +100,7 @@ class NodeAllocation { .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); - this.params = params; + this.makeExclusive = makeExclusive; } /** @@ -133,7 +133,7 @@ class NodeAllocation { } } else if ( ! saturated() && hasCompatibleResources(candidate)) { - if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(params, candidate, application, cluster)) { + if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } @@ -169,7 +169,7 @@ class NodeAllocation { boolean alreadyRetired = candidate.allocation().map(a -> a.membership().retired()).orElse(false); return alreadyRetired ? Retirement.alreadyRetired : Retirement.none; } - if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(params, candidate, application, cluster)) return Retirement.outsideRealLimits; + if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) return Retirement.outsideRealLimits; if (violatesParentHostPolicy(candidate)) return Retirement.violatesParentHostPolicy; if ( ! hasCompatibleResources(candidate)) return Retirement.incompatibleResources; if (candidate.parent.map(node -> node.status().wantToUpgradeFlavor()).orElse(false)) return Retirement.violatesHostFlavorGeneration; @@ -197,7 +197,11 @@ class NodeAllocation { } private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { - return candidate.violatesExclusivity(params, allNodes); + return candidate.violatesExclusivity(cluster, application, + nodeRepository.exclusiveClusterType(cluster), + nodeRepository.exclusiveAllocation(cluster), + nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.zone().cloud().allowHostSharing(), allNodes, makeExclusive); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 76c8445eb8c..8c29b40bc26 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -594,8 +594,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable spareHosts; - public NodePrioritizer(AllocationParams params, LockedNodeList allNodes, NodeSpec nodeSpec, + public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, boolean dynamicProvisioning, boolean allowHostSharing, IP.Allocation.Context ipAllocationContext, Nodes nodes, - HostResourcesCalculator hostResourcesCalculator, int spareCount) { + HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean exclusiveAllocation, boolean makeExclusive) { this.allNodes = allNodes; this.calculator = hostResourcesCalculator; this.capacity = new HostCapacity(this.allNodes, hostResourcesCalculator); this.requested = nodeSpec; + this.clusterSpec = clusterSpec; + this.application = application; this.dynamicProvisioning = dynamicProvisioning; this.allowHostSharing = allowHostSharing; - this.params = params; + this.exclusiveAllocation = exclusiveAllocation; + this.makeExclusive = makeExclusive; this.spareHosts = dynamicProvisioning ? capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodes.asList()) : capacity.findSpareHosts(this.allNodes.asList(), spareCount); this.ipAllocationContext = ipAllocationContext; this.nodes = nodes; - NodeList nodesInCluster = this.allNodes.owner(params.application()).type(params.cluster().type()).cluster(params.cluster().id()); + NodeList nodesInCluster = this.allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); long currentGroups = nonRetiredNodesInCluster.state(Node.State.active).stream() .flatMap(node -> node.allocation() @@ -75,7 +81,7 @@ public class NodePrioritizer { // In dynamically provisioned zones, we can always take spare hosts since we can provision new on-demand, // NodeCandidate::compareTo will ensure that they will not be used until there is no room elsewhere. // In non-dynamically provisioned zones, we only allow allocating to spare hosts to replace failed nodes. - this.canAllocateToSpareHosts = dynamicProvisioning || isReplacement(nodesInCluster, params.cluster().group()); + this.canAllocateToSpareHosts = dynamicProvisioning || isReplacement(nodesInCluster, clusterSpec.group()); } /** Collects all node candidates for this application and returns them in the most-to-least preferred order */ @@ -119,19 +125,19 @@ public class NodePrioritizer { for (Node host : allNodes) { if ( ! nodes.canAllocateTenantNodeTo(host, dynamicProvisioning)) continue; if (nodes.suspended(host)) continue; // Hosts that are suspended may be down for some time, e.g. for OS upgrade - if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(params.application().tenant())) continue; - if (host.reservedTo().isPresent() && params.application().instance().isTester()) continue; - if (params.makeExclusive()) { - if ( ! allowHostSharing && params.exclusiveAllocation() && ! fitsPerfectly(host)) continue; + if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; + if (host.reservedTo().isPresent() && application.instance().isTester()) continue; + if (makeExclusive) { + if ( ! allowHostSharing && exclusiveAllocation && ! fitsPerfectly(host)) continue; } else { if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; } - if ( ! host.provisionedForApplicationId().map(params.application()::equals).orElse(true)) continue; - if ( ! host.exclusiveToApplicationId().map(params.application()::equals).orElse(true)) continue; - if ( ! host.exclusiveToClusterType().map(params.cluster().type()::equals).orElse(true)) continue; + if ( ! host.provisionedForApplicationId().map(application::equals).orElse(true)) continue; + if ( ! host.exclusiveToApplicationId().map(application::equals).orElse(true)) continue; + if ( ! host.exclusiveToClusterType().map(clusterSpec.type()::equals).orElse(true)) continue; if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requested.resources().get())) continue; - if ( ! allNodes.childrenOf(host).owner(params.application()).cluster(params.cluster().id()).isEmpty()) continue; + if ( ! allNodes.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; if ( ! requested.cloudAccount().isUnspecified() && ! requested.cloudAccount().equals(host.cloudAccount())) continue; candidates.add(NodeCandidate.createNewChild(requested.resources().get(), @@ -154,8 +160,8 @@ public class NodePrioritizer { .filter(node -> node.type() == requested.type()) .filter(node -> legalStates.contains(node.state())) .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(params.application())) - .filter(node -> node.allocation().get().membership().cluster().id().equals(params.cluster().id())) + .filter(node -> node.allocation().get().owner().equals(application)) + .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) .filter(node -> node.state() == Node.State.active || canStillAllocate(node)) .map(node -> candidateFrom(node, false)) .forEach(candidates::add); @@ -185,7 +191,7 @@ public class NodePrioritizer { parent.exclusiveToApplicationId().isEmpty() && requested.canResize(node.resources(), capacity.unusedCapacityOf(parent), - params.cluster().type(), + clusterSpec.type(), topologyChange, currentClusterSize)); } else { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 23065055228..e2f1b7358cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -85,42 +85,41 @@ public class NodeRepositoryProvisioner implements Provisioner { public List prepare(ApplicationId application, ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { log.log(Level.FINE, "Received deploy prepare request for " + requested + " for application " + application + ", cluster " + cluster); - var params = AllocationParams.from(nodeRepository, application, cluster, cluster.vespaVersion()); - validate(params, requested, logger); - - params = params.with(cluster.withExclusivity(capacityPolicies.decideExclusivity(requested, cluster))); + validate(application, cluster, requested, logger); NodeResources resources; NodeSpec nodeSpec; if (requested.type() == NodeType.tenant) { + cluster = capacityPolicies.decideExclusivity(requested, cluster); Capacity actual = capacityPolicies.applyOn(requested, application, cluster.isExclusive()); - ClusterResources target = decideTargetResources(params, actual); + ClusterResources target = decideTargetResources(application, cluster, actual); validate(actual, target, cluster, application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); - resources = getNodeResources(params, cluster, target.nodeResources(), application); + resources = getNodeResources(cluster, target.nodeResources(), application); nodeSpec = NodeSpec.from(target.nodes(), target.groups(), resources, cluster.isExclusive(), actual.canFail(), requested.cloudAccount().orElse(nodeRepository.zone().cloud().account()), requested.clusterInfo().hostTTL()); } else { - resources = getNodeResources(params, cluster, requested.minResources().nodeResources(), application); + cluster = cluster.withExclusivity(true); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account()); } - return asSortedHosts(preparer.prepare(params, application, cluster, nodeSpec), + return asSortedHosts(preparer.prepare(application, cluster, nodeSpec), requireCompatibleResources(resources, cluster)); } - private void validate(AllocationParams params, Capacity requested, ProvisionLogger logger) { - if (params.cluster().group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group"); + private void validate(ApplicationId application, ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { + if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group"); - nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits(params, "Min", requested.minResources().nodeResources()); - nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits(params, "Max", requested.maxResources().nodeResources()); + nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), application, cluster); + nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), application, cluster); if (!requested.minResources().nodeResources().gpuResources().equals(requested.maxResources().nodeResources().gpuResources())) throw new IllegalArgumentException(requested + " is invalid: GPU capacity cannot have ranges"); - logInsufficientDiskResources(params.cluster(), requested, logger); + logInsufficientDiskResources(cluster, requested, logger); } private void logInsufficientDiskResources(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { @@ -133,8 +132,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } } - private NodeResources getNodeResources(AllocationParams params, ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { - return capacityPolicies.specifyFully(params, nodeResources); + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { + return capacityPolicies.specifyFully(nodeResources, cluster, applicationId); } @Override @@ -166,39 +165,41 @@ public class NodeRepositoryProvisioner implements Provisioner { * Returns the target cluster resources, a value between the min and max in the requested capacity, * and updates the application store with the received min and max. */ - private ClusterResources decideTargetResources(AllocationParams params, Capacity requested) { - try (Mutex lock = nodeRepository.applications().lock(params.application())) { - var application = nodeRepository.applications().get(params.application()).orElse(Application.empty(params.application())) - .withCluster(params.cluster().id(), params.cluster().isExclusive(), requested); + private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { + try (Mutex lock = nodeRepository.applications().lock(applicationId)) { + var application = nodeRepository.applications().get(applicationId).orElse(Application.empty(applicationId)) + .withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested); nodeRepository.applications().put(application, lock); - var cluster = application.cluster(params.cluster().id()).get(); - return cluster.target().resources().orElseGet(() -> currentResources(params, application, cluster, requested)); + var cluster = application.cluster(clusterSpec.id()).get(); + return cluster.target().resources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested)); } } /** Returns the current resources of this cluster, or requested min if none */ - private ClusterResources currentResources(AllocationParams params, Application application, Cluster cluster, Capacity requested) { + private ClusterResources currentResources(Application application, + ClusterSpec clusterSpec, + Cluster cluster, + Capacity requested) { NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(application.id()) - .cluster(params.cluster().id()) + .cluster(clusterSpec.id()) .not().retired() .not().removable(); boolean firstDeployment = nodes.isEmpty(); var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableResources(params, initialResourcesFrom(params, requested)) + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) : new AllocatableResources(nodes, nodeRepository); - var model = new ClusterModel(params, application, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); - return within(params, Limits.of(requested), model, firstDeployment); + var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); + return within(Limits.of(requested), model, firstDeployment); } - private ClusterResources initialResourcesFrom(AllocationParams params, Capacity requested) { - return capacityPolicies.specifyFully(params, requested.minResources()); + private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { + return capacityPolicies.specifyFully(requested.minResources(), clusterSpec, applicationId); } /** Make the minimal adjustments needed to the current resources to stay within the limits */ - private ClusterResources within(AllocationParams params, - Limits limits, + private ClusterResources within(Limits limits, ClusterModel model, boolean firstDeployment) { if (limits.min().equals(limits.max())) return limits.min(); @@ -208,11 +209,10 @@ public class NodeRepositoryProvisioner implements Provisioner { return model.current().advertisedResources(); // Otherwise, find an allocation that preserves the current resources as well as possible - return allocationOptimizer.findBestAllocation(params, - Load.one(), + return allocationOptimizer.findBestAllocation(Load.one(), model, limits) - .orElseThrow(() -> newNoAllocationPossible(params, model.current().clusterSpec(), limits)) + .orElseThrow(() -> newNoAllocationPossible(model.current().clusterSpec(), limits)) .advertisedResources(); } @@ -277,10 +277,10 @@ public class NodeRepositoryProvisioner implements Provisioner { return nodeResources; } - private IllegalArgumentException newNoAllocationPossible(AllocationParams params, ClusterSpec spec, Limits limits) { + private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); - if (params.exclusiveAllocation() && findNearestNodeResources(limits).isPresent()) + if (nodeRepository.exclusiveAllocation(spec) && findNearestNodeResources(limits).isPresent()) message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits).get()); return new IllegalArgumentException(message.toString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index f592851f45e..cd331cbd5fa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -31,14 +31,14 @@ public class NodeResourceLimits { } /** Validates the resources applications ask for (which are in "advertised" resource space) */ - public void ensureWithinAdvertisedLimits(AllocationParams params, String type, NodeResources requested) { - boolean exclusive = params.exclusiveAllocation(); - if (! requested.vcpuIsUnspecified() && requested.vcpu() < minAdvertisedVcpu(params.application(), params.cluster(), exclusive)) - illegal(type, "vcpu", "", params.cluster(), requested.vcpu(), minAdvertisedVcpu(params.application(), params.cluster(), exclusive)); - if (! requested.memoryGbIsUnspecified() && requested.memoryGb() < minAdvertisedMemoryGb(params.application(), params.cluster(), exclusive)) - illegal(type, "memoryGb", "Gb", params.cluster(), requested.memoryGb(), minAdvertisedMemoryGb(params.application(), params.cluster(), exclusive)); + public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ApplicationId applicationId, ClusterSpec cluster) { + boolean exclusive = nodeRepository.exclusiveAllocation(cluster); + if (! requested.vcpuIsUnspecified() && requested.vcpu() < minAdvertisedVcpu(applicationId, cluster, exclusive)) + illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(applicationId, cluster, exclusive)); + if (! requested.memoryGbIsUnspecified() && requested.memoryGb() < minAdvertisedMemoryGb(applicationId, cluster, exclusive)) + illegal(type, "memoryGb", "Gb", cluster, requested.memoryGb(), minAdvertisedMemoryGb(applicationId, cluster, exclusive)); if (! requested.diskGbIsUnspecified() && requested.diskGb() < minAdvertisedDiskGb(requested, exclusive)) - illegal(type, "diskGb", "Gb", params.cluster(), requested.diskGb(), minAdvertisedDiskGb(requested, exclusive)); + illegal(type, "diskGb", "Gb", cluster, requested.diskGb(), minAdvertisedDiskGb(requested, exclusive)); } // TODO: Remove this when we are ready to fail, not just warn on this. */ @@ -48,17 +48,17 @@ public class NodeResourceLimits { } /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ - public boolean isWithinRealLimits(AllocationParams params, NodeCandidate candidateNode, ApplicationId applicationId, ClusterSpec cluster) { + public boolean isWithinRealLimits(NodeCandidate candidateNode, ApplicationId applicationId, ClusterSpec cluster) { if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes - return isWithinRealLimits(params, nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository), + return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository), applicationId, cluster); } /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ - public boolean isWithinRealLimits(AllocationParams params, NodeResources realResources, ApplicationId applicationId, ClusterSpec cluster) { + public boolean isWithinRealLimits(NodeResources realResources, ApplicationId applicationId, ClusterSpec cluster) { if (realResources.isUnspecified()) return true; - if (realResources.vcpu() < minRealVcpu(params)) return false; + if (realResources.vcpu() < minRealVcpu(applicationId, cluster)) return false; if (realResources.memoryGb() < minRealMemoryGb(cluster)) return false; if (realResources.diskGb() < minRealDiskGb()) return false; return true; @@ -115,8 +115,8 @@ public class NodeResourceLimits { return 4; } - private double minRealVcpu(AllocationParams params) { - return minAdvertisedVcpu(params.application(), params.cluster(), params.exclusiveAllocation()); + private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { + return minAdvertisedVcpu(applicationId, cluster, nodeRepository.exclusiveAllocation(cluster)); } private static double minRealMemoryGb(ClusterSpec cluster) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 2f1f39c3301..978a69978b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -13,6 +13,9 @@ import com.yahoo.jdisc.Metric; import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -47,37 +50,42 @@ public class Preparer { private final Optional hostProvisioner; private final Optional loadBalancerProvisioner; private final ProvisioningThrottler throttler; + private final BooleanFlag makeExclusiveFlag; public Preparer(NodeRepository nodeRepository, Optional hostProvisioner, Optional loadBalancerProvisioner, Metric metric) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; this.loadBalancerProvisioner = loadBalancerProvisioner; this.throttler = new ProvisioningThrottler(nodeRepository, metric); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(nodeRepository.flagSource()); } /** * Ensure sufficient nodes are reserved or active for the given application, group and cluster * - * @param params misc constants used in preparation - * @param application the application we are allocating to - * @param cluster the cluster and group we are allocating to - * @param requested a specification of the requested nodes + * @param application the application we are allocating to + * @param cluster the cluster and group we are allocating to + * @param requested a specification of the requested nodes * @return the list of nodes this cluster group will have allocated if activated */ // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List prepare(AllocationParams params, ApplicationId application, ClusterSpec cluster, NodeSpec requested) { + public List prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requested) { log.log(Level.FINE, () -> "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " + requested.resources().orElse(NodeResources.unspecified())); loadBalancerProvisioner.ifPresent(provisioner -> provisioner.prepare(application, cluster, requested)); + boolean makeExclusive = makeExclusiveFlag.with(FetchVector.Dimension.TENANT_ID, application.tenant().value()) + .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) + .with(FetchVector.Dimension.VESPA_VERSION, cluster.vespaVersion().toFullString()) + .value(); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, // and we can return nodes previously allocated. LockedNodeList allNodes = nodeRepository.nodes().list(PROBE_LOCK); NodeIndices indices = new NodeIndices(cluster.id(), allNodes); - NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes, params); + NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes, makeExclusive); if (probeAllocation.fulfilledAndNoChanges()) { List acceptedNodes = probeAllocation.finalNodes(); indices.commitProbe(); @@ -85,28 +93,28 @@ public class Preparer { } else { // There were some changes, so re-do the allocation with locks indices.resetProbe(); - return prepareWithLocks(application, cluster, requested, indices, params); + return prepareWithLocks(application, cluster, requested, indices, makeExclusive); } } - private ApplicationMutex parentLockOrNull(AllocationParams params, NodeType type) { - return NodeCandidate.canMakeHostExclusive(params.makeExclusive(), type, nodeRepository.zone().cloud().allowHostSharing()) ? + private ApplicationMutex parentLockOrNull(boolean makeExclusive, NodeType type) { + return NodeCandidate.canMakeHostExclusive(makeExclusive, type, nodeRepository.zone().cloud().allowHostSharing()) ? nodeRepository.applications().lock(InfrastructureApplication.withNodeType(type.parentNodeType()).id()) : null; } /// Note that this will write to the node repo. - private List prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices, AllocationParams params) { + private List prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices, boolean makeExclusive) { Runnable waiter = null; List acceptedNodes; try (Mutex lock = nodeRepository.applications().lock(application); - ApplicationMutex parentLockOrNull = parentLockOrNull(params, requested.type()); + ApplicationMutex parentLockOrNull = parentLockOrNull(makeExclusive, requested.type()); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); - NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes, params); + NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes, makeExclusive); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { - HostSharing sharing = hostSharing(params, hostType); + HostSharing sharing = hostSharing(cluster, hostType); Version osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); NodeAllocation.HostDeficit deficit = allocation.hostDeficit().get(); Set hosts = new LinkedHashSet<>(); @@ -139,9 +147,8 @@ public class Preparer { Optional.of(cluster.id()), requested.cloudAccount(), deficit.dueToFlavorUpgrade()); - Predicate realHostResourcesWithinLimits = resources -> - nodeRepository.nodeResourceLimits().isWithinRealLimits(params, resources, application, cluster); - waiter = hostProvisioner.get().provisionHosts(params.sharedHost(), request, realHostResourcesWithinLimits, whenProvisioned); + Predicate realHostResourcesWithinLimits = resources -> nodeRepository.nodeResourceLimits().isWithinRealLimits(resources, application, cluster); + waiter = hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been @@ -155,7 +162,7 @@ public class Preparer { // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. // Try again, but without retiring indices.resetProbe(); - List accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices, params); + List accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices, makeExclusive); log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); return accepted; } @@ -187,22 +194,24 @@ public class Preparer { } private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier nextIndex, LockedNodeList allNodes, AllocationParams params) { + Supplier nextIndex, LockedNodeList allNodes, boolean makeExclusive) { validateAccount(requested.cloudAccount(), application, allNodes); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository, params); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository, makeExclusive); var allocationContext = IP.Allocation.Context.from(nodeRepository.zone().cloud().name(), requested.cloudAccount().isExclave(nodeRepository.zone()), nodeRepository.nameResolver()); - NodePrioritizer prioritizer = new NodePrioritizer(params, - allNodes, + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, + application, + cluster, requested, nodeRepository.zone().cloud().dynamicProvisioning(), nodeRepository.zone().cloud().allowHostSharing(), allocationContext, nodeRepository.nodes(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount() - ); + nodeRepository.spareCount(), + nodeRepository.exclusiveAllocation(cluster), + makeExclusive); allocation.offer(prioritizer.collect()); return allocation; } @@ -229,12 +238,13 @@ public class Preparer { (hostType == NodeType.host || hostType.isConfigServerHostLike()); } - private HostSharing hostSharing(AllocationParams params, NodeType hostType) { - if (hostType.isSharable()) { - if (params.exclusiveProvisioning()) return HostSharing.provision; - if (params.exclusiveAllocation()) return HostSharing.exclusive; - } - return HostSharing.any; + private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { + if ( hostType.isSharable()) + return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; + else + return HostSharing.any; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index ded0aaf513e..225eb3e4e8d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; -import com.yahoo.component.Vtag; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ClusterResources; import com.yahoo.slime.Cursor; @@ -15,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.Limits; import com.yahoo.vespa.hosted.provision.autoscale.Load; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.net.URI; import java.util.List; @@ -59,13 +56,11 @@ public class ApplicationSerializer { NodeRepository nodeRepository, Cursor clustersObject) { NodeList nodes = applicationNodes.not().retired().cluster(cluster.id()); - ClusterSpec clusterSpec = nodes.clusterSpec(); if (nodes.isEmpty()) return; ClusterResources currentResources = nodes.toResources(); Cursor clusterObject = clustersObject.setObject(cluster.id().value()); - clusterObject.setString("type", clusterSpec.type().name()); - var params = AllocationParams.from(nodeRepository, application.id(), clusterSpec, clusterSpec.vespaVersion()); - Limits limits = Limits.of(cluster).fullySpecified(params); + clusterObject.setString("type", nodes.clusterSpec().type().name()); + Limits limits = Limits.of(cluster).fullySpecified(nodes.clusterSpec(), nodeRepository, application.id()); toSlime(limits.min(), clusterObject.setObject("min")); toSlime(limits.max(), clusterObject.setObject("max")); if ( ! cluster.groupSize().isEmpty()) @@ -75,7 +70,7 @@ public class ApplicationSerializer { toSlime(cluster.suggested(), clusterObject.setObject("suggested")); toSlime(cluster.target(), clusterObject.setObject("target")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); - clusterObject.setLong("scalingDuration", cluster.scalingDuration(clusterSpec).toMillis()); + clusterObject.setLong("scalingDuration", cluster.scalingDuration(nodes.clusterSpec()).toMillis()); } private static void toSlime(Autoscaling autoscaling, Cursor autoscalingObject) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index eddb7fd40ec..9080030f026 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; @@ -47,7 +46,6 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.maintenance.InfraApplicationRedeployer; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 40734cc25c4..b5bb91af71a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -9,11 +9,9 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisionRequest; @@ -75,10 +73,7 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public Runnable provisionHosts(SharedHost sharedHost, - HostProvisionRequest request, - Predicate realHostResourcesWithinLimits, - Consumer> whenProvisioned) throws NodeAllocationException { + public Runnable provisionHosts(HostProvisionRequest request, Predicate realHostResourcesWithinLimits, Consumer> whenProvisioned) throws NodeAllocationException { if (behaviour(Behaviour.failProvisionRequest)) throw new NodeAllocationException("No capacity for provision request", true); Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) @@ -269,7 +264,7 @@ public class MockHostProvisioner implements HostProvisioner { /** Fail call to {@link MockHostProvisioner#provision(com.yahoo.vespa.hosted.provision.Node)} */ failProvisioning, - /** Fail call to {@link MockHostProvisioner#provisionHosts(SharedHost, HostProvisionRequest, Predicate, Consumer)} */ + /** Fail call to {@link MockHostProvisioner#provisionHosts(HostProvisionRequest, Predicate, Consumer)} */ failProvisionRequest, /** Fail call to {@link MockHostProvisioner#deprovision(com.yahoo.vespa.hosted.provision.Node)} */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 613a0434054..d4d34ab66e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -389,7 +389,7 @@ public class AutoscalingTest { .build(); NodeResources defaultResources = - new CapacityPolicies(fixture.tester().nodeRepository()).specifyFully(fixture.allocationParams, NodeResources.unspecified()); + new CapacityPolicies(fixture.tester().nodeRepository()).specifyFully(NodeResources.unspecified(), fixture.clusterSpec, fixture.applicationId); fixture.tester().assertResources("Min number of nodes and default resources", 2, 1, defaultResources, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java index 6f3b348c5b0..5e4dfdc974d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java @@ -10,7 +10,6 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.Status; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; @@ -90,11 +89,10 @@ public class ClusterModelTest { Cluster cluster = cluster(); application = application.with(cluster); var nodeRepository = new ProvisioningTester.Builder().build().nodeRepository(); - var params = AllocationParams.from(nodeRepository, application.id(), clusterSpec, clusterSpec.vespaVersion()); - return new ClusterModel(params, + return new ClusterModel(nodeRepository, application.with(status), - cluster, - new AllocatableResources(params, clusterResources()), + clusterSpec, cluster, + new AllocatableResources(clusterResources(), clusterSpec, nodeRepository), clock, Duration.ofMinutes(10), Duration.ofMinutes(5), timeseries(cluster,100, queryRate, writeRate, clock), ClusterNodesTimeseries.empty()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 4c34dd1f7c8..1ecd5736e17 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsHostResourcesCalcu import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes; import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.provisioning.AllocationParams; import java.time.Duration; import java.util.Arrays; @@ -41,7 +40,6 @@ import java.util.Optional; public class Fixture { final DynamicProvisioningTester tester; - final AllocationParams allocationParams; final ApplicationId applicationId; final ClusterSpec clusterSpec; final Capacity capacity; @@ -54,7 +52,6 @@ public class Fixture { clusterSpec = builder.cluster; capacity = builder.capacity; tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); - allocationParams = AllocationParams.from(tester.nodeRepository(), builder.application, builder.cluster, builder.cluster.vespaVersion()); var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity; tester.deploy(builder.application, builder.cluster, deployCapacity); this.loader = new Loader(this); @@ -83,8 +80,9 @@ public class Fixture { public Capacity capacity() { return capacity; } public ClusterModel clusterModel() { - return new ClusterModel(allocationParams, + return new ClusterModel(tester.nodeRepository(), application(), + clusterSpec, cluster(), nodes(), new AllocatableResources(nodes(), tester.nodeRepository()), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 50696583b88..be2b2ca896a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -155,7 +155,7 @@ public class DynamicProvisioningTester { } public Autoscaling autoscale(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster)); + capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(cluster.id(), false, capacity); try (Mutex lock = nodeRepository().applications().lock(applicationId)) { -- cgit v1.2.3