diff options
author | Jon Bratseth <bratseth@gmail.com> | 2024-05-19 14:12:41 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-19 14:12:41 +0200 |
commit | cbb09179e61b54c568ed9976194402e935977946 (patch) | |
tree | 464cf634df5a326f7065a6d6a226a1154d8a6fd2 /node-repository | |
parent | dbd26dbe414187f6f5acbd18785549d9ced2a281 (diff) | |
parent | 12ea6e856952196c38fc47f6b9318900a046f2a0 (diff) |
Merge pull request #31241 from vespa-engine/hakonhall/quota-capacity-policies
Hakonhall/quota capacity policies
Diffstat (limited to 'node-repository')
21 files changed, 130 insertions, 292 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 539f3128091..cc46fc381f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -4,25 +4,25 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.maintenance.JobControl; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; -import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources.Architecture; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -45,6 +45,8 @@ import java.time.Clock; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; + /** * The top level singleton in the node repo, providing access to all its state as child objects. * @@ -55,6 +57,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr private final CuratorDb db; private final Clock clock; private final Zone zone; + private final Exclusivity exclusivity; private final Nodes nodes; private final NodeFlavors flavors; private final HostResourcesCalculator resourcesCalculator; @@ -73,7 +76,6 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr private final Orchestrator orchestrator; private final int spareCount; private final ProtoHealthChecker healthChecker; - private final JacksonFlag<SharedHost> sharedHosts; /** * Creates a node repository from a zookeeper provider. @@ -85,6 +87,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr ProvisionServiceProvider provisionServiceProvider, Curator curator, Zone zone, + Exclusivity exclusivity, FlagSource flagSource, MetricsDb metricsDb, Orchestrator orchestrator) { @@ -93,6 +96,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr curator, Clock.systemUTC(), zone, + exclusivity, new DnsNameResolver(), DockerImage.fromString(config.containerImage()), optionalImage(config.tenantContainerImage()), @@ -113,6 +117,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr Curator curator, Clock clock, Zone zone, + Exclusivity exclusivity, NameResolver nameResolver, DockerImage containerImage, Optional<DockerImage> tenantContainerImage, @@ -129,8 +134,9 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.flagSource = flagSource; this.db = new CuratorDb(flavors, curator, clock, useCuratorClientCache); - this.zone = zone; this.clock = clock; + this.zone = zone; + this.exclusivity = exclusivity; this.applications = new Applications(db); this.nodes = new Nodes(db, zone, clock, orchestrator, applications); this.flavors = flavors; @@ -147,7 +153,6 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.metricsDb = metricsDb; this.orchestrator = orchestrator; this.spareCount = spareCount; - this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(flagSource()); this.healthChecker = provisionServiceProvider.getHealthChecker(); nodes.rewrite(); } @@ -208,25 +213,14 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr /** The number of nodes we should ensure has free capacity for node failures whenever possible */ public int spareCount() { return spareCount; } - /** Returns whether nodes must be allocated to hosts that are exclusive to the cluster type. */ - public boolean exclusiveClusterType(ClusterSpec cluster) { - return sharedHosts.value().hasClusterType(cluster.type().name()); - } - - /** - * Returns whether nodes are allocated exclusively in this instance given this cluster spec. - * Exclusive allocation requires that the wanted node resources matches the advertised resources of the node - * perfectly. - */ - public boolean exclusiveAllocation(ClusterSpec clusterSpec) { - return clusterSpec.isExclusive() || - ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || - ( !zone().cloud().allowHostSharing() && !sharedHosts.value().supportsClusterType(clusterSpec.type().name())); - } + public Exclusivity exclusivity() { return exclusivity; } - /** Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ - public boolean exclusiveProvisioning(ClusterSpec clusterSpec) { - return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + public CapacityPolicies capacityPoliciesFor(ApplicationId applicationId) { + String adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE + .bindTo(flagSource) + .with(INSTANCE_ID, applicationId.serializedForm()) + .value(); + return new CapacityPolicies(zone, exclusivity, applicationId, Architecture.valueOf(adminClusterNodeArchitecture)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 75a00fa951e..98c5af2688a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -41,7 +41,7 @@ public class AllocatableResources { this.nodes = requested.nodes(); this.groups = requested.groups(); this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), cloudAccount, - nodeRepository.exclusiveAllocation(clusterSpec), false); + nodeRepository.exclusivity().allocation(clusterSpec), false); this.advertisedResources = requested.nodeResources(); this.clusterSpec = clusterSpec; this.fulfilment = 1; @@ -178,7 +178,7 @@ public class AllocatableResources { ClusterModel model, NodeRepository nodeRepository) { var systemLimits = nodeRepository.nodeResourceLimits(); - boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); + boolean exclusive = nodeRepository.exclusivity().allocation(clusterSpec); if (! exclusive) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources var allocatableResources = calculateAllocatableResources(wantedResources, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 5a790a1fe19..82199888a48 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -103,7 +103,7 @@ public class AllocationOptimizer { /** Returns the max resources of a host one node may allocate. */ private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel model) { - if (nodeRepository.exclusiveAllocation(model.clusterSpec())) return hostResources; + if (nodeRepository.exclusivity().allocation(model.clusterSpec())) return hostResources; // static, shared hosts: Allocate at most half of the host cpu to simplify management return hostResources.withVcpu(hostResources.vcpu() / 2); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 74907feee41..504965f1992 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; +import com.yahoo.config.provision.CapacityPolicies; import java.time.Clock; import java.time.Duration; @@ -48,6 +48,7 @@ public class ClusterModel { private static final double fixedCpuCostFraction = 0.1; private final NodeRepository nodeRepository; + private final CapacityPolicies capacityPolicies; private final Application application; private final ClusterSpec clusterSpec; private final Cluster cluster; @@ -84,6 +85,7 @@ public class ClusterModel { MetricsDb metricsDb, Clock clock) { this.nodeRepository = nodeRepository; + this.capacityPolicies = nodeRepository.capacityPoliciesFor(application.id()); this.application = application; this.clusterSpec = clusterSpec; this.cluster = cluster; @@ -108,6 +110,7 @@ public class ClusterModel { ClusterTimeseries clusterTimeseries, ClusterNodesTimeseries nodeTimeseries) { this.nodeRepository = nodeRepository; + this.capacityPolicies = nodeRepository.capacityPoliciesFor(application.id()); this.application = application; this.clusterSpec = clusterSpec; this.cluster = cluster; @@ -171,7 +174,7 @@ public class ClusterModel { } public boolean isExclusive() { - return nodeRepository.exclusiveAllocation(clusterSpec); + return nodeRepository.exclusivity().allocation(clusterSpec); } /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ @@ -436,12 +439,10 @@ public class ClusterModel { double averageReal() { if (nodes.isEmpty()) { // we're estimating - var initialResources = new CapacityPolicies(nodeRepository).specifyFully(cluster.minResources().nodeResources(), - clusterSpec, - application.id()); + var initialResources = capacityPolicies.specifyFully(cluster.minResources().nodeResources(), clusterSpec); return nodeRepository.resourcesCalculator().requestToReal(initialResources, cloudAccount(), - nodeRepository.exclusiveAllocation(clusterSpec), + nodeRepository.exclusivity().allocation(clusterSpec), false).memoryGb(); } else { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index ab93e585c88..51046df90af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import java.util.Objects; @@ -68,9 +67,9 @@ public class Limits { public Limits fullySpecified(ClusterSpec clusterSpec, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var capacityPolicies = new CapacityPolicies(nodeRepository); - return new Limits(capacityPolicies.specifyFully(min, clusterSpec, applicationId), - capacityPolicies.specifyFully(max, clusterSpec, applicationId), + var capacityPolicies = nodeRepository.capacityPoliciesFor(applicationId); + return new Limits(capacityPolicies.specifyFully(min, clusterSpec), + capacityPolicies.specifyFully(max, clusterSpec), groupSize); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 98f0dfebc4a..c9e920dc243 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -223,7 +223,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); - HostSharing sharingMode = nodeRepository().exclusiveAllocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; + HostSharing sharingMode = nodeRepository().exclusivity().allocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, sharingMode, clusterType.map(ClusterSpec.Type::valueOf), Optional.empty(), @@ -290,13 +290,13 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, nodeSpec, true, false, allocationContext, nodeRepository().nodes(), nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), - nodeRepository().exclusiveAllocation(cluster)); + nodeRepository().exclusivity().allocation(cluster)); List<NodeCandidate> nodeCandidates = prioritizer.collect() .stream() .filter(node -> node.violatesExclusivity(cluster, application, - nodeRepository().exclusiveClusterType(cluster), - nodeRepository().exclusiveAllocation(cluster), + nodeRepository().exclusivity().clusterType(cluster), + nodeRepository().exclusivity().allocation(cluster), false, nodeRepository().zone().cloud().allowHostSharing(), allNodes) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java deleted file mode 100644 index 46dfb51a433..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeResources.DiskSpeed; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.util.Map; -import java.util.TreeMap; - -import static com.yahoo.config.provision.NodeResources.Architecture; -import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; -import static java.util.Objects.requireNonNull; - -/** - * Defines the policies for assigning cluster capacity in various environments - * - * @author bratseth - * @see NodeResourceLimits - */ -public class CapacityPolicies { - - private final NodeRepository nodeRepository; - private final Zone zone; - private final StringFlag adminClusterNodeArchitecture; - - public CapacityPolicies(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - this.zone = nodeRepository.zone(); - this.adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(nodeRepository.flagSource()); - } - - public Capacity applyOn(Capacity capacity, ApplicationId application, boolean exclusive) { - var min = applyOn(capacity.minResources(), capacity, application, exclusive); - var max = applyOn(capacity.maxResources(), capacity, application, exclusive); - var groupSize = capacity.groupSize().fromAtMost(max.nodes() / min.groups()) - .toAtLeast(min.nodes() / max.groups()); - return capacity.withLimits(min, max, groupSize); - } - - private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application, boolean exclusive) { - int nodes = decideCount(resources.nodes(), capacity.isRequired(), application.instance().isTester()); - int groups = decideGroups(resources.nodes(), resources.groups(), nodes); - var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired(), exclusive); - return new ClusterResources(nodes, groups, nodeResources); - } - - private int decideCount(int requested, boolean required, boolean isTester) { - if (isTester) return 1; - - if (required) return requested; - return switch (zone.environment()) { - case dev, test -> 1; - case perf -> Math.min(requested, 3); - case staging -> requested <= 1 ? requested : Math.max(2, requested / 10); - case prod -> requested; - }; - } - - private int decideGroups(int requestedNodes, int requestedGroups, int decidedNodes) { - if (requestedNodes == decidedNodes) return requestedGroups; - int groups = Math.min(requestedGroups, decidedNodes); // cannot have more groups than nodes - while (groups > 1 && decidedNodes % groups != 0) - groups--; // Must be divisible by the number of groups - return groups; - } - - private NodeResources decideNodeResources(NodeResources target, boolean required, boolean exclusive) { - if (required || exclusive) return target; // Cannot downsize if resources are required, or exclusively allocated - if (target.isUnspecified()) return target; // Cannot be modified - - if (zone.environment() == Environment.dev && zone.cloud().allowHostSharing()) { - // Dev does not cap the cpu or network of containers since usage is spotty: Allocate just a small amount exclusively - target = target.withVcpu(0.1).withBandwidthGbps(0.1); - - // Allocate without GPU in dev - target = target.with(NodeResources.GpuResources.zero()); - } - - // Allow slow storage in zones which are not performance sensitive - if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test) - target = target.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any).withBandwidthGbps(0.1); - - return target; - } - - public ClusterResources specifyFully(ClusterResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.with(specifyFully(resources.nodeResources(), clusterSpec, applicationId)); - } - - public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); - } - - private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { - if (clusterSpec.type() == ClusterSpec.Type.admin) { - Architecture architecture = adminClusterArchitecture(applicationId); - - if (nodeRepository.exclusiveAllocation(clusterSpec)) { - return smallestExclusiveResources().with(architecture); - } - - if (clusterSpec.id().value().equals("cluster-controllers")) { - return clusterControllerResources(clusterSpec, architecture).with(architecture); - } - - if (clusterSpec.id().value().equals("logserver")) { - return logserverResources(architecture).with(architecture); - } - - return versioned(clusterSpec, Map.of(new Version(0), smallestSharedResources())).with(architecture); - } - - if (clusterSpec.type() == ClusterSpec.Type.content) { - // When changing defaults here update cloud.vespa.ai/en/reference/services - return zone.cloud().dynamicProvisioning() - ? versioned(clusterSpec, Map.of(new Version(0), new NodeResources(2, 16, 300, 0.3))) - : versioned(clusterSpec, Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); - } - else { - // When changing defaults here update cloud.vespa.ai/en/reference/services - return zone.cloud().dynamicProvisioning() - ? versioned(clusterSpec, Map.of(new Version(0), new NodeResources(2.0, 8, 50, 0.3))) - : versioned(clusterSpec, Map.of(new Version(0), new NodeResources(1.5, 8, 50, 0.3))); - } - } - - private NodeResources clusterControllerResources(ClusterSpec clusterSpec, Architecture architecture) { - // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have - // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes. - if (architecture == Architecture.x86_64) - return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.32, 10, 0.3))); - else - // arm64 nodes need more memory - return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.50, 10, 0.3))); - } - - private NodeResources logserverResources(Architecture architecture) { - if (zone.cloud().name() == CloudName.AZURE) - return new NodeResources(2, 4, 50, 0.3); - - if (zone.cloud().name() == CloudName.GCP) - return new NodeResources(1, 4, 50, 0.3); - - return architecture == Architecture.arm64 - ? new NodeResources(0.5, 2.5, 50, 0.3) - : new NodeResources(0.5, 2, 50, 0.3); - } - - private Architecture adminClusterArchitecture(ApplicationId instance) { - return Architecture.valueOf(adminClusterNodeArchitecture.with(INSTANCE_ID, instance.serializedForm()).value()); - } - - /** - * Returns the resources for the newest version not newer than that requested in the cluster spec. - */ - static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { - return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), - "no default resources applicable for " + spec + " among: " + resources) - .getValue(); - } - - // The lowest amount of resources that can be exclusive allocated (i.e. a matching host flavor for this exists) - private NodeResources smallestExclusiveResources() { - return zone.cloud().name() == CloudName.AZURE || zone.cloud().name() == CloudName.GCP - ? new NodeResources(2, 8, 50, 0.3) - : new NodeResources(0.5, 8, 50, 0.3); - } - - // The lowest amount of resources that can be shared (i.e. a matching host flavor for this exists) - private NodeResources smallestSharedResources() { - return zone.cloud().name() == CloudName.GCP - ? new NodeResources(1, 4, 50, 0.3) - : new NodeResources(0.5, 2, 50, 0.3); - } - - /** Returns whether the nodes requested can share physical host with other applications */ - public ClusterSpec decideExclusivity(Capacity capacity, ClusterSpec requestedCluster) { - if (capacity.cloudAccount().isPresent()) return requestedCluster.withExclusivity(true); // Implicit exclusive - boolean exclusive = requestedCluster.isExclusive() && (capacity.isRequired() || zone.environment() == Environment.prod); - return requestedCluster.withExclusivity(exclusive); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index b149a9af2c2..433bd0bbefe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -206,9 +206,9 @@ class NodeAllocation { private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { return candidate.violatesExclusivity(cluster, application, - nodeRepository.exclusiveClusterType(cluster), - nodeRepository.exclusiveAllocation(cluster), - nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.exclusivity().clusterType(cluster), + nodeRepository.exclusivity().allocation(cluster), + nodeRepository.exclusivity().provisioning(cluster), nodeRepository.zone().cloud().allowHostSharing(), allNodes); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 2588b02d712..8c056ad1a93 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -53,7 +54,6 @@ public class NodeRepositoryProvisioner implements Provisioner { private final NodeRepository nodeRepository; private final AllocationOptimizer allocationOptimizer; - private final CapacityPolicies capacityPolicies; private final Zone zone; private final Preparer preparer; private final Activator activator; @@ -66,7 +66,6 @@ public class NodeRepositoryProvisioner implements Provisioner { Metric metric) { this.nodeRepository = nodeRepository; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); - this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService() .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); @@ -88,23 +87,24 @@ public class NodeRepositoryProvisioner implements Provisioner { " for application " + application + ", cluster " + cluster); validate(application, cluster, requested, logger); + var capacityPolicies = nodeRepository.capacityPoliciesFor(application); NodeResources resources; NodeSpec nodeSpec; if (requested.type() == NodeType.tenant) { cluster = capacityPolicies.decideExclusivity(requested, cluster); - Capacity actual = capacityPolicies.applyOn(requested, application, cluster.isExclusive()); - ClusterResources target = decideTargetResources(application, cluster, actual); + Capacity actual = capacityPolicies.applyOn(requested, cluster.isExclusive()); + ClusterResources target = decideTargetResources(application, cluster, actual, capacityPolicies); validate(actual, target, cluster, application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); - resources = getNodeResources(cluster, target.nodeResources(), application); + resources = getNodeResources(cluster, target.nodeResources(), application, capacityPolicies); nodeSpec = NodeSpec.from(target.nodes(), target.groups(), resources, cluster.isExclusive(), actual.canFail(), requested.cloudAccount().orElse(nodeRepository.zone().cloud().account()), requested.clusterInfo().hostTTL()); } else { cluster = cluster.withExclusivity(true); - resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application, capacityPolicies); nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account()); } return asSortedHosts(preparer.prepare(application, cluster, nodeSpec), @@ -133,8 +133,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } } - private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { - return capacityPolicies.specifyFully(nodeResources, cluster, applicationId); + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId, CapacityPolicies capacityPolicies) { + return capacityPolicies.specifyFully(nodeResources, cluster); } @Override @@ -166,13 +166,14 @@ public class NodeRepositoryProvisioner implements Provisioner { * Returns the target cluster resources, a value between the min and max in the requested capacity, * and updates the application store with the received min and max. */ - private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { + private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested, + CapacityPolicies capacityPolicies) { try (Mutex lock = nodeRepository.applications().lock(applicationId)) { var application = nodeRepository.applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested); nodeRepository.applications().put(application, lock); var cluster = application.cluster(clusterSpec.id()).get(); - return cluster.target().resources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested)); + return cluster.target().resources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested, capacityPolicies)); } } @@ -180,7 +181,8 @@ public class NodeRepositoryProvisioner implements Provisioner { private ClusterResources currentResources(Application application, ClusterSpec clusterSpec, Cluster cluster, - Capacity requested) { + Capacity requested, + CapacityPolicies capacityPolicies) { NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(application.id()) .cluster(clusterSpec.id()) .not().retired() @@ -188,15 +190,15 @@ public class NodeRepositoryProvisioner implements Provisioner { boolean firstDeployment = nodes.isEmpty(); var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id(), capacityPolicies), clusterSpec, nodeRepository, requested.cloudAccount().orElse(CloudAccount.empty)) : new AllocatableResources(nodes, nodeRepository); var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); return within(Limits.of(requested), model, firstDeployment); } - private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { - return capacityPolicies.specifyFully(requested.minResources(), clusterSpec, applicationId); + private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId, CapacityPolicies capacityPolicies) { + return capacityPolicies.specifyFully(requested.minResources(), clusterSpec); } @@ -278,7 +280,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); - if (nodeRepository.exclusiveAllocation(spec) && findNearestNodeResources(limits).isPresent()) + if (nodeRepository.exclusivity().allocation(spec) && findNearestNodeResources(limits).isPresent()) message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits).get()); return new IllegalArgumentException(message.toString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 194207ca69d..5eb8c2e7fd7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; @@ -27,7 +28,7 @@ public class NodeResourceLimits { /** Validates the resources applications ask for (which are in "advertised" resource space) */ public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ClusterSpec cluster) { - boolean exclusive = nodeRepository.exclusiveAllocation(cluster); + boolean exclusive = nodeRepository.exclusivity().allocation(cluster); if (! requested.vcpuIsUnspecified() && requested.vcpu() < minAdvertisedVcpu(cluster, exclusive)) illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster, exclusive)); if (! requested.memoryGbIsUnspecified() && requested.memoryGb() < minAdvertisedMemoryGb(cluster, exclusive)) @@ -104,7 +105,7 @@ public class NodeResourceLimits { } private double minRealVcpu(ClusterSpec cluster) { - return minAdvertisedVcpu(cluster, nodeRepository.exclusiveAllocation(cluster)); + return minAdvertisedVcpu(cluster, nodeRepository.exclusivity().allocation(cluster)); } private static double minRealMemoryGb(ClusterSpec cluster) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 0206c3a4a26..ec5c807c9b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -202,7 +202,7 @@ public class Preparer { nodeRepository.nodes(), nodeRepository.resourcesCalculator(), nodeRepository.spareCount(), - nodeRepository.exclusiveAllocation(cluster)); + nodeRepository.exclusivity().allocation(cluster)); allocation.offer(prioritizer.collect()); if (requested.type() == NodeType.tenant && !requested.canFail() && allocation.changes()) { // This should not happen and indicates a bug in the allocation code because boostrap redeployment @@ -237,8 +237,8 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : - nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + return nodeRepository.exclusivity().provisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusivity().allocation(cluster) ? HostSharing.exclusive : HostSharing.any; else return HostSharing.any; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 9d9771b3b0f..c085a3ed27c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; @@ -19,6 +20,7 @@ import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; import com.yahoo.config.provision.WireguardKeyWithTimestamp; @@ -90,6 +92,7 @@ public class MockNodeRepository extends NodeRepository { curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 79215028dba..fb96d6f798e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; @@ -40,16 +42,18 @@ public class NodeRepositoryTester { clock = new ManualClock(); curator = new MockCurator(); curator.setZooKeeperEnsembleConnectionSpec("server1:1234,server2:5678"); + var flagSource = new InMemoryFlagSource(); nodeRepository = new NodeRepository(nodeFlavors, new EmptyProvisionServiceProvider(), curator, clock, zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), Optional.empty(), - new InMemoryFlagSource(), + flagSource, new MemoryMetricsDb(clock), new OrchestratorMock(), true, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 8374c41ea25..e0c8199a882 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; @@ -162,7 +161,7 @@ public class AutoscalingTest { .awsProdSetup(false) .capacity(Capacity.from(min, max)) .initialResources(Optional.empty()) - .hostSharingFlag() + .hostSharing() .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 7, 1, 2.0, 10.0, 384.0, @@ -178,7 +177,7 @@ public class AutoscalingTest { .clusterType(ClusterSpec.Type.container) .capacity(Capacity.from(min, max)) .initialResources(Optional.empty()) - .hostSharingFlag() + .hostSharing() .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 1, 1, 0.5, 4.0, 10.0, @@ -389,7 +388,8 @@ public class AutoscalingTest { .build(); NodeResources defaultResources = - new CapacityPolicies(fixture.tester().nodeRepository()).specifyFully(NodeResources.unspecified(), fixture.clusterSpec, fixture.applicationId); + fixture.tester().nodeRepository().capacityPoliciesFor(fixture.applicationId) + .specifyFully(NodeResources.unspecified(), fixture.clusterSpec); fixture.tester().assertResources("Min number of nodes and default resources", 2, 1, defaultResources, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 4ce909fece3..f9f3ca9db8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -11,12 +11,10 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.custom.HostResources; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Application; @@ -51,7 +49,7 @@ public class Fixture { applicationId = builder.application; clusterSpec = builder.cluster; capacity = builder.capacity; - tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); + tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, builder.sharedHosts, hostCount); var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity; tester.deploy(builder.application, builder.cluster, deployCapacity); this.loader = new Loader(this); @@ -178,6 +176,7 @@ public class Fixture { HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone); final InMemoryFlagSource flagSource = new InMemoryFlagSource(); int hostCount = 0; + SharedHosts sharedHosts = SharedHosts.empty(); public Fixture.Builder zone(Zone zone) { this.zone = zone; @@ -283,9 +282,8 @@ public class Fixture { return this; } - public Fixture.Builder hostSharingFlag() { - var resources = new HostResources(8.0, 32.0, 100.0, 10.0, "fast", "local", null, 6, "x86_64"); - flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(resources)), SharedHost.class); + public Fixture.Builder hostSharing() { + sharedHosts = SharedHosts.ofConstant(true, false); return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index 1de8a766ea9..68303535f8c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.Exclusivity; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.json.Jackson; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -68,6 +70,7 @@ public class CapacityCheckerTester { curator, clock, zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index 9fefc9d34e1..dd43a27b0c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -7,12 +7,14 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; @@ -258,11 +260,13 @@ public class SpareCapacityMaintainerTest { private SpareCapacityMaintainerTester(int maxIterations) { NodeFlavors flavors = new NodeFlavors(new FlavorConfigBuilder().build()); ManualClock clock = new ManualClock(); + var zone = new Zone(Environment.prod, RegionName.from("us-east-3")); nodeRepository = new NodeRepository(flavors, new EmptyProvisionServiceProvider(), new MockCurator(), clock, - new Zone(Environment.prod, RegionName.from("us-east-3")), + zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 78a34326949..3a9d03ed6b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -207,9 +208,8 @@ public class DynamicProvisioningTest { private void assertHostSharing(Environment environment, ClusterSpec.Type clusterType, boolean expectShared) { Zone zone = new Zone(Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(), SystemName.Public, environment, RegionName.defaultName()); MockHostProvisioner hostProvisioner = new MockHostProvisioner(new NodeFlavors(ProvisioningTester.createConfig()).getFlavors(), nameResolver, 0); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone).hostProvisioner(hostProvisioner).nameResolver(nameResolver).sharedHosts(SharedHosts.ofConstant(true, false)).build(); tester.makeReadyHosts(2, new NodeResources(12, 12, 200, 12)); - tester.flagSource().withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(new HostResources(4.0, 16.0, 50.0, 0.3, "fast", "local", null, 10, "x86_64"))), SharedHost.class); ApplicationId application = applicationId(); ClusterSpec cluster = ClusterSpec.request(clusterType, ClusterSpec.Id.from("default")).vespaVersion("6.42").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 401b6d83651..7a6955de811 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -10,6 +11,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.Mutex; @@ -51,16 +53,15 @@ public class DynamicProvisioningTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; private final HostResourcesCalculator hostResourcesCalculator; - private final CapacityPolicies capacityPolicies; - public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { - this(zone, hostFlavors, resourcesCalculator, flagSource); + public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, SharedHosts sharedHosts, int hostCount) { + this(zone, hostFlavors, resourcesCalculator, flagSource, sharedHosts); for (Flavor flavor : hostFlavors) provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8); provisioningTester.activateTenantHosts(); } - private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource, SharedHosts sharedHosts) { MockHostProvisioner hostProvisioner = null; if (zone.cloud().dynamicProvisioning()) { hostProvisioner = new MockHostProvisioner(flavors); @@ -73,11 +74,11 @@ public class DynamicProvisioningTester { .resourcesCalculator(resourcesCalculator) .flagSource(flagSource) .hostProvisioner(hostProvisioner) + .sharedHosts(sharedHosts) .build(); hostResourcesCalculator = resourcesCalculator; autoscaler = new Autoscaler(nodeRepository()); - capacityPolicies = new CapacityPolicies(provisioningTester.nodeRepository()); } public InMemoryProvisionLogger provisionLogger() { return provisioningTester.provisionLogger(); } @@ -158,7 +159,8 @@ public class DynamicProvisioningTester { } public Autoscaling autoscale(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); + var capacityPolicies = provisioningTester.nodeRepository().capacityPoliciesFor(applicationId); + capacity = capacityPolicies.applyOn(capacity, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(cluster.id(), false, capacity); try (Mutex lock = nodeRepository().applications().lock(applicationId)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 7b690b880c2..9e6ae9f010c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -46,12 +47,14 @@ import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import static java.time.temporal.ChronoUnit.MILLIS; +import static java.util.Objects.requireNonNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -1114,12 +1117,12 @@ public class ProvisioningTest { new Version("6"), new NodeResources(1, 1, 1, 1)); assertThrows(NullPointerException.class, - () -> CapacityPolicies.versioned(spec.vespaVersion("5.0").build(), resources)); - assertEquals(new NodeResources(1, 1, 1, 1), CapacityPolicies.versioned(spec.vespaVersion("6.0").build(), resources)); - assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.0").build(), resources)); - assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.1").build(), resources)); - assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("8.0").build(), resources)); - assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources)); + () -> versioned(spec.vespaVersion("5.0").build(), resources)); + assertEquals(new NodeResources(1, 1, 1, 1), versioned(spec.vespaVersion("6.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), versioned(spec.vespaVersion("7.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), versioned(spec.vespaVersion("7.1").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), versioned(spec.vespaVersion("8.0").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), versioned(spec.vespaVersion("9.0").build(), resources)); } @Test @@ -1292,4 +1295,13 @@ public class ProvisioningTest { return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); } + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ + static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { + return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), + "no default resources applicable for " + spec + " among: " + resources) + .getValue(); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 4ec290dd7ba..390da7c719a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -8,12 +8,14 @@ import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -24,6 +26,7 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -92,7 +95,6 @@ public class ProvisioningTester { private final NodeRepository nodeRepository; private final HostProvisioner hostProvisioner; private final NodeRepositoryProvisioner provisioner; - private final CapacityPolicies capacityPolicies; private final InMemoryProvisionLogger provisionLogger; private final LoadBalancerServiceMock loadBalancerService; @@ -110,7 +112,8 @@ public class ProvisioningTester { LoadBalancerServiceMock loadBalancerService, FlagSource flagSource, int spareCount, - ManualClock clock) { + ManualClock clock, + SharedHosts sharedHosts) { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = clock; @@ -121,6 +124,7 @@ public class ProvisioningTester { curator, clock, zone, + new Exclusivity(zone, sharedHosts), nameResolver, containerImage, Optional.empty(), @@ -131,7 +135,6 @@ public class ProvisioningTester { true, spareCount); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, new MockMetric()); - this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new InMemoryProvisionLogger(); this.loadBalancerService = loadBalancerService; } @@ -159,14 +162,13 @@ public class ProvisioningTester { public NodeRepositoryProvisioner provisioner() { return provisioner; } public HostProvisioner hostProvisioner() { return hostProvisioner; } public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } - public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return nodeRepository.nodes().list(inState).owner(id); } public InMemoryFlagSource flagSource() { return (InMemoryFlagSource) nodeRepository.flagSource(); } public InMemoryProvisionLogger provisionLogger() { return provisionLogger; } public Node node(String hostname) { return nodeRepository.nodes().node(hostname).get(); } public int decideSize(Capacity capacity, ApplicationId application) { - return capacityPolicies.applyOn(capacity, application, false).minResources().nodes(); + return nodeRepository.capacityPoliciesFor(application).applyOn(capacity, false).minResources().nodes(); } public Node patchNode(Node node, UnaryOperator<Node> patcher) { @@ -661,6 +663,7 @@ public class ProvisioningTester { private int spareCount = 0; private ManualClock clock = new ManualClock(); private DockerImage defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); + private SharedHosts sharedHosts = SharedHosts.empty(); public Builder curator(Curator curator) { this.curator = curator; @@ -742,6 +745,11 @@ public class ProvisioningTester { return this; } + public Builder sharedHosts(SharedHosts sharedHosts) { + this.sharedHosts = sharedHosts; + return this; + } + private FlagSource defaultFlagSource() { return new InMemoryFlagSource(); } @@ -758,7 +766,8 @@ public class ProvisioningTester { new LoadBalancerServiceMock(), Optional.ofNullable(flagSource).orElse(defaultFlagSource()), spareCount, - clock); + clock, + sharedHosts); } private static FlavorsConfig asConfig(List<Flavor> flavors) { |