diff options
45 files changed, 466 insertions, 815 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index 9aaf0d365cd..11ae0845fb0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -37,8 +37,6 @@ public class ClusterResources { public boolean smallerThan(ClusterResources other) { if (this.nodes < other.nodes) return true; if (this.groups < other.groups) return true; - if (this.nodeResources.isUnspecified() || other.nodeResources.isUnspecified()) return false; - if ( ! this.nodeResources.justNumbers().satisfies(other.nodeResources.justNumbers())) return true; return false; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index bed36d0d5d2..05b604b263f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -241,8 +241,6 @@ public class NodeResources { return true; } - public boolean isUnspecified() { return this == unspecified; } - /** * Create this from serial form. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index a459cc2826f..d612e8b102f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -38,8 +38,6 @@ import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.restapi.NotFoundException; import java.time.Clock; @@ -94,7 +92,6 @@ public class NodeRepository extends AbstractComponent { private final Clock clock; private final Zone zone; private final NodeFlavors flavors; - private final HostResourcesCalculator resourcesCalculator; private final NameResolver nameResolver; private final OsVersions osVersions; private final InfrastructureVersions infrastructureVersions; @@ -108,37 +105,20 @@ public class NodeRepository extends AbstractComponent { * This will use the system time to make time-sensitive decisions */ @Inject - public NodeRepository(NodeRepositoryConfig config, - NodeFlavors flavors, - ProvisionServiceProvider provisionServiceProvider, - Curator curator, - Zone zone) { - this(flavors, - provisionServiceProvider.getHostResourcesCalculator(), - curator, - Clock.systemUTC(), - zone, - new DnsNameResolver(), - DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) { + this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); } /** * Creates a node repository from a zookeeper provider and a clock instance * which will be used for time-sensitive decisions. */ - public NodeRepository(NodeFlavors flavors, - HostResourcesCalculator resourcesCalculator, - Curator curator, - Clock clock, - Zone zone, - NameResolver nameResolver, - DockerImage dockerImage, - boolean useCuratorClientCache) { + public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver, + DockerImage dockerImage, boolean useCuratorClientCache) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; this.flavors = flavors; - this.resourcesCalculator = resourcesCalculator; this.nameResolver = nameResolver; this.osVersions = new OsVersions(this); this.infrastructureVersions = new InfrastructureVersions(db); @@ -182,12 +162,6 @@ public class NodeRepository extends AbstractComponent { /** Returns this node repo's view of the applications deployed to it */ public Applications applications() { return applications; } - public NodeFlavors flavors() { - return flavors; - } - - public HostResourcesCalculator resourcesCalculator() { return resourcesCalculator; } - // ---------------- Query API ---------------------------------------------------------------- /** @@ -354,6 +328,10 @@ public class NodeRepository extends AbstractComponent { return Collections.singletonList(getNodeAcl(node, candidates)); } + public NodeFlavors getAvailableFlavors() { + return flavors; + } + // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 847ec1290f6..15a5545bc2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -72,6 +72,19 @@ public class Cluster { return new Cluster(id, min, max, suggested, target); } + public NodeResources capAtLimits(NodeResources resources) { + resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu())); + resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb())); + resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb())); + return resources; + } + + private double between(double min, double max, double value) { + value = Math.max(min, value); + value = Math.min(max, value); + return value; + } + @Override public int hashCode() { return id.hashCode(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 6c143ab4bbd..2414bd95b85 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -1,20 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.List; -import java.util.Optional; /** * @author bratseth @@ -39,19 +33,9 @@ public class AllocatableClusterResources { private final double fulfilment; - /** Fake allocatable resources from requested capacity */ - public AllocatableClusterResources(ClusterResources requested, ClusterSpec.Type clusterType) { - this.advertisedResources = requested.nodeResources(); - this.realResources = requested.nodeResources(); // we don't know - this.nodes = requested.nodes(); - this.groups = requested.groups(); - this.clusterType = clusterType; - this.fulfilment = 1; - } - - public AllocatableClusterResources(List<Node> nodes, NodeRepository nodeRepository) { + public AllocatableClusterResources(List<Node> nodes, HostResourcesCalculator calculator) { this.advertisedResources = nodes.get(0).flavor().resources(); - this.realResources = nodeRepository.resourcesCalculator().realResourcesOf(nodes.get(0), nodeRepository); + this.realResources = calculator.realResourcesOf(nodes.get(0)); this.nodes = nodes.size(); this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); this.clusterType = nodes.get(0).allocation().get().membership().cluster().type(); @@ -101,12 +85,6 @@ public class AllocatableClusterResources { public int nodes() { return nodes; } public int groups() { return groups; } - - public int groupSize() { - // ceil: If the division does not produce a whole number we assume some node is missing - return (int)Math.ceil((double)nodes / groups); - } - public ClusterSpec.Type clusterType() { return clusterType; } public double cost() { return nodes * costOf(advertisedResources); } @@ -143,61 +121,4 @@ public class AllocatableClusterResources { (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : ""); } - /** - * Returns the best matching allocatable node resources given ideal node resources, - * or empty if none available within the limits. - */ - public static Optional<AllocatableClusterResources> from(ClusterResources resources, - ClusterSpec.Type clusterType, - Limits limits, - NodeRepository nodeRepository) { - NodeResources cappedNodeResources = limits.cap(resources.nodeResources()); - cappedNodeResources = new NodeResourceLimits(nodeRepository.zone()).enlargeToLegal(cappedNodeResources, clusterType); - - if (allowsHostSharing(nodeRepository.zone().cloud())) { - // return the requested resources, or empty if they cannot fit on existing hosts - for (Flavor flavor : nodeRepository.flavors().getFlavors()) { - if (flavor.resources().satisfies(cappedNodeResources)) - return Optional.of(new AllocatableClusterResources(resources.with(cappedNodeResources), - cappedNodeResources, - resources.nodeResources(), - clusterType)); - } - return Optional.empty(); - } - else { - // return the cheapest flavor satisfying the target resources, if any - Optional<AllocatableClusterResources> best = Optional.empty(); - for (Flavor flavor : nodeRepository.flavors().getFlavors()) { - NodeResources flavorResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); - if (flavor.resources().storageType() == NodeResources.StorageType.remote) { - flavor = flavor.with(FlavorOverrides.ofDisk(cappedNodeResources.diskGb())); - flavorResources = flavorResources.withDiskGb(cappedNodeResources.diskGb()); // TODO: Do this in resourcesCalculator - } - if ( ! between(limits.min().nodeResources(), limits.max().nodeResources(), flavorResources)) continue; - - var candidate = new AllocatableClusterResources(resources.with(flavor.resources()), - flavor, - resources.nodeResources(), - clusterType, - nodeRepository.resourcesCalculator()); - if (best.isEmpty() || candidate.preferableTo(best.get())) - best = Optional.of(candidate); - } - return best; - } - } - - private static boolean between(NodeResources min, NodeResources max, NodeResources r) { - if ( ! min.isUnspecified() && ! r.justNumbers().satisfies(min.justNumbers())) return false; - if ( ! max.isUnspecified() && ! max.justNumbers().satisfies(r.justNumbers())) return false; - return true; - } - - // TODO: Put this in zone config instead? - private static boolean allowsHostSharing(CloudName cloudName) { - if (cloudName.value().equals("aws")) return false; - return true; - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java deleted file mode 100644 index 8d26bb89959..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.util.Optional; - -/** - * A searcher of the space of possible allocation - * - * @author bratseth - */ -public class AllocationOptimizer { - - private final NodeRepository nodeRepository; - - public AllocationOptimizer(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - } - - /** - * An AllocationSearcher searches the space of possible allocations given a target - * and (optionally) cluster limits and returns the best alternative. - * - * @return the best allocation, if there are any possible legal allocations, fulfilling the target - * fully or partially, within the limits - */ - public Optional<AllocatableClusterResources> findBestAllocation(ResourceTarget target, - AllocatableClusterResources current, - Limits limits) { - Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(target, current, limits); i.hasNext(); ) { - var allocatableResources = AllocatableClusterResources.from(i.next(), current.clusterType(), limits, nodeRepository); - if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) - bestAllocation = allocatableResources; - } - return bestAllocation; - } - - /** - * Provides iteration over possible cluster resource allocations given a target total load - * and current groups/nodes allocation. - */ - private static class ResourceIterator { - - // The min and max nodes to consider when not using application supplied limits - private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 - private static final int maximumNodes = 150; - - // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component - // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. - // TODO: Measure this, and only take it into account with queries - private static final double fixedCpuCostFraction = 0.1; - - // Given state - private final Limits limits; - private final AllocatableClusterResources current; - private final ResourceTarget target; - - // Derived from the observed state - private final int nodeIncrement; - private final boolean singleGroupMode; - - // Iterator state - private int currentNodes; - - public ResourceIterator(ResourceTarget target, AllocatableClusterResources current, Limits limits) { - this.target = target; - this.current = current; - this.limits = limits; - - // What number of nodes is it effective to add or remove at the time from this cluster? - // This is the group size, since we (for now) assume the group size is decided by someone wiser than us - // and we decide the number of groups. - // The exception is when we only have one group, where we can add and remove single nodes in it. - singleGroupMode = current.groups() == 1; - nodeIncrement = singleGroupMode ? 1 : current.groupSize(); - - // Step to the right starting point - currentNodes = current.nodes(); - if (currentNodes < minNodes()) { // step up - while (currentNodes < minNodes() - && (singleGroupMode || currentNodes + nodeIncrement > current.groupSize())) // group level redundancy - currentNodes += nodeIncrement; - } - else { // step down - while (currentNodes - nodeIncrement >= minNodes() - && (singleGroupMode || currentNodes - nodeIncrement > current.groupSize())) // group level redundancy - currentNodes -= nodeIncrement; - } - } - - public ClusterResources next() { - ClusterResources next = resourcesWith(currentNodes); - currentNodes += nodeIncrement; - return next; - } - - public boolean hasNext() { - return currentNodes <= maxNodes(); - } - - private int minNodes() { - if (limits.isEmpty()) return minimumNodes; - if (singleGroupMode) return limits.min().nodes(); - return Math.max(limits.min().nodes(), limits.min().groups() * current.groupSize() ); - } - - private int maxNodes() { - if (limits.isEmpty()) return maximumNodes; - if (singleGroupMode) return limits.max().nodes(); - return Math.min(limits.max().nodes(), limits.max().groups() * current.groupSize() ); - } - - private ClusterResources resourcesWith(int nodes) { - int nodesAdjustedForRedundancy = nodes; - if (target.adjustForRedundancy()) - nodesAdjustedForRedundancy = nodes - (singleGroupMode ? 1 : current.groupSize()); - return new ClusterResources(nodes, - singleGroupMode ? 1 : nodes / current.groupSize(), - nodeResourcesWith(nodesAdjustedForRedundancy)); - } - - /** - * For the observed load this instance is initialized with, returns the resources needed per node to be at - * ideal load given a target node count - */ - private NodeResources nodeResourcesWith(int nodeCount) { - // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) - // Memory and disk: Scales with group size - - double cpu, memory, disk; - if (singleGroupMode) { - // The fixed cost portion of cpu does not scale with changes to the node count - // TODO: Only for the portion of cpu consumed by queries - cpu = fixedCpuCostFraction * target.clusterCpu() / current.groupSize() + - (1 - fixedCpuCostFraction) * target.clusterCpu() / nodeCount; - - if (current.clusterType().isContent()) { // load scales with node share of content - memory = target.groupMemory() / nodeCount; - disk = target.groupDisk() / nodeCount; - } - else { - memory = target.nodeMemory(); - disk = target.nodeDisk(); - } - } - else { - cpu = target.clusterCpu() / nodeCount; - if (current.clusterType().isContent()) { // load scales with node share of content - memory = target.groupMemory() / current.groupSize(); - disk = target.groupDisk() / current.groupSize(); - } - else { - memory = target.nodeMemory(); - disk = target.nodeDisk(); - } - } - - // Combine the scaled resource values computed here - // with the currently configured non-scaled values, given in the limits, if any - NodeResources nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() - ? current.toAdvertisedClusterResources().nodeResources() - : limits.min().nodeResources(); // min=max for non-scaled - return nonScaled.withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); - } - - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 8930bf34f4a..6dca9c9a796 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -1,11 +1,17 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.time.Duration; import java.util.List; @@ -35,14 +41,18 @@ public class Autoscaler { /** What difference factor for a resource is worth a reallocation? */ private static final double resourceDifferenceWorthReallocation = 0.1; + private final HostResourcesCalculator resourcesCalculator; private final NodeMetricsDb metricsDb; private final NodeRepository nodeRepository; - private final AllocationOptimizer allocationOptimizer; + private final NodeResourceLimits nodeResourceLimits; - public Autoscaler(NodeMetricsDb metricsDb, NodeRepository nodeRepository) { + public Autoscaler(HostResourcesCalculator resourcesCalculator, + NodeMetricsDb metricsDb, + NodeRepository nodeRepository) { + this.resourcesCalculator = resourcesCalculator; this.metricsDb = metricsDb; this.nodeRepository = nodeRepository; - this.allocationOptimizer = new AllocationOptimizer(nodeRepository); + this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone()); } /** @@ -53,41 +63,61 @@ public class Autoscaler { * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) { - return autoscale(clusterNodes, Limits.empty()) + return autoscale(cluster, clusterNodes, false) .map(AllocatableClusterResources::toAdvertisedClusterResources); } /** - * Autoscale a cluster by load. This returns a better allocation (if found) inside the min and max limits. + * Autoscale a cluster. This returns a better allocation (if found) inside the min and max limits. * * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut - return autoscale(clusterNodes, Limits.of(cluster)) + return autoscale(cluster, clusterNodes, true) .map(AllocatableClusterResources::toAdvertisedClusterResources); } - private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits) { + private Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes, boolean respectLimits) { if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository); + AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, resourcesCalculator); Optional<Double> cpuLoad = averageLoad(Resource.cpu, clusterNodes, clusterType); Optional<Double> memoryLoad = averageLoad(Resource.memory, clusterNodes, clusterType); Optional<Double> diskLoad = averageLoad(Resource.disk, clusterNodes, clusterType); if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty(); - var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation); - Optional<AllocatableClusterResources> bestAllocation = - allocationOptimizer.findBestAllocation(target, currentAllocation, limits); + Optional<AllocatableClusterResources> bestAllocation = findBestAllocation(cpuLoad.get(), + memoryLoad.get(), + diskLoad.get(), + currentAllocation, + cluster, + respectLimits); if (bestAllocation.isEmpty()) return Optional.empty(); if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; } + private Optional<AllocatableClusterResources> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, + AllocatableClusterResources currentAllocation, + Cluster cluster, boolean respectLimits) { + Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster, respectLimits); + i.hasNext(); ) { + Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(), + currentAllocation.clusterType(), + cluster, + respectLimits); + if (allocatableResources.isEmpty()) continue; + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) + bestAllocation = allocatableResources; + } + return bestAllocation; + } + /** Returns true if both total real resources and total cost are similar */ private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) { return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) && @@ -105,6 +135,51 @@ public class Autoscaler { } /** + * Returns the smallest allocatable node resources larger than the given node resources, + * or empty if none available. + */ + private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources, + ClusterSpec.Type clusterType, + Cluster cluster, + boolean respectLimits) { + NodeResources nodeResources = resources.nodeResources(); + if (respectLimits) + nodeResources = cluster.capAtLimits(nodeResources); + nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits + + if (allowsHostSharing(nodeRepository.zone().cloud())) { + // return the requested resources, or empty if they cannot fit on existing hosts + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { + if (flavor.resources().satisfies(nodeResources)) + return Optional.of(new AllocatableClusterResources(resources.with(nodeResources), + nodeResources, + resources.nodeResources(), + clusterType)); + } + return Optional.empty(); + } + else { + // return the cheapest flavor satisfying the target resources, if any + Optional<AllocatableClusterResources> best = Optional.empty(); + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { + if ( ! flavor.resources().satisfies(nodeResources)) continue; + + if (flavor.resources().storageType() == NodeResources.StorageType.remote) + flavor = flavor.with(FlavorOverrides.ofDisk(nodeResources.diskGb())); + var candidate = new AllocatableClusterResources(resources.with(flavor.resources()), + flavor, + resources.nodeResources(), + clusterType, + resourcesCalculator); + + if (best.isEmpty() || candidate.cost() <= best.get().cost()) + best = Optional.of(candidate); + } + return best; + } + } + + /** * Returns the average load of this resource in the measurement window, * or empty if we are not in a position to make decisions from these measurements at this time. */ @@ -125,6 +200,12 @@ public class Autoscaler { return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account } + // TODO: Put this in zone config instead? + private boolean allowsHostSharing(CloudName cloudName) { + if (cloudName.value().equals("aws")) return false; + return true; + } + public static boolean unstable(List<Node> nodes) { return nodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java deleted file mode 100644 index 7ca60c4c86d..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.applications.Cluster; - -/** - * Optional allocation limits - * - * @author bratseth - */ -public class Limits { - - private static final Limits empty = new Limits(null, null); - - private final ClusterResources min, max; - - private Limits(ClusterResources min, ClusterResources max) { - this.min = min; - this.max = max; - } - - public static Limits empty() { return empty; } - - public boolean isEmpty() { return this == empty; } - - public ClusterResources min() { - if (isEmpty()) throw new IllegalStateException("Empty: No min"); - return min; - } - - public ClusterResources max() { - if (isEmpty()) throw new IllegalStateException("Empty: No max"); - return max; - } - - /** Caps the given resources at the limits of this. If it is empty the node resources are returned as-is */ - public NodeResources cap(NodeResources resources) { - if (isEmpty()) return resources; - if (min.nodeResources().isUnspecified()) return resources; // means max is also unspecified - resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu())); - resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb())); - resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb())); - return resources; - } - - private double between(double min, double max, double value) { - value = Math.max(min, value); - value = Math.min(max, value); - return value; - } - - public static Limits of(Cluster cluster) { - return new Limits(cluster.minResources(), cluster.maxResources()); - } - - public static Limits of(Capacity capacity) { - return new Limits(capacity.minResources(), capacity.maxResources()); - } - - @Override - public String toString() { - if (isEmpty()) return "no limits"; - return "limits: from " + min + " to " + max; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java new file mode 100644 index 00000000000..207eecc1871 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -0,0 +1,155 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.applications.Cluster; + +/** + * Provides iteration over possible cluster resource allocations given a target total load + * and current groups/nodes allocation. + */ +public class ResourceIterator { + + // The min and max nodes to consider when not using application supplied limits + private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 + private static final int maximumNodes = 150; + + // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component + // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. + // TODO: Measure this, and only take it into account with queries + private static final double fixedCpuCostFraction = 0.1; + + // Prescribed state + private final Cluster cluster; + private final boolean respectLimits; + + // Observed state + private final AllocatableClusterResources allocation; + private final double cpuLoad; + private final double memoryLoad; + private final double diskLoad; + private final int groupSize; + + // Derived from the observed state + private final int nodeIncrement; + private final boolean singleGroupMode; + + // Iterator state + private int currentNodes; + + public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, + AllocatableClusterResources currentAllocation, + Cluster cluster, + boolean respectLimits) { + this.cpuLoad = cpuLoad; + this.memoryLoad = memoryLoad; + this.diskLoad = diskLoad; + this.respectLimits = respectLimits; + + // ceil: If the division does not produce a whole number we assume some node is missing + groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); + allocation = currentAllocation; + + this.cluster = cluster; + + // What number of nodes is it effective to add or remove at the time from this cluster? + // This is the group size, since we (for now) assume the group size is decided by someone wiser than us + // and we decide the number of groups. + // The exception is when we only have one group, where we can add and remove single nodes in it. + singleGroupMode = currentAllocation.groups() == 1; + nodeIncrement = singleGroupMode ? 1 : groupSize; + + // Step down to the right starting point + currentNodes = currentAllocation.nodes(); + while (currentNodes - nodeIncrement >= minNodes() + && ( singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy + currentNodes -= nodeIncrement; + } + + public ClusterResources next() { + ClusterResources next = resourcesWith(currentNodes); + currentNodes += nodeIncrement; + return next; + } + + public boolean hasNext() { + return currentNodes <= maxNodes(); + } + + private int minNodes() { + if ( ! respectLimits) return minimumNodes; + if (singleGroupMode) return cluster.minResources().nodes(); + return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); + } + + private int maxNodes() { + if ( ! respectLimits) return maximumNodes; + if (singleGroupMode) return cluster.maxResources().nodes(); + return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); + } + + private ClusterResources resourcesWith(int nodes) { + int nodesWithRedundancy = nodes - (singleGroupMode ? 1 : groupSize); + return new ClusterResources(nodes, + singleGroupMode ? 1 : nodes / groupSize, + nodeResourcesWith(nodesWithRedundancy)); + } + + /** + * For the observed load this instance is initialized with, returns the resources needed per node to be at + * ideal load given a target node count + */ + private NodeResources nodeResourcesWith(int nodeCount) { + // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) + // Memory and disk: Scales with group size + + double cpu, memory, disk; + if (singleGroupMode) { + // The fixed cost portion of cpu does not scale with changes to the node count + // TODO: Only for the portion of cpu consumed by queries + double totalCpu = clusterUsage(Resource.cpu, cpuLoad); + cpu = fixedCpuCostFraction * totalCpu / groupSize / Resource.cpu.idealAverageLoad() + + (1 - fixedCpuCostFraction) * totalCpu / nodeCount / Resource.cpu.idealAverageLoad(); + if (allocation.clusterType().isContent()) { // load scales with node share of content + memory = groupUsage(Resource.memory, memoryLoad) / nodeCount / Resource.memory.idealAverageLoad(); + disk = groupUsage(Resource.disk, diskLoad) / nodeCount / Resource.disk.idealAverageLoad(); + } + else { + memory = nodeUsage(Resource.memory, memoryLoad) / Resource.memory.idealAverageLoad(); + disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad(); + } + } + else { + cpu = clusterUsage(Resource.cpu, cpuLoad) / nodeCount / Resource.cpu.idealAverageLoad(); + if (allocation.clusterType().isContent()) { // load scales with node share of content + memory = groupUsage(Resource.memory, memoryLoad) / groupSize / Resource.memory.idealAverageLoad(); + disk = groupUsage(Resource.disk, diskLoad) / groupSize / Resource.disk.idealAverageLoad(); + } + else { + memory = nodeUsage(Resource.memory, memoryLoad) / Resource.memory.idealAverageLoad(); + disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad(); + } + } + + // Combine the scaled resource values computed here + // and the currently combined values of non-scaled resources + return new NodeResources(cpu, memory, disk, + cluster.minResources().nodeResources().bandwidthGbps(), + cluster.minResources().nodeResources().diskSpeed(), + cluster.minResources().nodeResources().storageType()); + } + + private double clusterUsage(Resource resource, double load) { + return nodeUsage(resource, load) * allocation.nodes(); + } + + private double groupUsage(Resource resource, double load) { + return nodeUsage(resource, load) * groupSize; + } + + private double nodeUsage(Resource resource, double load) { + return load * resource.valueFrom(allocation.realResources()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java deleted file mode 100644 index 287cd2ae86a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.NodeResources; - -/** - * A resource target to hit for the allocation optimizer. - * The target is measured in cpu, memory and disk per node in the allocation given by current. - */ -public class ResourceTarget { - - private final boolean adjustForRedundancy; - - /** The target resources per node, assuming the node assignment in current */ - private final double cpu, memory, disk; - - /** The current allocation leading to this target */ - private final AllocatableClusterResources current; - - private ResourceTarget(double cpu, double memory, double disk, - boolean adjustForRedundancy, - AllocatableClusterResources current) { - this.cpu = cpu; - this.memory = memory; - this.disk = disk; - this.adjustForRedundancy = adjustForRedundancy; - this.current = current; - } - - /** Are the target resources given by this including redundancy or not */ - public boolean adjustForRedundancy() { return adjustForRedundancy; } - - /** Returns the target total cpu to allocate to the entire cluster */ - public double clusterCpu() { return nodeCpu() * current.nodes(); } - - /** Returns the target total memory to allocate to each group */ - public double groupMemory() { return nodeMemory() * current.groupSize(); } - - /** Returns the target total disk to allocate to each group */ - public double groupDisk() { return nodeDisk() * current.groupSize(); } - - /** Returns the target cpu per node, in terms of the current allocation */ - public double nodeCpu() { return cpu; } - - /** Returns the target memory per node, in terms of the current allocation */ - public double nodeMemory() { return memory; } - - /** Returns the target disk per node, in terms of the current allocation */ - public double nodeDisk() { return disk; } - - private static double nodeUsage(Resource resource, double load, AllocatableClusterResources current) { - return load * resource.valueFrom(current.realResources()); - } - - /** Create a target of achieving ideal load given a current load */ - public static ResourceTarget idealLoad(double currentCpuLoad, double currentMemoryLoad, double currentDiskLoad, - AllocatableClusterResources current) { - return new ResourceTarget(nodeUsage(Resource.cpu, currentCpuLoad, current) / Resource.cpu.idealAverageLoad(), - nodeUsage(Resource.memory, currentMemoryLoad, current) / Resource.memory.idealAverageLoad(), - nodeUsage(Resource.disk, currentDiskLoad, current) / Resource.disk.idealAverageLoad(), - true, - current); - } - - /** Crete a target of preserving a current allocation */ - public static ResourceTarget preserve(AllocatableClusterResources current) { - return new ResourceTarget(current.realResources().vcpu(), - current.realResources().memoryGb(), - current.realResources().diskGb(), - false, - current); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index fa8e8375e23..da8a0a14171 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -34,12 +34,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private final Metric metric; public AutoscalingMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, NodeMetricsDb metricsDb, Deployer deployer, Metric metric, Duration interval) { super(nodeRepository, interval); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); this.metric = metric; this.deployer = deployer; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 85909daa240..5c899f74bdb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -44,15 +44,15 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabled; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - DynamicProvisioningMaintainer(NodeRepository nodeRepository, - Duration interval, - HostProvisioner hostProvisioner, - FlagSource flagSource) { + DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -111,7 +111,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { NodeResources resources = it.next(); removableHosts.stream() .filter(nodeRepository()::canAllocateTenantNodeTo) - .filter(host -> nodeRepository().resourcesCalculator().advertisedResourcesOf(host.flavor()).satisfies(resources)) + .filter(host -> hostResourcesCalculator.advertisedResourcesOf(host.flavor()).satisfies(resources)) .min(Comparator.comparingInt(n -> n.flavor().cost())) .ifPresent(host -> { removableHosts.remove(host); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index d388fb5a967..a50e6d10696 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -87,13 +87,13 @@ public class NodeRepositoryMaintenance extends AbstractComponent { loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); - rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); + rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); - autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); - scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval); + autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); + scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.scalingSuggestionsInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 7ffb541be2a..0ba3765f470 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -25,18 +25,21 @@ import java.util.Optional; public class Rebalancer extends NodeRepositoryMaintainer { private final Deployer deployer; + private final HostResourcesCalculator hostResourcesCalculator; private final Optional<HostProvisioner> hostProvisioner; private final Metric metric; private final Clock clock; public Rebalancer(Deployer deployer, NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, Optional<HostProvisioner> hostProvisioner, Metric metric, Clock clock, Duration interval) { super(nodeRepository, interval); this.deployer = deployer; + this.hostResourcesCalculator = hostResourcesCalculator; this.hostProvisioner = hostProvisioner; this.metric = metric; this.clock = clock; @@ -61,7 +64,7 @@ public class Rebalancer extends NodeRepositoryMaintainer { /** We do this here rather than in MetricsReporter because it is expensive and frequent updates are unnecessary */ private void updateSkewMetric(NodeList allNodes) { - DockerHostCapacity capacity = new DockerHostCapacity(allNodes, nodeRepository().resourcesCalculator()); + DockerHostCapacity capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); double totalSkew = 0; int hostCount = 0; for (Node host : allNodes.nodeType((NodeType.host)).state(Node.State.active)) { @@ -83,7 +86,7 @@ public class Rebalancer extends NodeRepositoryMaintainer { * Returns Move.none if no moves can be made to reduce skew. */ private Move findBestMove(NodeList allNodes) { - DockerHostCapacity capacity = new DockerHostCapacity(allNodes, nodeRepository().resourcesCalculator()); + DockerHostCapacity capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); Move bestMove = Move.none; for (Node node : allNodes.nodeType(NodeType.tenant).state(Node.State.active)) { if (node.parentHostname().isEmpty()) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index b68e8eacbaa..332126690be 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -30,10 +30,11 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { private final Autoscaler autoscaler; public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, NodeMetricsDb metricsDb, Duration interval) { super(nodeRepository, interval); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index a2bf83eb6c3..648bf52f455 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -46,7 +46,7 @@ public class CapacityPolicies { } public NodeResources decideNodeResources(NodeResources requested, Capacity capacity, ClusterSpec cluster) { - if (requested.isUnspecified()) + if (requested == NodeResources.unspecified) requested = defaultNodeResources(cluster.type()); ensureSufficientResources(requested, cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index da54d89b4e5..9566213bc91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -14,7 +13,7 @@ import java.util.Optional; */ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { - private final HostResourcesCalculator hostResourcesCalculator = new IdentityHostResourcesCalculator(); + private final HostResourcesCalculator hostResourcesCalculator = new NoopHostResourcesCalculator(); @Override public Optional<LoadBalancerService> getLoadBalancerService() { @@ -31,10 +30,10 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { return hostResourcesCalculator; } - private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { + public static class NoopHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources realResourcesOf(Node node, NodeRepository repository) { + public NodeResources realResourcesOf(Node node) { return node.flavor().resources(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 2a39cc333e0..8143076a3b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -30,14 +30,15 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabledFlag; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - public GroupPreparer(NodeRepository nodeRepository, - Optional<HostProvisioner> hostProvisioner, - FlagSource flagSource) { + public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -71,23 +72,18 @@ public class GroupPreparer { // Create a prioritized set of nodes LockedNodeList nodeList = nodeRepository.list(allocationLock); - NodePrioritizer prioritizer = new NodePrioritizer(nodeList, - application, - cluster, - requestedNodes, - spareCount, - wantedGroups, - nodeRepository.nameResolver(), - nodeRepository.resourcesCalculator(), - allocateFully); + NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, + spareCount, wantedGroups, nodeRepository.nameResolver(), + hostResourcesCalculator, allocateFully); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo); + // Allocate from the prioritized list NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, - highestIndex, nodeRepository.flavors(), + highestIndex, nodeRepository.getAvailableFlavors(), nodeRepository.zone(), nodeRepository.clock()); allocation.offer(prioritizer.prioritize()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index 096e58e963e..b26351062e6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; /** * Some cloud providers advertise that a certain amount of resources are available in a flavor @@ -17,7 +16,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; public interface HostResourcesCalculator { /** Nodes use advertised resources. This returns the real resources for the node. */ - NodeResources realResourcesOf(Node node, NodeRepository nodeRepository); + NodeResources realResourcesOf(Node node); /** Flavors use real resources. This returns the advertised resources of the flavor. */ NodeResources advertisedResourcesOf(Flavor flavor); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a7d83bbfad9..6f3f83c3349 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; - -import java.util.ArrayList; import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; @@ -197,8 +195,6 @@ public class NodePrioritizer { .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); } - public List<PrioritizableNode> nodes() { return new ArrayList<>(nodes.values()); } - /** * Convert a list of nodes to a list of node priorities. This includes finding, calculating * parameters to the priority sorting procedure. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 6f026254bcd..4000354243f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -22,10 +22,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; -import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; -import com.yahoo.vespa.hosted.provision.autoscale.Limits; -import com.yahoo.vespa.hosted.provision.autoscale.ResourceTarget; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -35,6 +31,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -50,7 +47,6 @@ public class NodeRepositoryProvisioner implements Provisioner { private static final int SPARE_CAPACITY_NONPROD = 0; private final NodeRepository nodeRepository; - private final AllocationOptimizer allocationOptimizer; private final CapacityPolicies capacityPolicies; private final Zone zone; private final Preparer preparer; @@ -65,7 +61,6 @@ public class NodeRepositoryProvisioner implements Provisioner { public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { this.nodeRepository = nodeRepository; - this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(zone); this.zone = zone; this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); @@ -99,7 +94,7 @@ public class NodeRepositoryProvisioner implements Provisioner { NodeResources resources; NodeSpec nodeSpec; if ( requested.type() == NodeType.tenant) { - ClusterResources target = decideTargetResources(application, cluster, requested); + ClusterResources target = decideTargetResources(application, cluster.id(), requested); int nodeCount = capacityPolicies.decideSize(target.nodes(), requested, cluster, application); resources = capacityPolicies.decideNodeResources(target.nodeResources(), requested, cluster); boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive()); @@ -136,39 +131,66 @@ public class NodeRepositoryProvisioner implements Provisioner { * Returns the target cluster resources, a value between the min and max in the requested capacity, * and updates the application store with the received min and max. */ - private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { + private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) { try (Mutex lock = nodeRepository.lock(applicationId)) { Application application = nodeRepository.applications().get(applicationId).orElse(new Application(applicationId)); - application = application.withClusterLimits(clusterSpec.id(), requested.minResources(), requested.maxResources()); + application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); - return application.clusters().get(clusterSpec.id()).targetResources() - .orElseGet(() -> currentResources(applicationId, clusterSpec, requested)); + return application.clusters().get(clusterId).targetResources() + .orElseGet(() -> currentResources(applicationId, clusterId, requested)); } } /** Returns the current resources of this cluster, or the closes */ private ClusterResources currentResources(ApplicationId applicationId, - ClusterSpec clusterSpec, + ClusterSpec.Id clusterId, Capacity requested) { List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active)) - .cluster(clusterSpec.id()) + .cluster(clusterId) .not().retired() .not().removable() .asList(); - AllocatableClusterResources currentResources = - nodes.isEmpty() ? new AllocatableClusterResources(requested.minResources(), clusterSpec.type()) // new deployment: Use min - : new AllocatableClusterResources(nodes, nodeRepository); - return ensureWithin(Limits.of(requested), currentResources); + if (nodes.isEmpty()) return requested.minResources(); // New deployment: Start at min + + long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + var currentResources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources()); + return ensureWithin(requested.minResources(), requested.maxResources(), currentResources); } /** Make the minimal adjustments needed to the current resources to stay within the limits */ - private ClusterResources ensureWithin(Limits limits, AllocatableClusterResources current) { - if (limits.isEmpty()) return current.toAdvertisedClusterResources(); - if (limits.min().equals(limits.max())) return limits.min(); + private ClusterResources ensureWithin(ClusterResources min, ClusterResources max, ClusterResources current) { + int nodes = between(min.nodes(), max.nodes(), current.nodes()); + int groups = between(min.groups(), max.groups(), current.groups()); + if (nodes % groups != 0) { + // That didn't work - try to preserve current group size instead. + // Rounding here is needed because a node may be missing due to node failing. + int currentGroupsSize = Math.round((float)current.nodes() / current.groups()); + nodes = currentGroupsSize * groups; + if (nodes != between(min.nodes(), max.nodes(), nodes)) { + // Give up: Use max + nodes = max.nodes(); + groups = max.groups(); + } + } + if (min.nodeResources() != NodeResources.unspecified && max.nodeResources() != NodeResources.unspecified) { + double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); + double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); + double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); + // Combine computed scaled resources with requested non-scaled resources (for which min=max) + NodeResources nodeResources = min.nodeResources().withVcpu(vcpu).withMemoryGb(memoryGb).withDiskGb(diskGb); + return new ClusterResources(nodes, groups, nodeResources); + } + else { + return new ClusterResources(nodes, groups, current.nodeResources()); + } + } + + private int between(int min, int max, int n) { + return Math.min(max, Math.max(min, n)); + } - return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits) - .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits)) - .toAdvertisedClusterResources(); + private double between(double min, double max, double n) { + return Math.min(max, Math.max(min, n)); } private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index f88caffa6c6..91c15cdb61b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -33,7 +33,7 @@ class Preparer { this.nodeRepository = nodeRepository; this.spareCount = spareCount; this.loadBalancerProvisioner = loadBalancerProvisioner; - this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, flagSource); + this.groupPreparer = new GroupPreparer(nodeRepository, hostProvisioner, hostResourcesCalculator, flagSource); } /** Prepare all required resources for the given application and cluster */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index f5a68f71a1c..6fafd496174 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -27,9 +27,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.time.Clock; import java.time.Instant; @@ -57,11 +55,7 @@ public class MockNodeRepository extends NodeRepository { * @param flavors flavors to have in node repo */ public MockNodeRepository(MockCurator curator, NodeFlavors flavors) { - super(flavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), - Zone.defaultZone(), + super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 9a02f65daf9..0d5950fe33a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -26,7 +26,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner) { - this(loadBalancerService, hostProvisioner, new EmptyProvisionServiceProvider().getHostResourcesCalculator()); + this(loadBalancerService, hostProvisioner, new EmptyProvisionServiceProvider.NoopHostResourcesCalculator()); } public MockProvisionServiceProvider(LoadBalancerService loadBalancerService, HostProvisioner hostProvisioner, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 88804576310..ddbbb8e4482 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -10,7 +10,6 @@ import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -34,11 +33,7 @@ public class NodeRepositoryTester { clock = new ManualClock(); curator = new MockCurator(); curator.setZooKeeperEnsembleConnectionSpec("server1:1234,server2:5678"); - nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - Zone.defaultZone(), + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 032375943c8..b4f6adc2d26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -35,7 +35,7 @@ public class AutoscalingIntegrationTest { NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), new OrchestratorMock(), new MockHttpClient(tester.clock())); - Autoscaler autoscaler = new Autoscaler(tester.nodeMetricsDb(), tester.nodeRepository()); + Autoscaler autoscaler = new Autoscaler(new MockHostResourcesCalculator(), tester.nodeMetricsDb(), tester.nodeRepository()); ApplicationId application1 = tester.applicationId("test1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "test"); @@ -123,4 +123,14 @@ public class AutoscalingIntegrationTest { } + private static class MockHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources realResourcesOf(Node node) { return node.flavor().resources(); } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); } + + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 39a848a17f5..f98d55511ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -269,9 +269,9 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 1, resources); - tester.addMeasurements(Resource.memory, 0.02f, 0.95f, 120, application1); + tester.addMeasurements(Resource.memory, 0.02f, 1f, 120, application1); tester.assertResources("Scaling down", - 6, 1, 2.8, 4.0, 95.0, + 6, 1, 3.0, 4.0, 100.0, tester.autoscale(application1, cluster1.id(), min, max)); } @@ -291,7 +291,7 @@ public class AutoscalingTest { ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); - // deploy (Why 103 Gb memory? See AutoscalingTester.MockHostResourcesCalculator + // deploy (Why 83 Gb memory? See AutoscalingTester.MockHostResourcesCalculator tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index a5b60ae6e16..49bb51c1d79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -47,28 +47,28 @@ class AutoscalingTester { /** Creates an autoscaling tester with a single host type ready */ public AutoscalingTester(NodeResources hostResources) { - this(new Zone(Environment.prod, RegionName.from("us-east")), List.of(new Flavor("hostFlavor", hostResources)), null); - provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); + this(new Zone(Environment.prod, RegionName.from("us-east")), null, null, asConfig(hostResources)); + provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); // "hostFlavor" generated by asConfig provisioningTester.deployZoneApp(); } public AutoscalingTester(Zone zone, List<Flavor> flavors) { this(zone, flavors, - new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true)); + new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true), + asConfig(flavors)); } - private AutoscalingTester(Zone zone, List<Flavor> flavors, FlagSource flagSource) { + private AutoscalingTester(Zone zone, List<Flavor> flavors, FlagSource flagSource, FlavorsConfig flavorsConfig) { provisioningTester = new ProvisioningTester.Builder().zone(zone) - .flavors(flavors) - .resourcesCalculator(new MockHostResourcesCalculator(zone)) + .flavorsConfig(flavorsConfig) .hostProvisioner(new MockHostProvisioner(flavors)) .flagSource(flagSource) .build(); hostResourcesCalculator = new MockHostResourcesCalculator(zone); db = new NodeMetricsDb(); - autoscaler = new Autoscaler(db, nodeRepository()); + autoscaler = new Autoscaler(hostResourcesCalculator, db, nodeRepository()); } public ApplicationId applicationId(String applicationName) { @@ -196,6 +196,31 @@ class AutoscalingTester { public NodeMetricsDb nodeMetricsDb() { return db; } + private static FlavorsConfig asConfig(NodeResources hostResources) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + b.flavor(asFlavorConfig("hostFlavor", hostResources)); + return b.build(); + } + + private static FlavorsConfig asConfig(List<Flavor> flavors) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + for (Flavor flavor : flavors) + b.flavor(asFlavorConfig(flavor.name(), flavor.resources())); + return b.build(); + } + + private static FlavorsConfig.Flavor.Builder asFlavorConfig(String flavorName, NodeResources resources) { + FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.minCpuCores(resources.vcpu()); + flavor.minMainMemoryAvailableGb(resources.memoryGb()); + flavor.minDiskAvailableGb(resources.diskGb()); + flavor.bandwidth(resources.bandwidthGbps() * 1000); + flavor.fastDisk(resources.diskSpeed().compatibleWith(NodeResources.DiskSpeed.fast)); + flavor.remoteStorage(resources.storageType().compatibleWith(NodeResources.StorageType.remote)); + return flavor; + } + private static class MockHostResourcesCalculator implements HostResourcesCalculator { private final Zone zone; @@ -205,7 +230,7 @@ class AutoscalingTester { } @Override - public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { + public NodeResources realResourcesOf(Node node) { if (zone.cloud().value().equals("aws")) return node.flavor().resources().withMemoryGb(node.flavor().resources().memoryGb() - 3); else diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 4dab064ce1e..8f8f8d0f38b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -56,6 +56,7 @@ public class AutoscalingMaintainerTest { NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); AutoscalingMaintainer maintainer = new AutoscalingMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), nodeMetricsDb, deployer, new TestMetric(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index b1f6eaea502..0bc4d2c65a1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -55,14 +54,8 @@ public class CapacityCheckerTester { CapacityCheckerTester() { Curator curator = new MockCurator(); NodeFlavors f = new NodeFlavors(new FlavorConfigBuilder().build()); - nodeRepository = new NodeRepository(f, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + nodeRepository = new NodeRepository(f, curator, clock, zone, new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); } private void updateCapacityChecker() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 6fca4b98409..ebec07fe5dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; @@ -71,15 +70,12 @@ public class DynamicProvisioningMaintainerTest { private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); - private static final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true) .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); - private final DynamicProvisioningMaintainer maintainer = - new DynamicProvisioningMaintainer(tester.nodeRepository, - Duration.ofDays(1), - hostProvisioner, - flagSource); + private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer( + tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource); @Test public void delegates_to_host_provisioner_and_writes_back_result() { @@ -216,12 +212,8 @@ public class DynamicProvisioningMaintainerTest { private final ManualClock clock = new ManualClock(); private final Zone zone = new Zone(CloudName.from("aws"), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); - private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - hostResourcesCalculator, - new MockCurator(), - clock, - zone, - new MockNameResolver().mockAnyLookup(), + private final NodeRepository nodeRepository = new NodeRepository( + nodeFlavors, new MockCurator(), clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 575d84c0129..17521261e1b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Report; import com.yahoo.vespa.hosted.provision.node.Reports; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -251,11 +250,7 @@ public class FailedExpirerTest { public FailureScenario(SystemName system, Environment environment) { Zone zone = new Zone(system, environment, RegionName.defaultName()); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 11df6146b06..664809dc3ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -37,11 +36,7 @@ public class MaintenanceTester { public final ManualClock clock = new ManualClock(Instant.ofEpochMilli(0L)); // determinism private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, + public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 6dfb404d81a..665dd74176d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -77,11 +76,7 @@ public class MetricsReporterTest { public void test_registered_metric() { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); Curator curator = new MockCurator(); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - Clock.systemUTC(), - Zone.defaultZone(), + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); @@ -143,11 +138,7 @@ public class MetricsReporterTest { public void docker_metrics() { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); Curator curator = new MockCurator(); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - Clock.systemUTC(), - Zone.defaultZone(), + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index e33cc9e655e..ab97de80418 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -75,14 +74,8 @@ public class NodeFailTester { private NodeFailTester() { clock = new ManualClock(); curator = new MockCurator(); - nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index e57d57d4c4c..eb2a1d4db68 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -54,11 +53,7 @@ public class OperatorChangeApplicationMaintainerTest { ManualClock clock = new ManualClock(); Curator curator = new MockCurator(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 8a2a69bb437..91e4899e079 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -25,7 +25,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -60,11 +59,7 @@ public class PeriodicApplicationMaintainerTest { Curator curator = new MockCurator(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); this.clock = new ManualClock(); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index e6609caa4bc..d25ae234f35 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -54,6 +54,7 @@ public class RebalancerTest { Rebalancer rebalancer = new Rebalancer(deployer, tester.nodeRepository(), + tester.identityHostResourcesCalculator(), Optional.empty(), metric, tester.clock(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 59514cb3c95..0fd967cad1b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -45,11 +44,7 @@ public class ReservationExpirerTest { public void ensure_reservation_times_out() { ManualClock clock = new ManualClock(); NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(flavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - Zone.defaultZone(), + NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index e57bae09280..7ece8cba65e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -64,14 +63,9 @@ public class RetiredExpirerTest { private final ManualClock clock = new ManualClock(); private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider().getHostResourcesCalculator(), - curator, - clock, - zone, + private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 59da88790d7..40892d80759 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -67,13 +67,14 @@ public class ScalingSuggestionsMaintainerTest { addMeasurements(Resource.disk, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), nodeMetricsDb, Duration.ofMinutes(1)); maintainer.maintain(); - assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", + assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().toString()); - assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps, storage type: remote]", + assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps]", tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().toString()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index c6e50555d81..8f207ff9531 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -12,12 +9,10 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -32,7 +27,6 @@ import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -57,7 +51,7 @@ public class DynamicDockerProvisionTest { ApplicationId application1 = tester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); - mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small")); + mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); List<HostSpec> hostSpec = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 4, 1, flavor); verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), flavor, application1); @@ -66,7 +60,7 @@ public class DynamicDockerProvisionTest { assertEquals(4, tester.nodeRepository().getNodes(NodeType.host, Node.State.provisioned).size()); assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size()); assertEquals(List.of("host-100-1", "host-101-1", "host-102-1", "host-103-1"), - hostSpec.stream().map(HostSpec::hostname).collect(Collectors.toList())); + hostSpec.stream().map(HostSpec::hostname).collect(Collectors.toList())); } @Test @@ -77,7 +71,7 @@ public class DynamicDockerProvisionTest { ApplicationId application = tester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); - mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small")); + mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(List.of(100, 101), flavor, application); } @@ -88,7 +82,7 @@ public class DynamicDockerProvisionTest { NodeResources flavor = new NodeResources(1, 4, 10, 1); List<Integer> expectedProvisionIndexes = List.of(100, 101); - mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("large")); + mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("large")); tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application); @@ -102,7 +96,7 @@ public class DynamicDockerProvisionTest { } tester.deployZoneApp(); - mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small")); + mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); // Verify there was only 1 call to provision hosts (during the first prepare) verify(hostProvisioner).provisionHosts(any(), any(), any()); @@ -148,92 +142,10 @@ public class DynamicDockerProvisionTest { assertTrue(indices.containsAll(IntStream.range(0, 10).boxed().collect(Collectors.toList()))); } - @Test - public void test_changing_limits_on_aws() { - List<Flavor> flavors = List.of(new Flavor("1x", new NodeResources(1, 10, 100, 0.1)), - new Flavor("2x", new NodeResources(2, 20, 200, 0.1)), - new Flavor("4x", new NodeResources(4, 40, 400, 0.1))); - - mockHostProvisioner(hostProvisioner, flavors.get(0)); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(CloudName.from("aws"), - SystemName.main, - Environment.prod, - RegionName.from("us-east"))) - .flavors(flavors) - .hostProvisioner(hostProvisioner) - .flagSource(flagSource) - .nameResolver(nameResolver) - .resourcesCalculator(new MockResourcesCalculator()) - .build(); - - tester.deployZoneApp(); - - ApplicationId app1 = tester.makeApplicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); - - // Limits where each number are within flavor limits but but which don't contain any flavor leads to an error - try { - tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 3.8, 20, 40), - resources(10, 5, 5, 25, 50))); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - // success - } - - // Initial deployment - tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 0.5, 5, 20), - resources(6, 3, 4, 20, 40))); - tester.assertNodes("Initial allocation at first actual flavor above min (except for disk)", - 4, 2, 1, 10, 20, - app1, cluster1); - - - // Move window above current allocation - tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 3.8, 20, 40), - resources(10, 5, 5, 45, 50))); - tester.assertNodes("New allocation at new smallest flavor above limits", - 8, 4, 4, 40, 40, - app1, cluster1); - - // Move window below current allocation - System.out.println("--------- Moving window down"); - tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), - resources(6, 3, 3, 25, 25))); - tester.assertNodes("New allocation at new max", - 6, 3, 2, 20, 25, - app1, cluster1); - - // Widening window lets us find a cheaper alternative - tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), - resources(8, 4, 4, 20, 30))); - tester.assertNodes("Cheaper allocation", - 8, 4, 1, 10, 25, - app1, cluster1); - - // Changing group size - tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 0.5, 5, 5), - resources(9, 3, 5, 20, 15))); - tester.assertNodes("Groups changed", - 6, 3, 1, 10, 15, - app1, cluster1); - - // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), - new ClusterResources(9, 3, NodeResources.unspecified))); - tester.assertNodes("Minimal allocation", - 6, 3, 1, 10, 15, - app1, cluster1); - } - private static ClusterSpec clusterSpec(String clusterId) { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build(); } - private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { - return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); - } - @SuppressWarnings("unchecked") private static void mockHostProvisioner(HostProvisioner hostProvisioner, Flavor hostFlavor) { doAnswer(invocation -> { @@ -245,20 +157,4 @@ public class DynamicDockerProvisionTest { }).when(hostProvisioner).provisionHosts(any(), any(), any()); } - private static class MockResourcesCalculator implements HostResourcesCalculator { - - @Override - public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { - if (node.type() == NodeType.host) return node.flavor().resources(); - return node.flavor().resources().withMemoryGb(node.flavor().resources().memoryGb() - 3); - } - - @Override - public NodeResources advertisedResourcesOf(Flavor flavor) { - if (flavor.isConfigured()) return flavor.resources(); - return flavor.resources().withMemoryGb(flavor.resources().memoryGb() + 3); - } - - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index c9eb6466cd7..463a3ef3fb9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -409,11 +408,8 @@ public class ProvisioningTest { @Test public void test_changing_limits() { - Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) - .flavors(List.of(hostFlavor)) - .build(); - tester.makeReadyHosts(30, hostFlavor.resources()).deployZoneApp(); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)).deployZoneApp(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -457,14 +453,7 @@ public class ProvisioningTest { tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 5), resources(9, 3, 12, 35, 15))); tester.assertNodes("Groups changed", - 6, 3, 8, 30, 10, - app1, cluster1); - - // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), - new ClusterResources(9, 3, NodeResources.unspecified))); - tester.assertNodes("No change", - 6, 3, 8, 30, 10, + 6, 3, 10, 30, 10, app1, cluster1); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 33f30a8ecd5..cbb15311867 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -80,26 +80,15 @@ public class ProvisioningTester { private int nextHost = 0; private int nextIP = 0; - public ProvisioningTester(Curator curator, - NodeFlavors nodeFlavors, - HostResourcesCalculator resourcesCalculator, - Zone zone, - NameResolver nameResolver, - Orchestrator orchestrator, - HostProvisioner hostProvisioner, - LoadBalancerServiceMock loadBalancerService, - FlagSource flagSource) { + public ProvisioningTester( + Curator curator, NodeFlavors nodeFlavors, Zone zone, NameResolver nameResolver, + Orchestrator orchestrator, HostProvisioner hostProvisioner, + LoadBalancerServiceMock loadBalancerService, FlagSource flagSource) { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); - this.nodeRepository = new NodeRepository(nodeFlavors, - resourcesCalculator, - curator, - clock, - zone, - nameResolver, - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver, + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); this.orchestrator = orchestrator; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); @@ -161,26 +150,7 @@ public class ProvisioningTester { } public Collection<HostSpec> activate(ApplicationId application, ClusterSpec cluster, Capacity capacity) { - List<HostSpec> preparedNodes = prepare(application, cluster, capacity, true); - - // Add ip addresses and activate parent host if necessary - for (HostSpec prepared : preparedNodes) { - Node node = nodeRepository.getNode(prepared.hostname()).get(); - if (node.ipConfig().primary().isEmpty()) { - node = node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())); - nodeRepository.write(node, nodeRepository.lock(node)); - } - if (node.parentHostname().isEmpty()) continue; - Node parent = nodeRepository.getNode(node.parentHostname().get()).get(); - if (parent.state() == Node.State.active) continue; - NestedTransaction t = new NestedTransaction(); - if (parent.ipConfig().primary().isEmpty()) - parent = parent.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); - nodeRepository.activate(List.of(parent), t); - t.commit(); - } - - return activate(application, preparedNodes); + return activate(application, prepare(application, cluster, capacity, true)); } public Collection<HostSpec> activate(ApplicationId application, Collection<HostSpec> hosts) { @@ -241,11 +211,10 @@ public class ProvisioningTester { assertEquals(explanation + ": Group count", groups, nodeList.stream().map(n -> n.allocation().get().membership().cluster().group().get()).distinct().count()); - for (Node node : nodeList) { - var expected = new NodeResources(vcpu, memory, disk, 0.1); - assertTrue(explanation + ": Resources: Expected " + expected + " but was " + node.flavor().resources(), - expected.compatibleWith(node.flavor().resources())); - } + for (Node node : nodeList) + assertEquals(explanation + ": Resources", + new NodeResources(vcpu, memory, disk, 0.1), + node.flavor().resources()); } public void fail(HostSpec host) { @@ -511,7 +480,6 @@ public class ProvisioningTester { private Curator curator; private FlavorsConfig flavorsConfig; - private HostResourcesCalculator resourcesCalculator = new EmptyProvisionServiceProvider().getHostResourcesCalculator(); private Zone zone; private NameResolver nameResolver; private Orchestrator orchestrator; @@ -529,16 +497,6 @@ public class ProvisioningTester { return this; } - public Builder flavors(List<Flavor> flavors) { - this.flavorsConfig = asConfig(flavors); - return this; - } - - public Builder resourcesCalculator(HostResourcesCalculator resourcesCalculator) { - this.resourcesCalculator = resourcesCalculator; - return this; - } - public Builder zone(Zone zone) { this.zone = zone; return this; @@ -581,40 +539,38 @@ public class ProvisioningTester { return orch; }); - return new ProvisioningTester(Optional.ofNullable(curator).orElseGet(MockCurator::new), - new NodeFlavors(Optional.ofNullable(flavorsConfig).orElseGet(ProvisioningTester::createConfig)), - resourcesCalculator, - Optional.ofNullable(zone).orElseGet(Zone::defaultZone), - Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()), - orchestrator, - hostProvisioner, - Optional.ofNullable(loadBalancerService).orElseGet(LoadBalancerServiceMock::new), - Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new)); + return new ProvisioningTester( + Optional.ofNullable(curator).orElseGet(MockCurator::new), + new NodeFlavors(Optional.ofNullable(flavorsConfig).orElseGet(ProvisioningTester::createConfig)), + Optional.ofNullable(zone).orElseGet(Zone::defaultZone), + Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()), + orchestrator, + hostProvisioner, + Optional.ofNullable(loadBalancerService).orElseGet(LoadBalancerServiceMock::new), + Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new)); } + } - private static FlavorsConfig asConfig(List<Flavor> flavors) { - FlavorsConfig.Builder b = new FlavorsConfig.Builder(); - for (Flavor flavor : flavors) - b.flavor(asFlavorConfig(flavor.name(), flavor.resources())); - return b.build(); - } + private static class NullProvisionLogger implements ProvisionLogger { + @Override public void log(Level level, String message) { } + } + + public IdentityHostResourcesCalculator identityHostResourcesCalculator() { + return new IdentityHostResourcesCalculator(); + } - private static FlavorsConfig.Flavor.Builder asFlavorConfig(String flavorName, NodeResources resources) { - FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); - flavor.name(flavorName); - flavor.minCpuCores(resources.vcpu()); - flavor.minMainMemoryAvailableGb(resources.memoryGb()); - flavor.minDiskAvailableGb(resources.diskGb()); - flavor.bandwidth(resources.bandwidthGbps() * 1000); - flavor.fastDisk(resources.diskSpeed().compatibleWith(NodeResources.DiskSpeed.fast)); - flavor.remoteStorage(resources.storageType().compatibleWith(NodeResources.StorageType.remote)); - return flavor; + private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources realResourcesOf(Node node) { + return node.flavor().resources(); } - } + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { + return flavor.resources(); + } - private static class NullProvisionLogger implements ProvisionLogger { - @Override public void log(Level level, String message) { } } } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 9c40e5ec54f..847be20e963 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -79,7 +79,6 @@ public abstract class Maintainer implements Runnable, AutoCloseable { protected Duration interval() { return interval; } /** Run this while holding the job lock */ - @SuppressWarnings("unused") public final void lockAndMaintain() { try (var lock = jobControl.lockJob(name())) { maintain(); |