diff options
15 files changed, 290 insertions, 46 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index e90a5f132ed..05aeae45ad9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -39,6 +39,10 @@ public class Application { public Map<ClusterSpec.Id, Cluster> clusters() { return clusters; } + public Optional<Cluster> cluster(ClusterSpec.Id id) { + return Optional.ofNullable(clusters.get(id)); + } + public Application with(Cluster cluster) { Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>(this.clusters); clusters.put(cluster.id(), cluster); @@ -52,7 +56,7 @@ public class Application { public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, min, max, Optional.empty()); + cluster = new Cluster(id, min, max, Optional.empty(), Optional.empty()); else cluster = cluster.withLimits(min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 914bc6c5a48..9e0a9f368a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -20,17 +20,19 @@ public class Cluster { private final ClusterSpec.Id id; private final ClusterResources min, max; + private final Optional<ClusterResources> suggested; private final Optional<ClusterResources> target; public Cluster(ClusterSpec.Id id, ClusterResources minResources, ClusterResources maxResources, + Optional<ClusterResources> suggestedResources, Optional<ClusterResources> targetResources) { this.id = Objects.requireNonNull(id); this.min = Objects.requireNonNull(minResources); this.max = Objects.requireNonNull(maxResources); + this.suggested = Objects.requireNonNull(suggestedResources); Objects.requireNonNull(targetResources); - if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources)) this.target = Optional.empty(); else @@ -47,20 +49,35 @@ public class Cluster { /** * Returns the computed resources (between min and max, inclusive) this cluster should - * have allocated at the moment, or empty if the system currently have no opinion on this. + * have allocated at the moment (whether or not it actually has it), + * or empty if the system currently has no target. */ public Optional<ClusterResources> targetResources() { return target; } + /** + * The suggested size of this cluster, which may or may not be within the min and max limits, + * or empty if there is currently no suggestion. + */ + public Optional<ClusterResources> suggestedResources() { return suggested; } + public Cluster withLimits(ClusterResources min, ClusterResources max) { - return new Cluster(id, min, max, target); + return new Cluster(id, min, max, suggested, target); + } + + public Cluster withSuggested(ClusterResources suggested) { + return new Cluster(id, min, max, Optional.of(suggested), target); + } + + public Cluster withoutSuggested() { + return new Cluster(id, min, max, Optional.empty(), target); } public Cluster withTarget(ClusterResources target) { - return new Cluster(id, min, max, Optional.of(target)); + return new Cluster(id, min, max, suggested, Optional.of(target)); } public Cluster withoutTarget() { - return new Cluster(id, min, max, Optional.empty()); + return new Cluster(id, min, max, suggested, Optional.empty()); } public NodeResources capAtLimits(NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 447d3494fbc..14d1dca3e81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -31,7 +31,6 @@ public class Autoscaler { - Scale group size - Consider taking spikes/variance into account - Measure observed regulation lag (startup+redistribution) and take it into account when deciding regulation observation window - - Test AutoscalingMaintainer - Scale by performance not just load+cost */ @@ -57,12 +56,28 @@ public class Autoscaler { } /** - * Autoscale a cluster + * Suggest a scaling of a cluster. This returns a better allocation (if found) + * without taking min and max limits into account. + * + * @param clusterNodes the list of all the active nodes in a cluster + * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time + */ + public Optional<AllocatableClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) { + return autoscale(cluster, clusterNodes, false); + } + + /** + * Autoscale a cluster. This returns a better allocation (if found) inside the min and max limits. * * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { + if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut + return autoscale(cluster, clusterNodes, true); + } + + private Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes, boolean respectLimits) { if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); @@ -76,7 +91,8 @@ public class Autoscaler { memoryLoad.get(), diskLoad.get(), currentAllocation, - cluster); + cluster, + respectLimits); if (bestAllocation.isEmpty()) return Optional.empty(); if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; @@ -84,12 +100,14 @@ public class Autoscaler { private Optional<AllocatableClusterResources> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, boolean respectLimits) { Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) { + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster, respectLimits); + i.hasNext(); ) { Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(), currentAllocation.clusterType(), - cluster); + cluster, + respectLimits); if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; @@ -119,9 +137,10 @@ public class Autoscaler { */ private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources, ClusterSpec.Type clusterType, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { NodeResources nodeResources = resources.nodeResources(); - if ( ! cluster.minResources().equals(cluster.maxResources())) // enforce application limits unless suggest mode + if (respectLimits) nodeResources = cluster.capAtLimits(nodeResources); nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index b7d5995884e..207eecc1871 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; */ public class ResourceIterator { - // Configured min and max nodes for suggestions for apps which have not activated autoscaling - private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2 + // The min and max nodes to consider when not using application supplied limits + private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 private static final int maximumNodes = 150; // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component @@ -22,6 +22,7 @@ public class ResourceIterator { // Prescribed state private final Cluster cluster; + private final boolean respectLimits; // Observed state private final AllocatableClusterResources allocation; @@ -39,10 +40,12 @@ public class ResourceIterator { public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { this.cpuLoad = cpuLoad; this.memoryLoad = memoryLoad; this.diskLoad = diskLoad; + this.respectLimits = respectLimits; // ceil: If the division does not produce a whole number we assume some node is missing groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); @@ -64,11 +67,6 @@ public class ResourceIterator { currentNodes -= nodeIncrement; } - /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */ - private boolean suggestMode() { - return cluster.minResources().equals(cluster.maxResources()); - } - public ClusterResources next() { ClusterResources next = resourcesWith(currentNodes); currentNodes += nodeIncrement; @@ -80,13 +78,13 @@ public class ResourceIterator { } private int minNodes() { - if (suggestMode()) return minimumNodes; + if ( ! respectLimits) return minimumNodes; if (singleGroupMode) return cluster.minResources().nodes(); return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); } private int maxNodes() { - if (suggestMode()) return maximumNodes; + if ( ! respectLimits) return maximumNodes; if (singleGroupMode) return cluster.maxResources().nodes(); return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index d17577af135..3b97195ad2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -34,7 +34,6 @@ public class AutoscalingMaintainer extends Maintainer { private final Autoscaler autoscaler; private final Deployer deployer; private final Metric metric; - private final Map<Pair<ApplicationId, ClusterSpec.Id>, Instant> lastLogged = new HashMap<>(); public AutoscalingMaintainer(NodeRepository nodeRepository, HostResourcesCalculator hostResourcesCalculator, @@ -68,17 +67,13 @@ public class AutoscalingMaintainer extends Maintainer { MaintenanceDeployment deployment) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)); Cluster cluster = application.clusters().get(clusterId); - if (cluster == null) return; // no information on limits for this cluster + if (cluster == null) return; + if (cluster.minResources().equals(cluster.maxResources())) return; Optional<AllocatableClusterResources> target = autoscaler.autoscale(cluster, clusterNodes); if (target.isEmpty()) return; // current resources are fine - if (cluster.minResources().equals(cluster.maxResources())) { // autoscaling is deactivated - logAutoscaling("Scaling suggestion for ", target.get(), applicationId, clusterId, clusterNodes); - } - else { - logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes); - autoscaleTo(target.get(), clusterId, application, deployment); - } + logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); + autoscaleTo(target.get(), clusterId, application, deployment); } private void autoscaleTo(AllocatableClusterResources target, @@ -90,20 +85,15 @@ public class AutoscalingMaintainer extends Maintainer { deployment.activate(); } - private void logAutoscaling(String prefix, - AllocatableClusterResources target, + private void logAutoscaling(AllocatableClusterResources target, ApplicationId application, ClusterSpec.Id clusterId, List<Node> clusterNodes) { - Instant lastLogTime = lastLogged.get(new Pair<>(application, clusterId)); - if (lastLogTime != null && lastLogTime.isAfter(nodeRepository().clock().instant().minus(Duration.ofHours(1)))) return; - int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - log.info(prefix + application + " " + clusterType + " " + clusterId + ":" + + log.info("Autoscaling " + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); - lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant()); } private String toString(int nodes, int groups, NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 054c273dc99..9e954e0a1dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -52,6 +52,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Rebalancer rebalancer; private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; private final AutoscalingMaintainer autoscalingMaintainer; + private final ScalingSuggestionsMaintainer scalingSuggestionsMaintainer; @SuppressWarnings("unused") @Inject @@ -92,6 +93,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); + scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.scalingSuggestionsInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); @@ -118,6 +120,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer.deconstruct(); nodeMetricsDbMaintainer.deconstruct(); autoscalingMaintainer.deconstruct(); + scalingSuggestionsMaintainer.deconstruct(); } private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() { @@ -160,6 +163,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration rebalancerInterval; private final Duration nodeMetricsCollectionInterval; private final Duration autoscalingInterval; + private final Duration scalingSuggestionsInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -182,6 +186,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancerInterval = Duration.ofMinutes(40); nodeMetricsCollectionInterval = Duration.ofMinutes(1); autoscalingInterval = Duration.ofMinutes(5); + scalingSuggestionsInterval = Duration.ofMinutes(31); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java new file mode 100644 index 00000000000..f80f2d11753 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Applications; +import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Maintainer computing scaling suggestions for all clusters + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainer extends Maintainer { + + private final Autoscaler autoscaler; + + public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Duration interval) { + super(nodeRepository, interval); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); + } + + @Override + protected void maintain() { + if ( ! nodeRepository().zone().environment().isProduction()) return; + + activeNodesByApplication().forEach((applicationId, nodes) -> suggest(applicationId, nodes)); + } + + private void suggest(ApplicationId application, List<Node> applicationNodes) { + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> + suggest(application, clusterId, clusterNodes)); + } + + private void suggest(ApplicationId applicationId, + ClusterSpec.Id clusterId, + List<Node> clusterNodes) { + Applications applications = nodeRepository().applications(); + Application application = applications.get(applicationId).orElse(new Application(applicationId)); + Cluster cluster = application.clusters().get(clusterId); + if (cluster == null) return; + Optional<AllocatableClusterResources> target = autoscaler.suggest(cluster, clusterNodes); + if (target.isEmpty()) return; + ClusterResources suggestion = target.get().toAdvertisedClusterResources(); + + try (Mutex lock = nodeRepository().lock(applicationId)) { + applications.get(applicationId).ifPresent(a -> a.cluster(clusterId).ifPresent(c -> + applications.put(a.with(c.withSuggested(suggestion)), lock))); + } + } + + private Map<ClusterSpec.Id, List<Node>> nodesByCluster(List<Node> applicationNodes) { + return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index f514cc20fe8..b1453a9729a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -37,6 +37,7 @@ public class ApplicationSerializer { private static final String clustersKey = "clusters"; private static final String minResourcesKey = "min"; private static final String maxResourcesKey = "max"; + private static final String suggestedResourcesKey = "suggested"; private static final String targetResourcesKey = "target"; private static final String nodesKey = "nodes"; private static final String groupsKey = "groups"; @@ -81,6 +82,7 @@ public class ApplicationSerializer { private static void toSlime(Cluster cluster, Cursor clusterObject) { toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey)); toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); } @@ -88,6 +90,7 @@ public class ApplicationSerializer { return new Cluster(ClusterSpec.Id.from(id), clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), + optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 103543917bc..5f1139fec2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -170,7 +170,7 @@ public class NodeRepositoryProvisioner implements Provisioner { // the non-numeric settings of the current min limit with the current numeric settings NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() .with(requested.minResources().nodeResources().diskSpeed()) - .with(requested.maxResources().nodeResources().storageType()); + .with(requested.minResources().nodeResources().storageType()); var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 7c6c4e2336b..b4f6adc2d26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -21,6 +21,9 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author bratseth + */ public class AutoscalingIntegrationTest { @Test @@ -55,8 +58,8 @@ public class AutoscalingIntegrationTest { Application application = tester.nodeRepository().applications().get(application1).orElse(new Application(application1)) .withClusterLimits(cluster1.id(), min, max); tester.nodeRepository().applications().put(application, tester.nodeRepository().lock(application1)); - var scaledResources = autoscaler.autoscale(application.clusters().get(cluster1.id()), - tester.nodeRepository().getNodes(application1)); + var scaledResources = autoscaler.suggest(application.clusters().get(cluster1.id()), + tester.nodeRepository().getNodes(application1)); assertTrue(scaledResources.isPresent()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 497a2a31ce5..4dcbcd10111 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -187,9 +187,24 @@ public class AutoscalingTest { tester.autoscale(application1, cluster1.id(), min, max)); } - /** This condition ensures we get recommendation suggestions when deactivated */ @Test - public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() { + public void testAutoscalingLimitsWhenMinEqualsMax() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = min; + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + assertTrue(tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + } + + @Test + public void testSuggestionsIgnoresLimits() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -203,7 +218,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1, cluster1.id(), min, max)); + tester.suggest(application1, cluster1.id(), min, max)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index c250a0a23b0..6eb3aaea66e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -163,6 +163,15 @@ class AutoscalingTester { nodeRepository().getNodes(applicationId, Node.State.active)); } + public Optional<AllocatableClusterResources> suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, + ClusterResources min, ClusterResources max) { + Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) + .withClusterLimits(clusterId, min, max); + nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); + return autoscaler.suggest(application.clusters().get(clusterId), + nodeRepository().getNodes(applicationId, Node.State.active)); + } + public AllocatableClusterResources assertResources(String message, int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java new file mode 100644 index 00000000000..40892d80759 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the scaling suggestions maintainer integration. + * The specific suggestions are not tested here. + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainerTest { + + @Test + public void testScalingSuggestionsMaintainer() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(); + + ApplicationId app2 = tester.makeApplicationId("app2"); + ClusterSpec cluster2 = tester.clusterSpec(); + + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + + tester.makeReadyNodes(20, "flt", NodeType.host, 8); + tester.deployZoneApp(); + + tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + false, true)); + tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), + false, true)); + + addMeasurements(Resource.cpu, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.cpu, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + + ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), + nodeMetricsDb, + Duration.ofMinutes(1)); + maintainer.maintain(); + + assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().toString()); + assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().toString()); + } + + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId, + NodeRepository nodeRepository, NodeMetricsDb db) { + List<Node> nodes = nodeRepository.getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + for (Node node : nodes) + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + nodeRepository.clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 1f07e26d045..c50805eebb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -28,10 +28,12 @@ public class ApplicationSerializerTest { clusters.add(new Cluster(ClusterSpec.Id.from("c1"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), + Optional.empty(), Optional.empty())); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), + Optional.of(new ClusterResources(20, 10, new NodeResources(0.5, 4, 14, 16))), Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))))); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -49,6 +51,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.id(), serializedCluster.id()); assertEquals(originalCluster.minResources(), serializedCluster.minResources()); assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); + assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index ab608bac2b4..e041a7b8b54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -53,6 +53,9 @@ }, { "name": "RetiredExpirer" + }, + { + "name":"ScalingSuggestionsMaintainer" } ], "inactive": [ |