diff options
author | Ola Aunronning <olaa@yahooinc.com> | 2024-01-12 16:22:01 +0100 |
---|---|---|
committer | Ola Aunronning <olaa@yahooinc.com> | 2024-01-12 16:22:01 +0100 |
commit | e6f30b96ad5e1d32be4aa29db4c526f5ece50625 (patch) | |
tree | 9f5a1c9503d8e940a7141aa3dbdda87d2141cb37 /node-repository/src | |
parent | f2ded8dd8ebfc2c567fffea98b5e750ab1ed0da1 (diff) |
Store multiple resource suggestions
Diffstat (limited to 'node-repository/src')
13 files changed, 216 insertions, 48 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 606605ed1e4..4134ea337ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -34,6 +34,7 @@ public class Cluster { private final IntRange groupSize; private final boolean required; private final Autoscaling suggested; + private final List<Autoscaling> suggestions; private final Autoscaling target; private final ClusterInfo clusterInfo; private final BcpGroupInfo bcpGroupInfo; @@ -48,6 +49,7 @@ public class Cluster { IntRange groupSize, boolean required, Autoscaling suggested, + List<Autoscaling> suggestions, Autoscaling target, ClusterInfo clusterInfo, BcpGroupInfo bcpGroupInfo, @@ -59,6 +61,7 @@ public class Cluster { this.groupSize = Objects.requireNonNull(groupSize); this.required = required; this.suggested = Objects.requireNonNull(suggested); + this.suggestions = Objects.requireNonNull(suggestions); Objects.requireNonNull(target); if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources)) this.target = target.withResources(Optional.empty()); // Delete illegal target @@ -102,12 +105,21 @@ public class Cluster { */ public Autoscaling suggested() { return suggested; } + /** + * The list of suggested resources, which may or may not be within the min and max limits, + * or empty if there is currently no recorded suggestion. + * List is sorted by preference + */ + public List<Autoscaling> suggestions() { return suggestions; } + /** Returns true if there is a current suggestion and we should actually make this suggestion to users. */ public boolean shouldSuggestResources(ClusterResources currentResources) { - if (suggested.resources().isEmpty()) return false; - if (suggested.resources().get().isWithin(min, max)) return false; - if ( ! Autoscaler.worthRescaling(currentResources, suggested.resources().get())) return false; - return true; + if (suggestions.isEmpty()) return false; + return suggestions.stream().noneMatch(suggestion -> + suggestion.resources().isEmpty() + || suggestion.resources().get().isWithin(min, max) + || ! Autoscaler.worthRescaling(currentResources, suggestion.resources().get()) + ); } public ClusterInfo clusterInfo() { return clusterInfo; } @@ -131,19 +143,23 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); + suggested, suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggested(Autoscaling suggested) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + } + + public Cluster withSuggestions(List<Autoscaling> suggestions) { + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -157,7 +173,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -189,7 +205,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); + Autoscaling.empty(), List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index ff30f9d6163..1c160beadf4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -6,7 +6,10 @@ import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.Autoscaler.headroomRequiredToScaleDown; @@ -37,13 +40,26 @@ public class AllocationOptimizer { public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment, ClusterModel model, Limits limits) { + return findBestAllocations(loadAdjustment, model, limits).stream().findFirst(); + } + + /** + * Searches the space of possible allocations given a target relative load + * and (optionally) cluster limits and returns the best alternative. + * + * @return the best allocations sorted by preference, if there are any possible legal allocations, fulfilling the target + * fully or partially, within the limits + */ + public List<AllocatableResources> findBestAllocations(Load loadAdjustment, + ClusterModel model, + Limits limits) { if (limits.isEmpty()) limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), IntRange.empty()); else limits = atLeast(minimumNodes, limits).fullySpecified(model.current().clusterSpec(), nodeRepository, model.application().id()); - Optional<AllocatableResources> bestAllocation = Optional.empty(); + List<AllocatableResources> bestAllocations = new ArrayList<>(); var availableRealHostResources = nodeRepository.zone().cloud().dynamicProvisioning() ? nodeRepository.flavors().getFlavors().stream().map(flavor -> flavor.resources()).toList() : nodeRepository.nodes().list().hosts().stream().map(host -> host.flavor().resources()) @@ -65,11 +81,20 @@ public class AllocationOptimizer { model, nodeRepository); if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get(), model)) - bestAllocation = allocatableResources; + bestAllocations.add(allocatableResources.get()); } } - return bestAllocation; + return bestAllocations.stream() + .sorted((one, other) -> { + if (one.preferableTo(other, model)) + return -1; + else if (other.preferableTo(one, model)) { + return 1; + } + return 0; + }) + .limit(3) + .toList(); } /** Returns the max resources of a host one node may allocate. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 738abddc31a..40819e709de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling.Status; import java.time.Duration; +import java.util.List; /** * The autoscaler gives advice about what resources should be allocated to a cluster based on observed behavior. @@ -39,8 +40,14 @@ public class Autoscaler { * @param clusterNodes the list of all the active nodes in a cluster * @return scaling advice for this cluster */ - public Autoscaling suggest(Application application, Cluster cluster, NodeList clusterNodes) { - return autoscale(application, cluster, clusterNodes, Limits.empty()); + public List<Autoscaling> suggest(Application application, Cluster cluster, NodeList clusterNodes) { + var model = model(application, cluster, clusterNodes); + if (model.isEmpty() || ! model.isStable(nodeRepository)) return List.of(); + + var targets = allocationOptimizer.findBestAllocations(model.loadAdjustment(), model, Limits.empty()); + return targets.stream() + .map(target -> toAutoscaling(target, model)) + .toList(); } /** @@ -50,18 +57,8 @@ public class Autoscaler { * @return scaling advice for this cluster */ public Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes) { - return autoscale(application, cluster, clusterNodes, Limits.of(cluster)); - } - - private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { - var model = new ClusterModel(nodeRepository, - application, - clusterNodes.not().retired().clusterSpec(), - cluster, - clusterNodes, - new AllocatableResources(clusterNodes.not().retired(), nodeRepository), - nodeRepository.metricsDb(), - nodeRepository.clock()); + var limits = Limits.of(cluster); + var model = model(application, cluster, clusterNodes); if (model.isEmpty()) return Autoscaling.empty(); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) @@ -78,18 +75,33 @@ public class Autoscaler { if (target.isEmpty()) return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", model); - if (target.get().nodes() == 1) + return toAutoscaling(target.get(), model); + } + + private ClusterModel model(Application application, Cluster cluster, NodeList clusterNodes) { + return new ClusterModel(nodeRepository, + application, + clusterNodes.not().retired().clusterSpec(), + cluster, + clusterNodes, + new AllocatableResources(clusterNodes.not().retired(), nodeRepository), + nodeRepository.metricsDb(), + nodeRepository.clock()); + } + + private Autoscaling toAutoscaling(AllocatableResources target, ClusterModel model) { + if (target.nodes() == 1) return Autoscaling.dontScale(Status.unavailable, "Autoscaling is disabled in single node clusters", model); - if (! worthRescaling(model.current().realResources(), target.get().realResources())) { - if (target.get().fulfilment() < 0.9999999) + if (! worthRescaling(model.current().realResources(), target.realResources())) { + if (target.fulfilment() < 0.9999999) return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", model); else if ( ! model.safeToScaleDown() && model.idealLoad().any(v -> v < 1.0)) return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", model); else return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", model); } - return Autoscaling.scaleTo(target.get().advertisedResources(), model); + return Autoscaling.scaleTo(target.advertisedResources(), model); } /** Returns true if it is worthwhile to make the given resource change, false if it is too insignificant */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index fd93d202795..c2199de247c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -64,9 +65,9 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return true; var suggestion = autoscaler.suggest(application, cluster.get(), clusterNodes); - if (suggestion.status() == Autoscaling.Status.waiting) return true; - if ( ! shouldUpdateSuggestion(cluster.get().suggested(), suggestion)) return true; + if ( ! shouldUpdateSuggestion(cluster.get().suggestions(), suggestion)) + return true; // Wait only a short time for the lock to avoid interfering with change deployments try (Mutex lock = nodeRepository().applications().lock(applicationId, Duration.ofSeconds(1))) { applications().get(applicationId).ifPresent(a -> updateSuggestion(suggestion, clusterId, a, lock)); @@ -77,19 +78,28 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { } } - private boolean shouldUpdateSuggestion(Autoscaling currentSuggestion, Autoscaling newSuggestion) { - return currentSuggestion.resources().isEmpty() - || currentSuggestion.at().isBefore(nodeRepository().clock().instant().minus(Duration.ofDays(7))) - || (newSuggestion.resources().isPresent() && isHigher(newSuggestion.resources().get(), currentSuggestion.resources().get())); + private boolean shouldUpdateSuggestion(List<Autoscaling> currentSuggestions, List<Autoscaling> newSuggestions) { + // Only compare previous best suggestion with current best suggestion + var currentSuggestion = currentSuggestions.stream().findFirst(); + var newSuggestion = newSuggestions.stream().findFirst(); + + if (currentSuggestion.isEmpty()) return true; + if (newSuggestion.isEmpty()) return false; + + return newSuggestion.get().status() != Autoscaling.Status.waiting + && (currentSuggestion.get().resources().isEmpty() + || currentSuggestion.get().at().isBefore(nodeRepository().clock().instant().minus(Duration.ofDays(7))) + || (newSuggestion.get().resources().isPresent() && isHigher(newSuggestion.get().resources().get(), currentSuggestion.get().resources().get()))); } - private void updateSuggestion(Autoscaling autoscaling, + private void updateSuggestion(List<Autoscaling> suggestions, ClusterSpec.Id clusterId, Application application, Mutex lock) { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - applications().put(application.with(cluster.get().withSuggested(autoscaling)), lock); + applications().put(application.with(cluster.get().withSuggestions(suggestions) + .withSuggested(suggestions.stream().findFirst().orElse(Autoscaling.empty()))), lock); } private boolean isHigher(ClusterResources r1, ClusterResources r2) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 6f325700401..7aee0610051 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -56,6 +57,7 @@ public class ApplicationSerializer { private static final String groupSizeKey = "groupSize"; private static final String requiredKey = "required"; private static final String suggestedKey = "suggested"; + private static final String suggestionsKey = "suggestionsKey"; private static final String clusterInfoKey = "clusterInfo"; private static final String bcpDeadlineKey = "bcpDeadline"; private static final String hostTTLKey = "hostTTL"; @@ -140,6 +142,7 @@ public class ApplicationSerializer { toSlime(cluster.groupSize(), clusterObject.setObject(groupSizeKey)); clusterObject.setBool(requiredKey, cluster.required()); toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey)); + toSlime(cluster.suggestions(), clusterObject.setArray(suggestionsKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); if (! cluster.clusterInfo().isEmpty()) toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey)); @@ -156,12 +159,20 @@ public class ApplicationSerializer { intRangeFromSlime(clusterObject.field(groupSizeKey)), clusterObject.field(requiredKey).asBool(), autoscalingFromSlime(clusterObject.field(suggestedKey)), + suggestionsFromSlime(clusterObject.field(suggestionsKey)), autoscalingFromSlime(clusterObject.field(targetKey)), clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)), scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); } + private static void toSlime(List<Autoscaling> suggestions, Cursor suggestionsArray) { + suggestions.forEach(suggestion -> { + var suggestionObject = suggestionsArray.addObject(); + toSlime(suggestion, suggestionObject); + }); + } + private static void toSlime(Autoscaling autoscaling, Cursor autoscalingObject) { autoscalingObject.setString(statusKey, toAutoscalingStatusCode(autoscaling.status())); autoscalingObject.setString(descriptionKey, autoscaling.description()); @@ -227,6 +238,13 @@ public class ApplicationSerializer { metricsObject.field(cpuCostPerQueryKey).asDouble()); } + private static List<Autoscaling> suggestionsFromSlime(Inspector suggestionsObject) { + var suggestions = new ArrayList<Autoscaling>(); + if (!suggestionsObject.valid()) return suggestions; + suggestionsObject.traverse((ArrayTraverser) (id, suggestion) -> suggestions.add(autoscalingFromSlime(suggestion))); + return suggestions; + } + private static Autoscaling autoscalingFromSlime(Inspector autoscalingObject) { if ( ! autoscalingObject.valid()) return Autoscaling.empty(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 89853896104..0adddb33e6b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -66,13 +66,23 @@ public class ApplicationSerializer { if ( ! cluster.groupSize().isEmpty()) toSlime(cluster.groupSize(), clusterObject.setObject("groupSize")); toSlime(currentResources, clusterObject.setObject("current")); - if (cluster.shouldSuggestResources(currentResources)) + if (cluster.shouldSuggestResources(currentResources)) { toSlime(cluster.suggested(), clusterObject.setObject("suggested")); + toSlime(cluster.suggestions(), clusterObject.setArray("suggestions")); + + } toSlime(cluster.target(), clusterObject.setObject("target")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); clusterObject.setLong("scalingDuration", cluster.scalingDuration(nodes.clusterSpec()).toMillis()); } + private static void toSlime(List<Autoscaling> suggestions, Cursor autoscalingArray) { + suggestions.forEach(suggestion -> { + var autoscalingObject = autoscalingArray.addObject(); + toSlime(suggestion, autoscalingObject); + }); + } + private static void toSlime(Autoscaling autoscaling, Cursor autoscalingObject) { autoscalingObject.setString("status", autoscaling.status().name()); autoscalingObject.setString("description", autoscaling.description()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index d3b88997059..e7c9d1079fb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -233,6 +233,14 @@ public class MockNodeRepository extends NodeRepository { Load.zero(), Load.zero(), Autoscaling.Metrics.zero())); + cluster1 = cluster1.withSuggestions(List.of(new Autoscaling(Autoscaling.Status.unavailable, + "", + Optional.of(new ClusterResources(6, 2, + new NodeResources(3, 20, 100, 1))), + clock().instant(), + Load.zero(), + Load.zero(), + Autoscaling.Metrics.zero()))); cluster1 = cluster1.withTarget(new Autoscaling(Autoscaling.Status.unavailable, "", Optional.of(new ClusterResources(4, 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 4236f7ac968..830ff170a90 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -462,12 +462,12 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.01, 0.01, 0.01, 0, 0), 120); - Autoscaling suggestion = fixture.suggest(); + List<Autoscaling> suggestions = fixture.suggest(); fixture.tester().assertResources("Choosing the remote disk flavor as it has less disk", 2, 1, 3.0, 100.0, 10.0, - suggestion); + suggestions); assertEquals("Choosing the remote disk flavor as it has less disk", - StorageType.remote, suggestion.resources().get().nodeResources().storageType()); + StorageType.remote, suggestions.stream().findFirst().flatMap(Autoscaling::resources).get().nodeResources().storageType()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index df85ca4865f..4ce909fece3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -108,7 +108,7 @@ public class Fixture { } /** Compute an autoscaling suggestion for this. */ - public Autoscaling suggest() { + public List<Autoscaling> suggest() { return tester().suggest(applicationId, clusterSpec.id(), capacity.minResources(), capacity.maxResources()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index f8be27300fe..51297a88cad 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -78,6 +78,12 @@ public class ScalingSuggestionsMaintainerTest { assertEquals("7 nodes with [vcpu: 4.1, memory: 5.3 Gb, disk: 16.5 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app2, cluster2, tester).resources().get().toString()); + // Secondary suggestions + assertEquals("7 nodes with [vcpu: 3.7, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + suggestionsOf(app1, cluster1, tester).get(1).resources().get().toString()); + assertEquals("8 nodes with [vcpu: 3.6, memory: 4.7 Gb, disk: 14.2 Gb, bandwidth: 0.1 Gbps, architecture: any]", + suggestionsOf(app2, cluster2, tester).get(1).resources().get().toString()); + // Utilization goes way down tester.clock().advance(Duration.ofHours(13)); addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); @@ -97,7 +103,7 @@ public class ScalingSuggestionsMaintainerTest { tester.clock().advance(Duration.ofDays(3)); addMeasurements(0.7f, 0.7f, 0.7f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); - var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get(); + var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestions().stream().findFirst().flatMap(Autoscaling::resources).get(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofDays(2)); @@ -121,7 +127,11 @@ public class ScalingSuggestionsMaintainerTest { } private Autoscaling suggestionOf(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) { - return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get().suggested(); + return suggestionsOf(app, cluster, tester).get(0); + } + + private List<Autoscaling> suggestionsOf(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) { + return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get().suggestions(); } private boolean shouldSuggest(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 918a9043c93..90af6dca090 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -41,6 +41,7 @@ public class ApplicationSerializerTest { IntRange.empty(), true, Autoscaling.empty(), + List.of(), Autoscaling.empty(), ClusterInfo.empty(), BcpGroupInfo.empty(), @@ -60,6 +61,14 @@ public class ApplicationSerializerTest { new Load(0.1, 0.2, 0.3, 0.4, 0.5), new Load(0.4, 0.5, 0.6, 0.7, 0.8), new Autoscaling.Metrics(0.7, 0.8, 0.9)), + List.of(new Autoscaling(Autoscaling.Status.unavailable, + "", + Optional.of(new ClusterResources(20, 10, + new NodeResources(0.5, 4, 14, 16))), + Instant.ofEpochMilli(1234L), + new Load(0.1, 0.2, 0.3, 0.4, 0.5), + new Load(0.4, 0.5, 0.6, 0.7, 0.8), + new Autoscaling.Metrics(0.7, 0.8, 0.9))), new Autoscaling(Autoscaling.Status.insufficient, "Autoscaling status", Optional.of(new ClusterResources(10, 5, @@ -98,6 +107,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.groupSize(), serializedCluster.groupSize()); assertEquals(originalCluster.required(), serializedCluster.required()); assertEquals(originalCluster.suggested(), serializedCluster.suggested()); + assertEquals(originalCluster.suggestions(), serializedCluster.suggestions()); assertEquals(originalCluster.target(), serializedCluster.target()); assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index be2b2ca896a..6b6ef49fa5d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -143,6 +144,7 @@ public class DynamicProvisioningTester { cluster.groupSize(), cluster.required(), cluster.suggested(), + cluster.suggestions(), cluster.target(), cluster.clusterInfo(), cluster.bcpGroupInfo(), @@ -165,7 +167,7 @@ public class DynamicProvisioningTester { nodeRepository().nodes().list(Node.State.active).owner(applicationId)); } - public Autoscaling suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, + public List<Autoscaling> suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(clusterId, false, Capacity.from(min, max)); @@ -199,6 +201,14 @@ public class DynamicProvisioningTester { public ClusterResources assertResources(String message, int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, + List<Autoscaling> autoscaling) { + assertFalse(autoscaling.isEmpty()); + return assertResources(message, nodeCount, groupCount, approxCpu, approxMemory, approxDisk, autoscaling.get(0)); + } + + public ClusterResources assertResources(String message, + int nodeCount, int groupCount, + double approxCpu, double approxMemory, double approxDisk, Autoscaling autoscaling) { assertTrue("Resources should be present: " + message + " (" + autoscaling + ": " + autoscaling.status() + ")", autoscaling.resources().isPresent()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index 7b2cf1dc8e4..1fa7b1a813e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -80,6 +80,45 @@ "cpuCostPerQuery" : 0.0 } }, + "suggestions": [ + { + "at": 123, + "description": "", + "ideal": { + "cpu": 0.0, + "disk": 0.0, + "gpu": 0.0, + "gpuMemory": 0.0, + "memory": 0.0 + }, + "metrics": { + "cpuCostPerQuery": 0.0, + "growthRateHeadroom": 0.0, + "queryRate": 0.0 + }, + "peak": { + "cpu": 0.0, + "disk": 0.0, + "gpu": 0.0, + "gpuMemory": 0.0, + "memory": 0.0 + }, + "resources": { + "groups": 2, + "nodes": 6, + "resources": { + "architecture": "any", + "bandwidthGbps": 1.0, + "diskGb": 100.0, + "diskSpeed": "fast", + "memoryGb": 20.0, + "storageType": "any", + "vcpu": 3.0 + } + }, + "status": "unavailable" + } + ], "target" : { "status" : "unavailable", "description" : "", |