Store multiple resource suggestions

13 files changed, 216 insertions, 48 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
index 606605ed1e4..4134ea337ab 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
@@ -34,6 +34,7 @@ public class Cluster {
     private final IntRange groupSize;
     private final boolean required;
     private final Autoscaling suggested;
+    private final List<Autoscaling> suggestions;
     private final Autoscaling target;
     private final ClusterInfo clusterInfo;
     private final BcpGroupInfo bcpGroupInfo;
@@ -48,6 +49,7 @@ public class Cluster {
                    IntRange groupSize,
                    boolean required,
                    Autoscaling suggested,
+                   List<Autoscaling> suggestions,
                    Autoscaling target,
                    ClusterInfo clusterInfo,
                    BcpGroupInfo bcpGroupInfo,
@@ -59,6 +61,7 @@ public class Cluster {
         this.groupSize = Objects.requireNonNull(groupSize);
         this.required = required;
         this.suggested = Objects.requireNonNull(suggested);
+        this.suggestions = Objects.requireNonNull(suggestions);
         Objects.requireNonNull(target);
         if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources))
             this.target = target.withResources(Optional.empty()); // Delete illegal target
@@ -102,12 +105,21 @@ public class Cluster {
      */
     public Autoscaling suggested() { return suggested; }
 
+    /**
+     * The list of suggested resources, which may or may not be within the min and max limits,
+     * or empty if there is currently no recorded suggestion.
+     * List is sorted by preference
+     */
+    public List<Autoscaling> suggestions() { return suggestions; }
+
     /** Returns true if there is a current suggestion and we should actually make this suggestion to users. */
     public boolean shouldSuggestResources(ClusterResources currentResources) {
-        if (suggested.resources().isEmpty()) return false;
-        if (suggested.resources().get().isWithin(min, max)) return false;
-        if ( ! Autoscaler.worthRescaling(currentResources, suggested.resources().get())) return false;
-        return true;
+        if (suggestions.isEmpty()) return false;
+        return suggestions.stream().noneMatch(suggestion ->
+                        suggestion.resources().isEmpty()
+                                || suggestion.resources().get().isWithin(min, max)
+                                || ! Autoscaler.worthRescaling(currentResources, suggestion.resources().get())
+        );
     }
 
     public ClusterInfo clusterInfo() { return clusterInfo; }
@@ -131,19 +143,23 @@ public class Cluster {
     public Cluster withConfiguration(boolean exclusive, Capacity capacity) {
         return new Cluster(id, exclusive,
                            capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(),
-                           suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents);
+                           suggested, suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents);
     }
 
     public Cluster withSuggested(Autoscaling suggested) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents);
+    }
+
+    public Cluster withSuggestions(List<Autoscaling> suggestions) {
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     public Cluster withTarget(Autoscaling target) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     public Cluster with(BcpGroupInfo bcpGroupInfo) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     /** Add or update (based on "at" time) a scaling event */
@@ -157,7 +173,7 @@ public class Cluster {
             scalingEvents.add(scalingEvent);
 
         prune(scalingEvents);
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     @Override
@@ -189,7 +205,7 @@ public class Cluster {
     public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) {
         return new Cluster(id, exclusive,
                            requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(),
-                           Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of());
+                           Autoscaling.empty(), List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of());
     }
 
     /** The predicted time it will take to rescale this cluster. */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
index ff30f9d6163..1c160beadf4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
@@ -6,7 +6,10 @@ import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.provision.autoscale.Autoscaler.headroomRequiredToScaleDown;
 
@@ -37,13 +40,26 @@ public class AllocationOptimizer {
     public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment,
                                                              ClusterModel model,
                                                              Limits limits) {
+        return findBestAllocations(loadAdjustment, model, limits).stream().findFirst();
+    }
+
+    /**
+     * Searches the space of possible allocations given a target relative load
+     * and (optionally) cluster limits and returns the best alternative.
+     *
+     * @return the best allocations sorted by preference, if there are any possible legal allocations, fulfilling the target
+     *         fully or partially, within the limits
+     */
+    public List<AllocatableResources> findBestAllocations(Load loadAdjustment,
+                                                         ClusterModel model,
+                                                         Limits limits) {
         if (limits.isEmpty())
             limits = Limits.of(new ClusterResources(minimumNodes,    1, NodeResources.unspecified()),
                                new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()),
                                IntRange.empty());
         else
             limits = atLeast(minimumNodes, limits).fullySpecified(model.current().clusterSpec(), nodeRepository, model.application().id());
-        Optional<AllocatableResources> bestAllocation = Optional.empty();
+        List<AllocatableResources> bestAllocations = new ArrayList<>();
         var availableRealHostResources = nodeRepository.zone().cloud().dynamicProvisioning()
                                          ? nodeRepository.flavors().getFlavors().stream().map(flavor -> flavor.resources()).toList()
                                          : nodeRepository.nodes().list().hosts().stream().map(host -> host.flavor().resources())
@@ -65,11 +81,20 @@ public class AllocationOptimizer {
                                                                      model,
                                                                      nodeRepository);
                 if (allocatableResources.isEmpty()) continue;
-                if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get(), model))
-                    bestAllocation = allocatableResources;
+                bestAllocations.add(allocatableResources.get());
             }
         }
-        return bestAllocation;
+        return bestAllocations.stream()
+                .sorted((one, other) -> {
+                    if (one.preferableTo(other, model))
+                        return -1;
+                    else if (other.preferableTo(one, model)) {
+                        return 1;
+                    }
+                    return 0;
+                })
+                .limit(3)
+                .toList();
     }
 
     /** Returns the max resources of a host one node may allocate. */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index 738abddc31a..40819e709de 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling.Status;
 
 import java.time.Duration;
+import java.util.List;
 
 /**
  * The autoscaler gives advice about what resources should be allocated to a cluster based on observed behavior.
@@ -39,8 +40,14 @@ public class Autoscaler {
      * @param clusterNodes the list of all the active nodes in a cluster
      * @return scaling advice for this cluster
      */
-    public Autoscaling suggest(Application application, Cluster cluster, NodeList clusterNodes) {
-        return autoscale(application, cluster, clusterNodes, Limits.empty());
+    public List<Autoscaling> suggest(Application application, Cluster cluster, NodeList clusterNodes) {
+        var model = model(application, cluster, clusterNodes);
+        if (model.isEmpty() || ! model.isStable(nodeRepository)) return List.of();
+
+        var targets = allocationOptimizer.findBestAllocations(model.loadAdjustment(), model, Limits.empty());
+        return targets.stream()
+                .map(target -> toAutoscaling(target, model))
+                .toList();
     }
 
     /**
@@ -50,18 +57,8 @@ public class Autoscaler {
      * @return scaling advice for this cluster
      */
     public Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes) {
-        return autoscale(application, cluster, clusterNodes, Limits.of(cluster));
-    }
-
-    private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) {
-        var model = new ClusterModel(nodeRepository,
-                                     application,
-                                     clusterNodes.not().retired().clusterSpec(),
-                                     cluster,
-                                     clusterNodes,
-                                     new AllocatableResources(clusterNodes.not().retired(), nodeRepository),
-                                     nodeRepository.metricsDb(),
-                                     nodeRepository.clock());
+        var limits = Limits.of(cluster);
+        var model = model(application, cluster, clusterNodes);
         if (model.isEmpty()) return Autoscaling.empty();
 
         if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources()))
@@ -78,18 +75,33 @@ public class Autoscaler {
         if (target.isEmpty())
             return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", model);
 
-        if (target.get().nodes() == 1)
+       return toAutoscaling(target.get(), model);
+    }
+
+    private ClusterModel model(Application application, Cluster cluster, NodeList clusterNodes) {
+        return new ClusterModel(nodeRepository,
+                application,
+                clusterNodes.not().retired().clusterSpec(),
+                cluster,
+                clusterNodes,
+                new AllocatableResources(clusterNodes.not().retired(), nodeRepository),
+                nodeRepository.metricsDb(),
+                nodeRepository.clock());
+    }
+
+    private Autoscaling toAutoscaling(AllocatableResources target, ClusterModel model) {
+        if (target.nodes() == 1)
             return Autoscaling.dontScale(Status.unavailable, "Autoscaling is disabled in single node clusters", model);
 
-        if (! worthRescaling(model.current().realResources(), target.get().realResources())) {
-            if (target.get().fulfilment() < 0.9999999)
+        if (! worthRescaling(model.current().realResources(), target.realResources())) {
+            if (target.fulfilment() < 0.9999999)
                 return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", model);
             else if ( ! model.safeToScaleDown() && model.idealLoad().any(v -> v < 1.0))
                 return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", model);
             else
                 return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", model);
         }
-        return Autoscaling.scaleTo(target.get().advertisedResources(), model);
+        return Autoscaling.scaleTo(target.advertisedResources(), model);
     }
 
     /** Returns true if it is worthwhile to make the given resource change, false if it is too insignificant */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
index fd93d202795..c2199de247c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
 
 import java.time.Duration;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -64,9 +65,9 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer {
         Optional<Cluster> cluster = application.cluster(clusterId);
         if (cluster.isEmpty()) return true;
         var suggestion = autoscaler.suggest(application, cluster.get(), clusterNodes);
-        if (suggestion.status() == Autoscaling.Status.waiting) return true;
-        if ( ! shouldUpdateSuggestion(cluster.get().suggested(), suggestion)) return true;
 
+        if ( ! shouldUpdateSuggestion(cluster.get().suggestions(), suggestion))
+            return true;
         // Wait only a short time for the lock to avoid interfering with change deployments
         try (Mutex lock = nodeRepository().applications().lock(applicationId, Duration.ofSeconds(1))) {
             applications().get(applicationId).ifPresent(a -> updateSuggestion(suggestion, clusterId, a, lock));
@@ -77,19 +78,28 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer {
         }
     }
 
-    private boolean shouldUpdateSuggestion(Autoscaling currentSuggestion, Autoscaling newSuggestion) {
-        return currentSuggestion.resources().isEmpty()
-               || currentSuggestion.at().isBefore(nodeRepository().clock().instant().minus(Duration.ofDays(7)))
-               || (newSuggestion.resources().isPresent() && isHigher(newSuggestion.resources().get(), currentSuggestion.resources().get()));
+    private boolean shouldUpdateSuggestion(List<Autoscaling> currentSuggestions, List<Autoscaling> newSuggestions) {
+        // Only compare previous best suggestion with current best suggestion
+        var currentSuggestion = currentSuggestions.stream().findFirst();
+        var newSuggestion = newSuggestions.stream().findFirst();
+
+        if (currentSuggestion.isEmpty()) return true;
+        if (newSuggestion.isEmpty()) return false;
+
+        return newSuggestion.get().status() != Autoscaling.Status.waiting
+               && (currentSuggestion.get().resources().isEmpty()
+               || currentSuggestion.get().at().isBefore(nodeRepository().clock().instant().minus(Duration.ofDays(7)))
+               || (newSuggestion.get().resources().isPresent() && isHigher(newSuggestion.get().resources().get(), currentSuggestion.get().resources().get())));
     }
 
-    private void updateSuggestion(Autoscaling autoscaling,
+    private void updateSuggestion(List<Autoscaling> suggestions,
                                   ClusterSpec.Id clusterId,
                                   Application application,
                                   Mutex lock) {
         Optional<Cluster> cluster = application.cluster(clusterId);
         if (cluster.isEmpty()) return;
-        applications().put(application.with(cluster.get().withSuggested(autoscaling)), lock);
+        applications().put(application.with(cluster.get().withSuggestions(suggestions)
+                .withSuggested(suggestions.stream().findFirst().orElse(Autoscaling.empty()))), lock);
     }
 
     private boolean isHigher(ClusterResources r1, ClusterResources r2) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
index 6f325700401..7aee0610051 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
@@ -56,6 +57,7 @@ public class ApplicationSerializer {
     private static final String groupSizeKey = "groupSize";
     private static final String requiredKey = "required";
     private static final String suggestedKey = "suggested";
+    private static final String suggestionsKey = "suggestionsKey";
     private static final String clusterInfoKey = "clusterInfo";
     private static final String bcpDeadlineKey = "bcpDeadline";
     private static final String hostTTLKey = "hostTTL";
@@ -140,6 +142,7 @@ public class ApplicationSerializer {
         toSlime(cluster.groupSize(), clusterObject.setObject(groupSizeKey));
         clusterObject.setBool(requiredKey, cluster.required());
         toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey));
+        toSlime(cluster.suggestions(), clusterObject.setArray(suggestionsKey));
         toSlime(cluster.target(), clusterObject.setObject(targetKey));
         if (! cluster.clusterInfo().isEmpty())
             toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey));
@@ -156,12 +159,20 @@ public class ApplicationSerializer {
                            intRangeFromSlime(clusterObject.field(groupSizeKey)),
                            clusterObject.field(requiredKey).asBool(),
                            autoscalingFromSlime(clusterObject.field(suggestedKey)),
+                           suggestionsFromSlime(clusterObject.field(suggestionsKey)),
                            autoscalingFromSlime(clusterObject.field(targetKey)),
                            clusterInfoFromSlime(clusterObject.field(clusterInfoKey)),
                            bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)),
                            scalingEventsFromSlime(clusterObject.field(scalingEventsKey)));
     }
 
+    private static void toSlime(List<Autoscaling> suggestions, Cursor suggestionsArray) {
+        suggestions.forEach(suggestion -> {
+            var suggestionObject = suggestionsArray.addObject();
+            toSlime(suggestion, suggestionObject);
+        });
+    }
+
     private static void toSlime(Autoscaling autoscaling, Cursor autoscalingObject) {
         autoscalingObject.setString(statusKey, toAutoscalingStatusCode(autoscaling.status()));
         autoscalingObject.setString(descriptionKey, autoscaling.description());
@@ -227,6 +238,13 @@ public class ApplicationSerializer {
                                        metricsObject.field(cpuCostPerQueryKey).asDouble());
     }
 
+    private static List<Autoscaling> suggestionsFromSlime(Inspector suggestionsObject) {
+        var suggestions = new ArrayList<Autoscaling>();
+        if (!suggestionsObject.valid()) return suggestions;
+        suggestionsObject.traverse((ArrayTraverser) (id, suggestion) -> suggestions.add(autoscalingFromSlime(suggestion)));
+        return suggestions;
+    }
+
     private static Autoscaling autoscalingFromSlime(Inspector autoscalingObject) {
         if ( ! autoscalingObject.valid()) return Autoscaling.empty();
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
index 89853896104..0adddb33e6b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
@@ -66,13 +66,23 @@ public class ApplicationSerializer {
         if ( ! cluster.groupSize().isEmpty())
             toSlime(cluster.groupSize(), clusterObject.setObject("groupSize"));
         toSlime(currentResources, clusterObject.setObject("current"));
-        if (cluster.shouldSuggestResources(currentResources))
+        if (cluster.shouldSuggestResources(currentResources)) {
             toSlime(cluster.suggested(), clusterObject.setObject("suggested"));
+            toSlime(cluster.suggestions(), clusterObject.setArray("suggestions"));
+
+        }
         toSlime(cluster.target(), clusterObject.setObject("target"));
         scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents"));
         clusterObject.setLong("scalingDuration", cluster.scalingDuration(nodes.clusterSpec()).toMillis());
     }
 
+    private static void toSlime(List<Autoscaling> suggestions, Cursor autoscalingArray) {
+        suggestions.forEach(suggestion -> {
+                var autoscalingObject = autoscalingArray.addObject();
+                toSlime(suggestion, autoscalingObject);
+        });
+    }
+
     private static void toSlime(Autoscaling autoscaling, Cursor autoscalingObject) {
         autoscalingObject.setString("status", autoscaling.status().name());
         autoscalingObject.setString("description", autoscaling.description());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index d3b88997059..e7c9d1079fb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -233,6 +233,14 @@ public class MockNodeRepository extends NodeRepository {
                                                           Load.zero(),
                                                           Load.zero(),
                                                           Autoscaling.Metrics.zero()));
+        cluster1 = cluster1.withSuggestions(List.of(new Autoscaling(Autoscaling.Status.unavailable,
+                                                          "",
+                                                          Optional.of(new ClusterResources(6, 2,
+                                                                  new NodeResources(3, 20, 100, 1))),
+                                                          clock().instant(),
+                                                          Load.zero(),
+                                                          Load.zero(),
+                                                          Autoscaling.Metrics.zero())));
         cluster1 = cluster1.withTarget(new Autoscaling(Autoscaling.Status.unavailable,
                                                        "",
                                                        Optional.of(new ClusterResources(4, 1,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
index 4236f7ac968..830ff170a90 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -462,12 +462,12 @@ public class AutoscalingTest {
 
         fixture.tester().clock().advance(Duration.ofDays(2));
         fixture.loader().applyLoad(new Load(0.01, 0.01, 0.01, 0, 0), 120);
-        Autoscaling suggestion = fixture.suggest();
+        List<Autoscaling> suggestions = fixture.suggest();
         fixture.tester().assertResources("Choosing the remote disk flavor as it has less disk",
                                          2, 1, 3.0,  100.0, 10.0,
-                                         suggestion);
+                                         suggestions);
         assertEquals("Choosing the remote disk flavor as it has less disk",
-                     StorageType.remote, suggestion.resources().get().nodeResources().storageType());
+                     StorageType.remote, suggestions.stream().findFirst().flatMap(Autoscaling::resources).get().nodeResources().storageType());
     }
 
     @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
index df85ca4865f..4ce909fece3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
@@ -108,7 +108,7 @@ public class Fixture {
     }
 
     /** Compute an autoscaling suggestion for this. */
-    public Autoscaling suggest() {
+    public List<Autoscaling> suggest() {
         return tester().suggest(applicationId, clusterSpec.id(), capacity.minResources(), capacity.maxResources());
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
index f8be27300fe..51297a88cad 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
@@ -78,6 +78,12 @@ public class ScalingSuggestionsMaintainerTest {
         assertEquals("7 nodes with [vcpu: 4.1, memory: 5.3 Gb, disk: 16.5 Gb, bandwidth: 0.1 Gbps, architecture: any]",
                      suggestionOf(app2, cluster2, tester).resources().get().toString());
 
+        // Secondary suggestions
+        assertEquals("7 nodes with [vcpu: 3.7, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]",
+                suggestionsOf(app1, cluster1, tester).get(1).resources().get().toString());
+        assertEquals("8 nodes with [vcpu: 3.6, memory: 4.7 Gb, disk: 14.2 Gb, bandwidth: 0.1 Gbps, architecture: any]",
+                suggestionsOf(app2, cluster2, tester).get(1).resources().get().toString());
+
         // Utilization goes way down
         tester.clock().advance(Duration.ofHours(13));
         addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository());
@@ -97,7 +103,7 @@ public class ScalingSuggestionsMaintainerTest {
         tester.clock().advance(Duration.ofDays(3));
         addMeasurements(0.7f, 0.7f, 0.7f, 0, 500, app1, tester.nodeRepository());
         maintainer.maintain();
-        var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get();
+        var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestions().stream().findFirst().flatMap(Autoscaling::resources).get();
         tester.deploy(app1, cluster1, Capacity.from(suggested, suggested,
                                                     IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()));
         tester.clock().advance(Duration.ofDays(2));
@@ -121,7 +127,11 @@ public class ScalingSuggestionsMaintainerTest {
     }
 
     private Autoscaling suggestionOf(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) {
-        return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get().suggested();
+        return suggestionsOf(app, cluster, tester).get(0);
+    }
+
+    private List<Autoscaling> suggestionsOf(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) {
+        return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get().suggestions();
     }
 
     private boolean shouldSuggest(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
index 918a9043c93..90af6dca090 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
@@ -41,6 +41,7 @@ public class ApplicationSerializerTest {
                                  IntRange.empty(),
                                  true,
                                  Autoscaling.empty(),
+                                 List.of(),
                                  Autoscaling.empty(),
                                  ClusterInfo.empty(),
                                  BcpGroupInfo.empty(),
@@ -60,6 +61,14 @@ public class ApplicationSerializerTest {
                                                  new Load(0.1, 0.2, 0.3, 0.4, 0.5),
                                                  new Load(0.4, 0.5, 0.6, 0.7, 0.8),
                                                  new Autoscaling.Metrics(0.7, 0.8, 0.9)),
+                                 List.of(new Autoscaling(Autoscaling.Status.unavailable,
+                                                 "",
+                                                 Optional.of(new ClusterResources(20, 10,
+                                                         new NodeResources(0.5, 4, 14, 16))),
+                                                 Instant.ofEpochMilli(1234L),
+                                                 new Load(0.1, 0.2, 0.3, 0.4, 0.5),
+                                                 new Load(0.4, 0.5, 0.6, 0.7, 0.8),
+                                                 new Autoscaling.Metrics(0.7, 0.8, 0.9))),
                                  new Autoscaling(Autoscaling.Status.insufficient,
                                                  "Autoscaling status",
                                                  Optional.of(new ClusterResources(10, 5,
@@ -98,6 +107,7 @@ public class ApplicationSerializerTest {
             assertEquals(originalCluster.groupSize(), serializedCluster.groupSize());
             assertEquals(originalCluster.required(), serializedCluster.required());
             assertEquals(originalCluster.suggested(), serializedCluster.suggested());
+            assertEquals(originalCluster.suggestions(), serializedCluster.suggestions());
             assertEquals(originalCluster.target(), serializedCluster.target());
             assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo());
             assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
index be2b2ca896a..6b6ef49fa5d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -143,6 +144,7 @@ public class DynamicProvisioningTester {
                               cluster.groupSize(),
                               cluster.required(),
                               cluster.suggested(),
+                              cluster.suggestions(),
                               cluster.target(),
                               cluster.clusterInfo(),
                               cluster.bcpGroupInfo(),
@@ -165,7 +167,7 @@ public class DynamicProvisioningTester {
                                     nodeRepository().nodes().list(Node.State.active).owner(applicationId));
     }
 
-    public Autoscaling suggest(ApplicationId applicationId, ClusterSpec.Id clusterId,
+    public List<Autoscaling> suggest(ApplicationId applicationId, ClusterSpec.Id clusterId,
                                      ClusterResources min, ClusterResources max) {
         Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId))
                                                   .withCluster(clusterId, false, Capacity.from(min, max));
@@ -199,6 +201,14 @@ public class DynamicProvisioningTester {
     public ClusterResources assertResources(String message,
                                             int nodeCount, int groupCount,
                                             double approxCpu, double approxMemory, double approxDisk,
+                                            List<Autoscaling> autoscaling) {
+        assertFalse(autoscaling.isEmpty());
+        return assertResources(message, nodeCount, groupCount, approxCpu, approxMemory, approxDisk, autoscaling.get(0));
+    }
+
+    public ClusterResources assertResources(String message,
+                                            int nodeCount, int groupCount,
+                                            double approxCpu, double approxMemory, double approxDisk,
                                             Autoscaling autoscaling) {
         assertTrue("Resources should be present: " + message + " (" + autoscaling + ": " + autoscaling.status() + ")",
                    autoscaling.resources().isPresent());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json
index 7b2cf1dc8e4..1fa7b1a813e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json
@@ -80,6 +80,45 @@
           "cpuCostPerQuery" : 0.0
         }
       },
+      "suggestions": [
+        {
+          "at": 123,
+          "description": "",
+          "ideal": {
+            "cpu": 0.0,
+            "disk": 0.0,
+            "gpu": 0.0,
+            "gpuMemory": 0.0,
+            "memory": 0.0
+          },
+          "metrics": {
+            "cpuCostPerQuery": 0.0,
+            "growthRateHeadroom": 0.0,
+            "queryRate": 0.0
+          },
+          "peak": {
+            "cpu": 0.0,
+            "disk": 0.0,
+            "gpu": 0.0,
+            "gpuMemory": 0.0,
+            "memory": 0.0
+          },
+          "resources": {
+            "groups": 2,
+            "nodes": 6,
+            "resources": {
+              "architecture": "any",
+              "bandwidthGbps": 1.0,
+              "diskGb": 100.0,
+              "diskSpeed": "fast",
+              "memoryGb": 20.0,
+              "storageType": "any",
+              "vcpu": 3.0
+            }
+          },
+          "status": "unavailable"
+        }
+      ],
       "target" : {
         "status" : "unavailable",
         "description" : "",