Test with memory driven rescaling

4 files changed, 41 insertions, 23 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index f1be485d368..8f3bed75fa6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -67,10 +67,12 @@ public class Autoscaler {
         Optional<Double> totalDiskSpent   = averageUseOf(Resource.disk,   applicationId, cluster, clusterNodes);
         if (totalCpuSpent.isEmpty() || totalMemorySpent.isEmpty() || totalDiskSpent.isEmpty()) return Optional.empty();
 
+        System.out.println("  Total cpu " + totalCpuSpent.get() + " total memory " + totalMemorySpent.get() + " total disk " + totalDiskSpent.get());
         Optional<ClusterResources> bestAllocation = findBestAllocation(totalCpuSpent.get(),
                                                                        totalMemorySpent.get(),
                                                                        totalDiskSpent.get(),
                                                                        currentAllocation);
+        System.out.println("  Best allocation: " + bestAllocation);
         if (bestAllocation.isPresent() && isSimilar(bestAllocation.get(), currentAllocation))
             return Optional.empty(); // Avoid small changes
         return bestAllocation;
@@ -81,9 +83,10 @@ public class Autoscaler {
         Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty();
         for (ResourceIterator i = new ResourceIterator(totalCpu, totalMemory, totalDisk, currentAllocation); i.hasNext(); ) {
             ClusterResources allocation = i.next();
+            System.out.println("    Considering " + allocation.nodes() + " nodes:");
             Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation);
             if (allocatableResources.isEmpty()) continue;
-
+            System.out.println("    -- Candidate: " + allocatableResources);
             if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost())
                 bestAllocation = allocatableResources;
         }
@@ -119,7 +122,10 @@ public class Autoscaler {
             double bestCost = Double.MAX_VALUE;
             Optional<Flavor> bestFlavor = Optional.empty();
             for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) {
-                if ( ! flavor.resources().satisfies(resources.nodeResources())) continue;
+                if ( ! flavor.resources().satisfies(resources.nodeResources())) {
+                    System.out.println("      " + flavor.name() + " does not satisfy " + resources.nodeResources());
+                    continue;
+                }
                 if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) {
                     bestFlavor = Optional.of(flavor);
                     bestCost = costOf(flavor);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java
index c581d2304fb..56053003876 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java
@@ -18,4 +18,9 @@ public class ClusterResourcesWithCost {
 
     public double cost() { return cost; }
 
+    @Override
+    public String toString() {
+        return "$" + cost + ": " + clusterResources();
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
index e188008b07f..f88e4a09fbd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -14,7 +14,9 @@ import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -35,10 +37,10 @@ public class AutoscalingTest {
 
         assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1).isEmpty());
 
-        tester.addMeasurements(Resource.cpu, 0.25f, 60, application1);
+        tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1);
         assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1).isEmpty());
 
-        tester.addMeasurements(Resource.cpu,  0.25f, 60, application1);
+        tester.addMeasurements(Resource.cpu,  0.25f, 1f, 60, application1);
         ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high",
                                                                  10, 1, 1.7,  44.4, 44.4,
                                                                   tester.autoscale(application1, cluster1));
@@ -47,14 +49,14 @@ public class AutoscalingTest {
         assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1).isEmpty());
 
         tester.deactivateRetired(application1, cluster1, scaledResources);
-        tester.addMeasurements(Resource.cpu, 0.8f, 3, application1);
+        tester.addMeasurements(Resource.cpu, 0.8f, 1f, 3, application1);
         assertTrue("Load change is large, but insufficient measurements for new config -> No change",
                    tester.autoscale(application1, cluster1).isEmpty());
 
-        tester.addMeasurements(Resource.cpu,  0.19f, 100, application1);
-        assertTrue("Load change is small -> No change", tester.autoscale(application1, cluster1).isEmpty());
+        tester.addMeasurements(Resource.cpu,  0.19f, 1f, 100, application1);
+        assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1));
 
-        tester.addMeasurements(Resource.cpu,  0.1f, 120, application1);
+        tester.addMeasurements(Resource.cpu,  0.1f, 1f, 120, application1);
         tester.assertResources("Scaling down since resource usage has gone down significantly",
                                10, 1, 1.2, 44.4, 44.4,
                                tester.autoscale(application1, cluster1));
@@ -70,7 +72,7 @@ public class AutoscalingTest {
 
         // deploy
         tester.deploy(application1, cluster1, 5, 5, resources);
-        tester.addMeasurements(Resource.cpu,  0.25f, 120, application1);
+        tester.addMeasurements(Resource.cpu,  0.25f, 1f, 120, application1);
         tester.assertResources("Scaling up since resource usage is too high",
                                10, 10, 1.7,  44.4, 44.4,
                                tester.autoscale(application1, cluster1));
@@ -86,7 +88,7 @@ public class AutoscalingTest {
 
         // deploy
         tester.deploy(application1, cluster1, 6, 2, resources);
-        tester.addMeasurements(Resource.cpu,  0.22f, 120, application1);
+        tester.addMeasurements(Resource.cpu,  0.22f, 1f, 120, application1);
         tester.assertResources("Scaling up since resource usage is too high",
                                9, 3, 2.7,  83.3, 83.3,
                                tester.autoscale(application1, cluster1));
@@ -95,10 +97,10 @@ public class AutoscalingTest {
     @Test
     public void testAutoscalingAws() {
         List<Flavor> flavors = new ArrayList<>();
-        flavors.add(new Flavor("aws-xlarge", new NodeResources(6, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
-        flavors.add(new Flavor("aws-large", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
-        flavors.add(new Flavor("aws-medium", new NodeResources(2, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
-        flavors.add(new Flavor("aws-small", new NodeResources(1, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-xlarge", new NodeResources(3, 200, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-small", new NodeResources(3, 80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
         AutoscalingTester tester = new AutoscalingTester(new Zone(CloudName.from("aws"), SystemName.main,
                                                                   Environment.prod, RegionName.from("us-east")),
                                                          flavors);
@@ -109,17 +111,18 @@ public class AutoscalingTest {
         // deploy
         tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 100, 100, 1));
 
-        tester.addMeasurements(Resource.cpu, 0.25f, 120, application1);
+        tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1);
         ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high",
-                                                                  7, 1, 3,  100, 100,
+                                                                  8, 1, 3,  80, 100,
                                                                   tester.autoscale(application1, cluster1));
 
         tester.deploy(application1, cluster1, scaledResources);
         tester.deactivateRetired(application1, cluster1, scaledResources);
 
-        tester.addMeasurements(Resource.cpu, 0.05f, 1000, application1);
-        tester.assertResources("Scaling down since resource usage has gone down significantly",
-                               8, 1, 1, 100, 100,
+        tester.addMeasurements(Resource.memory, 0.3f, 0.6f, 1000, application1);
+        System.out.println("Low memory usage");
+        tester.assertResources("Scaling down since resource usage has gone down",
+                               6, 1, 3, 80, 100,
                                tester.autoscale(application1, cluster1));
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index af03fc95d84..0bb063f8b01 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -117,8 +117,14 @@ class AutoscalingTester {
      * scaled to take one node redundancy into account.
      * (I.e we adjust to measure a bit lower load than "naively" wanted to offset for the autoscaler
      * wanting to see the ideal load with one node missing.)
+     *
+     * @param resource the resource we are explicitly setting the value of
+     * @param otherResourcesLoad the load factor relative to ideal to use for other resources
+     * @param count the number of measurements
+     * @param applicationId the application we're adding measurements for all nodes of
      */
-    public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId) {
+    public void addMeasurements(Resource resource, float value, float otherResourcesLoad,
+                                int count, ApplicationId applicationId) {
         List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active);
         float oneExtraNodeFactor = (float)(nodes.size() - 1.0) / (nodes.size());
         for (int i = 0; i < count; i++) {
@@ -126,7 +132,7 @@ class AutoscalingTester {
             for (Node node : nodes) {
                 for (Resource r : Resource.values())
                     db.add(node, r, clock().instant(),
-                           (r == resource ? value : (float)r.idealAverageLoad()) * oneExtraNodeFactor);
+                           (r == resource ? value : (float)r.idealAverageLoad() * otherResourcesLoad) * oneExtraNodeFactor);
             }
         }
     }
@@ -199,8 +205,6 @@ class AutoscalingTester {
 
         @Override
         public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId) {
-            for (Flavor f : hostFlavors)
-                System.out.println(f + ": " + f.resources());
             Flavor hostFlavor = hostFlavors.stream().filter(f -> f.resources().justNumbers().equals(resources.justNumbers())).findAny()
                                            .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors));