diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-21 10:56:07 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-21 10:56:07 +0100 |
commit | f47e13cfeaecf5f1152e3cadbd2362e618d54b63 (patch) | |
tree | 2d616fe2a813354512fc9202e7aea642bed53f14 /node-repository | |
parent | 6584a5218e87563c0a75d4a1be185911d32504b2 (diff) |
Test with memory driven rescaling
Diffstat (limited to 'node-repository')
4 files changed, 41 insertions, 23 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index f1be485d368..8f3bed75fa6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -67,10 +67,12 @@ public class Autoscaler { Optional<Double> totalDiskSpent = averageUseOf(Resource.disk, applicationId, cluster, clusterNodes); if (totalCpuSpent.isEmpty() || totalMemorySpent.isEmpty() || totalDiskSpent.isEmpty()) return Optional.empty(); + System.out.println(" Total cpu " + totalCpuSpent.get() + " total memory " + totalMemorySpent.get() + " total disk " + totalDiskSpent.get()); Optional<ClusterResources> bestAllocation = findBestAllocation(totalCpuSpent.get(), totalMemorySpent.get(), totalDiskSpent.get(), currentAllocation); + System.out.println(" Best allocation: " + bestAllocation); if (bestAllocation.isPresent() && isSimilar(bestAllocation.get(), currentAllocation)) return Optional.empty(); // Avoid small changes return bestAllocation; @@ -81,9 +83,10 @@ public class Autoscaler { Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty(); for (ResourceIterator i = new ResourceIterator(totalCpu, totalMemory, totalDisk, currentAllocation); i.hasNext(); ) { ClusterResources allocation = i.next(); + System.out.println(" Considering " + allocation.nodes() + " nodes:"); Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation); if (allocatableResources.isEmpty()) continue; - + System.out.println(" -- Candidate: " + allocatableResources); if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) bestAllocation = allocatableResources; } @@ -119,7 +122,10 @@ public class Autoscaler { double bestCost = Double.MAX_VALUE; Optional<Flavor> bestFlavor = Optional.empty(); for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { - if ( ! flavor.resources().satisfies(resources.nodeResources())) continue; + if ( ! flavor.resources().satisfies(resources.nodeResources())) { + System.out.println(" " + flavor.name() + " does not satisfy " + resources.nodeResources()); + continue; + } if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) { bestFlavor = Optional.of(flavor); bestCost = costOf(flavor); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java index c581d2304fb..56053003876 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java @@ -18,4 +18,9 @@ public class ClusterResourcesWithCost { public double cost() { return cost; } + @Override + public String toString() { + return "$" + cost + ": " + clusterResources(); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index e188008b07f..f88e4a09fbd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -14,7 +14,9 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -35,10 +37,10 @@ public class AutoscalingTest { assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1).isEmpty()); - tester.addMeasurements(Resource.cpu, 0.25f, 60, application1); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1).isEmpty()); - tester.addMeasurements(Resource.cpu, 0.25f, 60, application1); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", 10, 1, 1.7, 44.4, 44.4, tester.autoscale(application1, cluster1)); @@ -47,14 +49,14 @@ public class AutoscalingTest { assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1).isEmpty()); tester.deactivateRetired(application1, cluster1, scaledResources); - tester.addMeasurements(Resource.cpu, 0.8f, 3, application1); + tester.addMeasurements(Resource.cpu, 0.8f, 1f, 3, application1); assertTrue("Load change is large, but insufficient measurements for new config -> No change", tester.autoscale(application1, cluster1).isEmpty()); - tester.addMeasurements(Resource.cpu, 0.19f, 100, application1); - assertTrue("Load change is small -> No change", tester.autoscale(application1, cluster1).isEmpty()); + tester.addMeasurements(Resource.cpu, 0.19f, 1f, 100, application1); + assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1)); - tester.addMeasurements(Resource.cpu, 0.1f, 120, application1); + tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); tester.assertResources("Scaling down since resource usage has gone down significantly", 10, 1, 1.2, 44.4, 44.4, tester.autoscale(application1, cluster1)); @@ -70,7 +72,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 5, resources); - tester.addMeasurements(Resource.cpu, 0.25f, 120, application1); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 10, 10, 1.7, 44.4, 44.4, tester.autoscale(application1, cluster1)); @@ -86,7 +88,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 2, resources); - tester.addMeasurements(Resource.cpu, 0.22f, 120, application1); + tester.addMeasurements(Resource.cpu, 0.22f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 9, 3, 2.7, 83.3, 83.3, tester.autoscale(application1, cluster1)); @@ -95,10 +97,10 @@ public class AutoscalingTest { @Test public void testAutoscalingAws() { List<Flavor> flavors = new ArrayList<>(); - flavors.add(new Flavor("aws-xlarge", new NodeResources(6, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); - flavors.add(new Flavor("aws-large", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); - flavors.add(new Flavor("aws-medium", new NodeResources(2, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); - flavors.add(new Flavor("aws-small", new NodeResources(1, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-xlarge", new NodeResources(3, 200, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-small", new NodeResources(3, 80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); AutoscalingTester tester = new AutoscalingTester(new Zone(CloudName.from("aws"), SystemName.main, Environment.prod, RegionName.from("us-east")), flavors); @@ -109,17 +111,18 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 100, 100, 1)); - tester.addMeasurements(Resource.cpu, 0.25f, 120, application1); + tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 7, 1, 3, 100, 100, + 8, 1, 3, 80, 100, tester.autoscale(application1, cluster1)); tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); - tester.addMeasurements(Resource.cpu, 0.05f, 1000, application1); - tester.assertResources("Scaling down since resource usage has gone down significantly", - 8, 1, 1, 100, 100, + tester.addMeasurements(Resource.memory, 0.3f, 0.6f, 1000, application1); + System.out.println("Low memory usage"); + tester.assertResources("Scaling down since resource usage has gone down", + 6, 1, 3, 80, 100, tester.autoscale(application1, cluster1)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index af03fc95d84..0bb063f8b01 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -117,8 +117,14 @@ class AutoscalingTester { * scaled to take one node redundancy into account. * (I.e we adjust to measure a bit lower load than "naively" wanted to offset for the autoscaler * wanting to see the ideal load with one node missing.) + * + * @param resource the resource we are explicitly setting the value of + * @param otherResourcesLoad the load factor relative to ideal to use for other resources + * @param count the number of measurements + * @param applicationId the application we're adding measurements for all nodes of */ - public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId) { + public void addMeasurements(Resource resource, float value, float otherResourcesLoad, + int count, ApplicationId applicationId) { List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active); float oneExtraNodeFactor = (float)(nodes.size() - 1.0) / (nodes.size()); for (int i = 0; i < count; i++) { @@ -126,7 +132,7 @@ class AutoscalingTester { for (Node node : nodes) { for (Resource r : Resource.values()) db.add(node, r, clock().instant(), - (r == resource ? value : (float)r.idealAverageLoad()) * oneExtraNodeFactor); + (r == resource ? value : (float)r.idealAverageLoad() * otherResourcesLoad) * oneExtraNodeFactor); } } } @@ -199,8 +205,6 @@ class AutoscalingTester { @Override public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId) { - for (Flavor f : hostFlavors) - System.out.println(f + ": " + f.resources()); Flavor hostFlavor = hostFlavors.stream().filter(f -> f.resources().justNumbers().equals(resources.justNumbers())).findAny() .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors)); |