From 0565fb13966180e86aa0e0fd6de9b6be1fb544af Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 14 Nov 2022 17:17:29 +0100 Subject: Use an AWS-compatible node size by default --- .../provision/autoscale/AutoscalingTest.java | 57 +++++++++++----------- .../vespa/hosted/provision/autoscale/Fixture.java | 2 +- .../provision/autoscale/awsnodes/AwsNodeTypes.java | 10 +++- 3 files changed, 38 insertions(+), 31 deletions(-) (limited to 'node-repository/src/test/java/com') diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index aff93c805b0..6b15063d40e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -18,6 +18,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; +import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import org.junit.Test; @@ -39,7 +40,7 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.7f, 10); ClusterResources scaledResources = fixture.tester().assertResources("Scaling up since resource usage is too high", - 8, 1, 6, 5.7, 57.1, + 9, 1, 3.6, 8, 37.5, fixture.autoscale()); fixture.deploy(Capacity.from(scaledResources)); @@ -56,14 +57,14 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.1f, 10); fixture.tester().assertResources("Scaling cpu down since usage has gone down significantly", - 9, 1, 1.3, 4.4, 43.7, + 8, 1, 1.0, 8.1, 38.1, fixture.autoscale()); } /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_no_autoscaling_with_no_measurements() { - var fixture = AutoscalingTester.fixture().hostCount(20).build(); + var fixture = AutoscalingTester.fixture().awsProdSetup().build(); assertTrue(fixture.autoscale().target().isEmpty()); } @@ -73,7 +74,7 @@ public class AutoscalingTest { var fixture = AutoscalingTester.fixture().hostCount(20).build(); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 3); fixture.tester().assertResources("Scaling up since resource usage is too high", - 10, 1, 7.2, 8.5, 92.6, + 9, 1, 5.3, 15.4, 78.1, fixture.autoscale()); } @@ -83,7 +84,7 @@ public class AutoscalingTest { var fixture = AutoscalingTester.fixture().hostCount(20).build(); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling up (only) since resource usage is too high", - 10, 1, 8.6, 4.4, 92.6, + 8, 1, 7.1, 9.1, 89.3, fixture.autoscale()); } @@ -93,8 +94,8 @@ public class AutoscalingTest { var fixture = AutoscalingTester.fixture().hostCount(20).build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); - fixture.tester().assertResources("Scaling up (only) since resource usage is too high", - 10, 1, 8.6, 4.0, 92.6, + fixture.tester().assertResources("Scaling cpu and disk up and memory down", + 8, 1, 7.1, 4.0, 89.3, fixture.autoscale()); } @@ -105,7 +106,7 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); fixture.tester().assertResources("Scaling up since peak resource usage is too high", - 8, 1, 6.5, 5.7, 57.1, + 9, 1, 3.8, 8, 37.5, fixture.autoscale()); } @@ -133,13 +134,13 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.25f, 120); ClusterResources scaledResources = fixture.tester().assertResources("Scaling cpu up", - 5, 1, 3.8, 8.0, 80.0, + 4, 1, 3.3, 12.8, 60.0, fixture.autoscale()); fixture.deploy(Capacity.from(scaledResources)); fixture.deactivateRetired(Capacity.from(scaledResources)); fixture.loader().applyCpuLoad(0.1f, 120); fixture.tester().assertResources("Scaling down since cpu usage has gone down", - 4, 1, 2.5, 6.4, 64.0, + 3, 1, 2.5, 9.6, 45.0, fixture.autoscale()); } @@ -225,7 +226,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.05f, 0.05f, 0.05f), 120); fixture.tester().assertResources("Scaling down to limit since resource usage is low", - 4, 1, 1.8, 7.4, 13.9, + 4, 1, 1.8, 7.4, 10.4, fixture.autoscale()); } @@ -311,7 +312,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", - 8, 1, 9.3, 4.6, 45.7, + 8, 1, 6.2, 7.3, 34.3, fixture.tester().suggest(fixture.applicationId, fixture.clusterSpec.id(), min, min)); } @@ -431,7 +432,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.02, 120); fixture.tester().assertResources("Scaling down since enough time has passed", - 5, 1, 1.0, 8.0, 80.0, + 4, 1, 1.0, 17.1, 80.0, fixture.autoscale()); } @@ -444,7 +445,7 @@ public class AutoscalingTest { .build(); fixture.loader().applyLoad(new Load(1.0, 1.0, 0.7), 1000); fixture.tester().assertResources("Scaling up", - 9, 1, 5.0, 9.6, 72.9, + 10, 1, 3.0, 13.7, 48.6, fixture.autoscale()); } @@ -455,7 +456,7 @@ public class AutoscalingTest { .build(); fixture.loader().applyLoad(new Load(1.0, 1.0, 0.7), 1000); fixture.tester().assertResources("With 3Gb memory tax, we scale up memory more", - 7, 1, 6.4, 15.8, 97.2, + 8, 1, 3.7, 20.6, 62.5, fixture.autoscale()); } } @@ -503,20 +504,20 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.25, 120); // (no read share stored) fixture.tester().assertResources("Advice to scale up since we set aside for bcp by default", - 7, 1, 3, 100, 100, + 5, 1, 3, 100, 100, fixture.autoscale()); fixture.loader().applyCpuLoad(0.25, 120); fixture.storeReadShare(0.25, 0.5); fixture.tester().assertResources("Half of global share is the same as the default assumption used above", - 7, 1, 3, 100, 100, + 5, 1, 3, 100, 100, fixture.autoscale()); fixture.tester.clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); fixture.storeReadShare(0.5, 0.5); fixture.tester().assertResources("Advice to scale down since we don't need room for bcp", - 5, 1, 3, 100, 100, + 4, 1, 3, 100, 100, fixture.autoscale()); } @@ -530,7 +531,7 @@ public class AutoscalingTest { fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since we assume we need 2x cpu for growth when no data scaling time data", - 9, 1, 2.1, 4, 40, + 10, 1, 1.2, 5.7, 26.7, fixture.autoscale()); fixture.setScalingDuration(Duration.ofMinutes(5)); @@ -539,7 +540,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale down since observed growth is slower than scaling time", - 9, 1, 1.8, 4, 40, + 10, 1, 1.1, 5.7, 26.7, fixture.autoscale()); fixture.setScalingDuration(Duration.ofMinutes(60)); @@ -550,7 +551,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since observed growth is faster than scaling time", - 9, 1, 2.1, 4, 40, + 10, 1, 1.3, 5.7, 26.7, fixture.autoscale()); } @@ -568,7 +569,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester.assertResources("Query and write load is equal -> scale up somewhat", - 9, 1, 2.4, 4, 40, + 10, 1, 1.4, 5.7, 26.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -577,7 +578,7 @@ public class AutoscalingTest { fixture.loader().addCpuMeasurements(0.4, 200); // TODO: Ackhually, we scale down here - why? fixture.tester().assertResources("Query load is 4x write load -> scale up more", - 9, 1, 2.1, 4, 40, + 10, 1, 1.3, 5.7, 26.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -585,7 +586,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write load is 10x query load -> scale down", - 9, 1, 1.1, 4, 40, + 6, 1, 1.1, 10.2, 48, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -593,7 +594,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Query only -> largest possible", - 8, 1, 4.6, 4.6, 45.7, + 9, 1, 2.7, 6.4, 30, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -601,7 +602,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write only -> smallest possible", - 6, 1, 1.0, 6.4, 64, + 4, 1, 1.1, 17.1, 80, fixture.autoscale()); } @@ -635,8 +636,8 @@ public class AutoscalingTest { .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); - fixture.tester().assertResources("We scale even in dev because resources are required", - 3, 1, 1.0, 7.7, 83.3, + fixture.tester().assertResources("We scale even in dev because resources are 'required'", + 3, 1, 1.0, 12.3, 62.5, fixture.autoscale()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index fba060b5f07..0fb22445ab5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -126,7 +126,7 @@ public class Fixture { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster1")).vespaVersion("7").build(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); List hostFlavors = List.of(new Flavor(new NodeResources(100, 100, 100, 1))); - Optional initialResources = Optional.of(new ClusterResources(5, 1, new NodeResources(3, 10, 100, 1))); + Optional initialResources = Optional.of(new ClusterResources(5, 1, new NodeResources(2, 16, 75, 1))); Capacity capacity = Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any)), new ClusterResources(20, 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java index ceaa6861f20..df7c468f035 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.provision.autoscale.awsnodes; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -104,10 +106,14 @@ public class AwsNodeTypes { new VespaFlavor("i4i_16xlarge", 64.0, 64.0, 512.0, 498.0, 15000.0, 10.0, fast, local, x86_64), new VespaFlavor("i4i_32xlarge", 128.0, 128.0, 1024.0, 996.0, 30000.0, 10.0, fast, local, x86_64)); - public static List asVespaFlavors() { return hostFlavors; } + public static List asVespaFlavors() { return sorted(hostFlavors); } public static List asFlavors() { - return hostFlavors.stream().map(f -> toFlavor(f)).toList(); + return sorted(hostFlavors).stream().map(f -> toFlavor(f)).toList(); + } + + private static List sorted(List flavors) { + return flavors.stream().sorted(Comparator.comparing(n -> n.realResources().storageType() == NodeResources.StorageType.local ? 0 : 1)).toList(); } private static Flavor toFlavor(VespaFlavor vespaFlavor) { -- cgit v1.2.3