From a75af82380edeb58a20a978785c479ac3e3ae23d Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 4 Dec 2020 13:25:37 +0100 Subject: Adaptive scaling window --- .../maintenance/AutoscalingMaintainerTest.java | 79 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) (limited to 'node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java') diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index c8f1c499854..0c1a59c883d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -17,6 +18,7 @@ import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -28,7 +30,7 @@ import static org.junit.Assert.assertTrue; public class AutoscalingMaintainerTest { @Test - public void testAutoscalingMaintainer() { + public void test_autoscaling_maintainer() { ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); @@ -54,6 +56,7 @@ public class AutoscalingMaintainerTest { new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), false, true)); + tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); assertTrue(tester.deployer().lastDeployTime(app2).isEmpty()); @@ -61,6 +64,7 @@ public class AutoscalingMaintainerTest { tester.addMeasurements(0.9f, 0.9f, 0.9f, 0, 500, app1); tester.addMeasurements(0.9f, 0.9f, 0.9f, 0, 500, app2); + tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); // since autoscaling is off assertTrue(tester.deployer().lastDeployTime(app2).isPresent()); @@ -85,6 +89,7 @@ public class AutoscalingMaintainerTest { // Causes autoscaling tester.clock().advance(Duration.ofSeconds(1)); + tester.clock().advance(Duration.ofMinutes(10)); Instant firstMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isPresent()); @@ -112,7 +117,12 @@ public class AutoscalingMaintainerTest { assertEquals(firstMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); // Add measurement of the expected generation, leading to rescaling - tester.clock().advance(Duration.ofHours(2)); + // - record scaling completion + tester.clock().advance(Duration.ofMinutes(5)); + tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 1, app1); + tester.maintainer().maintain(); + // - measure underload + tester.clock().advance(Duration.ofHours(1)); tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 500, app1); Instant lastMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); @@ -131,7 +141,7 @@ public class AutoscalingMaintainerTest { } @Test - public void testScalingEventRecording() { + public void test_scaling_event_recording() { ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); @@ -144,6 +154,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, app1Capacity); for (int i = 0; i < 20; i++) { + // Record completion to keep scaling window at minimum + tester.addMeasurements(0.1f, 0.1f, 0.1f, i, 1, app1); + tester.maintainer().maintain(); + tester.clock().advance(Duration.ofDays(1)); if (i % 2 == 0) // high load @@ -153,8 +167,63 @@ public class AutoscalingMaintainerTest { tester.maintainer().maintain(); } - var events = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().scalingEvents(); - assertEquals(Cluster.maxScalingEvents, events.size()); + assertEquals(Cluster.maxScalingEvents, tester.cluster(app1, cluster1).scalingEvents().size()); } + @Test + public void test_autoscaling_window() { + ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); + ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); + NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); + NodeResources highResources = new NodeResources(8, 8, 20, 0.1); + Capacity app1Capacity = Capacity.from(new ClusterResources(2, 1, lowResources), + new ClusterResources(4, 2, highResources)); + var tester = new AutoscalingMaintainerTester(new MockDeployer.ApplicationContext(app1, cluster1, app1Capacity)); + ManualClock clock = tester.clock(); + + // deploy + tester.deploy(app1, cluster1, app1Capacity); + + autoscale(false, Duration.ofMinutes( 1), Duration.ofMinutes( 5), clock, app1, cluster1, tester); + autoscale( true, Duration.ofMinutes(19), Duration.ofMinutes(10), clock, app1, cluster1, tester); + autoscale( true, Duration.ofMinutes(40), Duration.ofMinutes(20), clock, app1, cluster1, tester); + } + + private void autoscale(boolean down, Duration completionTime, Duration expectedWindow, + ManualClock clock, ApplicationId application, ClusterSpec cluster, + AutoscalingMaintainerTester tester) { + long generation = tester.cluster(application, cluster).lastScalingEvent().get().generation(); + tester.maintainer().maintain(); + assertFalse("Not measured to be on the last generation yet", + tester.cluster(application, cluster).lastScalingEvent().get().completion().isPresent()); + + clock.advance(completionTime); + float load = down ? 0.1f : 1.0f; + tester.addMeasurements(load, load, load, generation, 200, application); + tester.maintainer().maintain(); + assertEvent("Measured completion of the last scaling event, but no new autoscaling yet", + generation, Optional.of(clock.instant()), + tester.cluster(application, cluster).lastScalingEvent().get()); + if (down) + clock.advance(expectedWindow.minus(completionTime).plus(expectedWindow.multipliedBy(2))); + else + clock.advance(expectedWindow.minus(completionTime)); + + tester.addMeasurements(load, load, load, generation, 200, application); + tester.maintainer().maintain(); + assertEquals("We passed window duration so a new autoscaling is started: " + + tester.cluster(application, cluster).autoscalingStatus(), + generation + 1, + tester.cluster(application, cluster).lastScalingEvent().get().generation()); + } + + private void assertEvent(String explanation, + long expectedGeneration, Optional expectedCompletion, ScalingEvent event) { + assertEquals(explanation + ". Generation", expectedGeneration, event.generation()); + assertEquals(explanation + ". Generation " + expectedGeneration + ". Completion", + expectedCompletion, event.completion()); + } + + + } -- cgit v1.2.3