diff options
10 files changed, 145 insertions, 61 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java index f4bd4e2020f..745fdcd736d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterResources; +import java.time.Duration; import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -47,6 +48,12 @@ public class ScalingEvent { /** Returns the instant this completed, or empty if it is not yet complete as far as we know */ public Optional<Instant> completion() { return completion; } + /** Returns the time this event took to completion, or empty if it's not yet complete */ + public Optional<Duration> duration() { + if (completion.isEmpty()) return Optional.empty(); + return Optional.of(Duration.between(at, completion.get())); + } + public ScalingEvent withCompletion(Instant completion) { return new ScalingEvent(from, to, generation, at, Optional.of(completion)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 0cb69c2defa..2bebbedbac7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -8,10 +8,10 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -71,16 +71,13 @@ public class Autoscaler { int measurementsPerNode = clusterTimeseries.measurementsPerNode(); if (measurementsPerNode < minimumMeasurementsPerNode(clusterNodes.clusterSpec())) return Advice.none("Collecting more data before making new scaling decisions" + - ": Has " + measurementsPerNode + " data points per node" + - " (all: " + clusterTimeseries.measurementCount + - ", without stale: " + clusterTimeseries.measurementCountWithoutStale + - ", without out of service: " + clusterTimeseries.measurementCountWithoutStaleOutOfService + - ", without unstable: " + clusterTimeseries.measurementCountWithoutStaleOutOfServiceUnstable + ")"); + " (as " + measurementsPerNode + " data points per node but need " + + minimumMeasurementsPerNode(clusterNodes.clusterSpec()) + ")"); int nodesMeasured = clusterTimeseries.nodesMeasured(); if (nodesMeasured != clusterNodes.size()) return Advice.none("Collecting more data before making new scaling decisions" + - ": Has measurements from " + nodesMeasured + " but need from " + clusterNodes.size()); + "(has measurements from " + nodesMeasured + " but need from " + clusterNodes.size() + ")"); double cpuLoad = clusterTimeseries.averageLoad(Resource.cpu); double memoryLoad = clusterTimeseries.averageLoad(Resource.memory); @@ -95,8 +92,12 @@ public class Autoscaler { if (similar(bestAllocation.get(), currentAllocation)) return Advice.dontScale("Cluster is ideally scaled (within configured limits)"); - if (isDownscaling(bestAllocation.get(), currentAllocation) && recentlyScaled(cluster, clusterNodes)) - return Advice.dontScale("Waiting a while before scaling down"); + + Duration scalingWindow = scalingWindow(clusterNodes.clusterSpec(), cluster); + if (scaledIn(scalingWindow, cluster)) + return Advice.dontScale("Won't autoscale now: Less than " + scalingWindow + " since last rescaling"); + if (isDownscaling(bestAllocation.get(), currentAllocation) && scaledIn(scalingWindow.multipliedBy(3), cluster)) + return Advice.dontScale("Waiting " + scalingWindow.multipliedBy(3) + " since last rescaling before reducing resources"); return Advice.scaleTo(bestAllocation.get().toAdvertisedClusterResources()); } @@ -123,35 +124,45 @@ public class Autoscaler { return ! targetTotal.justNumbers().satisfies(currentTotal.justNumbers()); } - private boolean recentlyScaled(Cluster cluster, NodeList clusterNodes) { - Duration downscalingDelay = downscalingDelay(clusterNodes.first().get().allocation().get().membership().cluster()); + private boolean scaledIn(Duration delay, Cluster cluster) { return cluster.lastScalingEvent().map(event -> event.at()).orElse(Instant.MIN) - .isAfter(nodeRepository.clock().instant().minus(downscalingDelay)); + .isAfter(nodeRepository.clock().instant().minus(delay)); } /** The duration of the window we need to consider to make a scaling decision. See also minimumMeasurementsPerNode */ - static Duration scalingWindow(ClusterSpec cluster) { - if (cluster.isStateful()) return Duration.ofHours(12); - return Duration.ofMinutes(30); + static Duration scalingWindow(ClusterSpec clusterSpec, Cluster cluster) { + int completedEventCount = 0; + Duration totalDuration = Duration.ZERO; + for (ScalingEvent event : cluster.scalingEvents()) { + if (event.duration().isEmpty()) continue; + completedEventCount++; + totalDuration = totalDuration.plus(event.duration().get()); + } + + if (completedEventCount == 0) { // Use defaults + if (clusterSpec.isStateful()) return Duration.ofHours(12); + return Duration.ofMinutes(30); + } + else { + Duration predictedDuration = totalDuration.dividedBy(completedEventCount); + + // TODO: Remove when we have reliable completion for content clusters + if (clusterSpec.isStateful() && predictedDuration.minus(Duration.ofHours(12)).isNegative()) + return Duration.ofHours(12); + + if (predictedDuration.minus(Duration.ofMinutes(5)).isNegative()) return Duration.ofMinutes(5); // minimum + return predictedDuration; + } } static Duration maxScalingWindow() { - return Duration.ofHours(12); + return Duration.ofHours(48); } /** Measurements are currently taken once a minute. See also scalingWindow */ static int minimumMeasurementsPerNode(ClusterSpec cluster) { if (cluster.isStateful()) return 60; - return 7; - } - - /** - * We should wait a while before scaling down after a scaling event as a peak in usage - * indicates more peaks may arrive in the near future. - */ - static Duration downscalingDelay(ClusterSpec cluster) { - if (cluster.isStateful()) return Duration.ofHours(12); - return Duration.ofHours(1); + return 4; } public static boolean stable(NodeList nodes, NodeRepository nodeRepository) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java index 33f29b714c5..d73f7985a7b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java @@ -18,31 +18,18 @@ public class ClusterTimeseries { private final NodeList clusterNodes; - final int measurementCount; - final int measurementCountWithoutStale; - final int measurementCountWithoutStaleOutOfService; - final int measurementCountWithoutStaleOutOfServiceUnstable; - /** The measurements for all nodes in this snapshot */ private final List<NodeTimeseries> allTimeseries; public ClusterTimeseries(Cluster cluster, NodeList clusterNodes, MetricsDb db, NodeRepository nodeRepository) { this.clusterNodes = clusterNodes; - var timeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterNodes.clusterSpec())), + var timeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterNodes.clusterSpec(), cluster)), clusterNodes); - measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - if (cluster.lastScalingEvent().isPresent()) timeseries = filter(timeseries, snapshot -> snapshot.generation() < 0 || // Content nodes do not yet send generation snapshot.generation() >= cluster.lastScalingEvent().get().generation()); - measurementCountWithoutStale = timeseries.stream().mapToInt(m -> m.size()).sum(); - - timeseries = filter(timeseries, snapshot -> snapshot.inService()); - measurementCountWithoutStaleOutOfService = timeseries.stream().mapToInt(m -> m.size()).sum(); - - timeseries = filter(timeseries, snapshot -> snapshot.stable()); - measurementCountWithoutStaleOutOfServiceUnstable = timeseries.stream().mapToInt(m -> m.size()).sum(); + timeseries = filter(timeseries, snapshot -> snapshot.inService() && snapshot.stable()); this.allTimeseries = timeseries; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index e04dda2ca9c..c4744f6cb6a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -77,18 +77,20 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { if (application.cluster(clusterId).isEmpty()) return; Cluster cluster = application.cluster(clusterId).get(); cluster = updateCompletion(cluster, clusterNodes); - var advice = autoscaler.autoscale(cluster, clusterNodes); cluster = cluster.withAutoscalingStatus(advice.reason()); - if (advice.isEmpty()) { + + if (advice.isPresent() && !cluster.targetResources().equals(advice.target())) { // autoscale + cluster = cluster.withTarget(advice.target()); applications().put(application.with(cluster), deployment.applicationLock().get()); - } else if (!cluster.targetResources().equals(advice.target())) { - applications().put(application.with(cluster.withTarget(advice.target())), deployment.applicationLock().get()); if (advice.target().isPresent()) { logAutoscaling(advice.target().get(), applicationId, cluster, clusterNodes); deployment.activate(); } } + else { // store cluster update + applications().put(application.with(cluster), deployment.applicationLock().get()); + } } private Applications applications() { @@ -106,7 +108,6 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { if (clusterNodes.retired().stream() .anyMatch(node -> node.history().hasEventAt(History.Event.Type.retired, event.at()))) return cluster; - // - 2. all nodes have switched to the right config generation for (NodeTimeseries nodeTimeseries : metricsDb.getNodeTimeseries(event.at(), clusterNodes)) { Optional<MetricSnapshot> firstOnNewGeneration = diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 75f49834f97..5694f84c555 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -62,7 +62,7 @@ public class AutoscalingTest { tester.deactivateRetired(application1, cluster1, scaledResources); - tester.clock().advance(Duration.ofDays(1)); + tester.clock().advance(Duration.ofDays(2)); tester.addCpuMeasurements(0.8f, 1f, 3, application1); assertTrue("Load change is large, but insufficient measurements for new config -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); @@ -120,7 +120,7 @@ public class AutoscalingTest { tester.nodeRepository().getNodes(application1).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.slow); - tester.clock().advance(Duration.ofDays(1)); + tester.clock().advance(Duration.ofDays(2)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); // Changing min and max from slow to any ClusterResources min = new ClusterResources( 2, 1, @@ -309,6 +309,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 2, new NodeResources(10, 100, 100, 1)); + tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(1.0f, 1f, 1000, application1); tester.assertResources("Increase group size to reduce memory load", 8, 2, 12.9, 89.3, 62.5, @@ -327,7 +328,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); - tester.clock().advance(Duration.ofDays(1)); + tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, @@ -350,8 +351,8 @@ public class AutoscalingTest { tester.addMemMeasurements(0.02f, 0.95f, 120, application1); assertTrue(tester.autoscale(application1, cluster1.id(), min, max).target().isEmpty()); - // Trying the same a day later causes autoscaling - tester.clock().advance(Duration.ofDays(1)); + // Trying the same later causes autoscaling + tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, @@ -413,7 +414,7 @@ public class AutoscalingTest { // deploy (Why 103 Gb memory? See AutoscalingTester.MockHostResourcesCalculator tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); - tester.clock().advance(Duration.ofDays(1)); + tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.9f, 0.6f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, @@ -422,6 +423,7 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); + tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.3f, 0.6f, 1000, application1); tester.assertResources("Scaling down since resource usage has gone down", 5, 1, 3, 83, 36, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 681fc082019..48c90125ac8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -50,7 +50,7 @@ public class NodeMetricsDbTest { assertEquals(35, measurementCount(db.getNodeTimeseries(clock.instant().minus(Duration.ofHours(72)), Set.of(node0)))); db.gc(); - assertEquals( 5, measurementCount(db.getNodeTimeseries(clock.instant().minus(Duration.ofHours(72)), Set.of(node0)))); + assertEquals(23, measurementCount(db.getNodeTimeseries(clock.instant().minus(Duration.ofHours(72)), Set.of(node0)))); } private int measurementCount(List<NodeTimeseries> measurements) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index ba60a6a2207..1f61faed735 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -109,9 +109,9 @@ public class QuestMetricsDbTest { assertEquals(24 * 10, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); db.gc(); - assertEquals(24 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); + assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); db.gc(); // no-op - assertEquals(24 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); + assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); } /** To manually test that we can read existing data */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index c8f1c499854..0c1a59c883d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; @@ -17,6 +18,7 @@ import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -28,7 +30,7 @@ import static org.junit.Assert.assertTrue; public class AutoscalingMaintainerTest { @Test - public void testAutoscalingMaintainer() { + public void test_autoscaling_maintainer() { ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); @@ -54,6 +56,7 @@ public class AutoscalingMaintainerTest { new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), false, true)); + tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); assertTrue(tester.deployer().lastDeployTime(app2).isEmpty()); @@ -61,6 +64,7 @@ public class AutoscalingMaintainerTest { tester.addMeasurements(0.9f, 0.9f, 0.9f, 0, 500, app1); tester.addMeasurements(0.9f, 0.9f, 0.9f, 0, 500, app2); + tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); // since autoscaling is off assertTrue(tester.deployer().lastDeployTime(app2).isPresent()); @@ -85,6 +89,7 @@ public class AutoscalingMaintainerTest { // Causes autoscaling tester.clock().advance(Duration.ofSeconds(1)); + tester.clock().advance(Duration.ofMinutes(10)); Instant firstMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isPresent()); @@ -112,7 +117,12 @@ public class AutoscalingMaintainerTest { assertEquals(firstMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); // Add measurement of the expected generation, leading to rescaling - tester.clock().advance(Duration.ofHours(2)); + // - record scaling completion + tester.clock().advance(Duration.ofMinutes(5)); + tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 1, app1); + tester.maintainer().maintain(); + // - measure underload + tester.clock().advance(Duration.ofHours(1)); tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 500, app1); Instant lastMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); @@ -131,7 +141,7 @@ public class AutoscalingMaintainerTest { } @Test - public void testScalingEventRecording() { + public void test_scaling_event_recording() { ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); @@ -144,6 +154,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, app1Capacity); for (int i = 0; i < 20; i++) { + // Record completion to keep scaling window at minimum + tester.addMeasurements(0.1f, 0.1f, 0.1f, i, 1, app1); + tester.maintainer().maintain(); + tester.clock().advance(Duration.ofDays(1)); if (i % 2 == 0) // high load @@ -153,8 +167,63 @@ public class AutoscalingMaintainerTest { tester.maintainer().maintain(); } - var events = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().scalingEvents(); - assertEquals(Cluster.maxScalingEvents, events.size()); + assertEquals(Cluster.maxScalingEvents, tester.cluster(app1, cluster1).scalingEvents().size()); } + @Test + public void test_autoscaling_window() { + ApplicationId app1 = AutoscalingMaintainerTester.makeApplicationId("app1"); + ClusterSpec cluster1 = AutoscalingMaintainerTester.containerClusterSpec(); + NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); + NodeResources highResources = new NodeResources(8, 8, 20, 0.1); + Capacity app1Capacity = Capacity.from(new ClusterResources(2, 1, lowResources), + new ClusterResources(4, 2, highResources)); + var tester = new AutoscalingMaintainerTester(new MockDeployer.ApplicationContext(app1, cluster1, app1Capacity)); + ManualClock clock = tester.clock(); + + // deploy + tester.deploy(app1, cluster1, app1Capacity); + + autoscale(false, Duration.ofMinutes( 1), Duration.ofMinutes( 5), clock, app1, cluster1, tester); + autoscale( true, Duration.ofMinutes(19), Duration.ofMinutes(10), clock, app1, cluster1, tester); + autoscale( true, Duration.ofMinutes(40), Duration.ofMinutes(20), clock, app1, cluster1, tester); + } + + private void autoscale(boolean down, Duration completionTime, Duration expectedWindow, + ManualClock clock, ApplicationId application, ClusterSpec cluster, + AutoscalingMaintainerTester tester) { + long generation = tester.cluster(application, cluster).lastScalingEvent().get().generation(); + tester.maintainer().maintain(); + assertFalse("Not measured to be on the last generation yet", + tester.cluster(application, cluster).lastScalingEvent().get().completion().isPresent()); + + clock.advance(completionTime); + float load = down ? 0.1f : 1.0f; + tester.addMeasurements(load, load, load, generation, 200, application); + tester.maintainer().maintain(); + assertEvent("Measured completion of the last scaling event, but no new autoscaling yet", + generation, Optional.of(clock.instant()), + tester.cluster(application, cluster).lastScalingEvent().get()); + if (down) + clock.advance(expectedWindow.minus(completionTime).plus(expectedWindow.multipliedBy(2))); + else + clock.advance(expectedWindow.minus(completionTime)); + + tester.addMeasurements(load, load, load, generation, 200, application); + tester.maintainer().maintain(); + assertEquals("We passed window duration so a new autoscaling is started: " + + tester.cluster(application, cluster).autoscalingStatus(), + generation + 1, + tester.cluster(application, cluster).lastScalingEvent().get().generation()); + } + + private void assertEvent(String explanation, + long expectedGeneration, Optional<Instant> expectedCompletion, ScalingEvent event) { + assertEquals(explanation + ". Generation", expectedGeneration, event.generation()); + assertEquals(explanation + ". Generation " + expectedGeneration + ". Completion", + expectedCompletion, event.completion()); + } + + + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java index f47202e1580..f53ae1baad2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java @@ -14,6 +14,8 @@ import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; @@ -68,7 +70,7 @@ public class AutoscalingMaintainerTester { return provisioningTester.deploy(application, cluster, capacity); } - public void addMeasurements(float cpu, float mem, float disk, int generation, int count, ApplicationId applicationId) { + public void addMeasurements(float cpu, float mem, float disk, long generation, int count, ApplicationId applicationId) { List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active); for (int i = 0; i < count; i++) { for (Node node : nodes) @@ -82,6 +84,10 @@ public class AutoscalingMaintainerTester { } } + public Cluster cluster(ApplicationId application, ClusterSpec cluster) { + return nodeRepository().applications().get(application).get().cluster(cluster.id()).get(); + } + private FlavorsConfig flavorsConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 35da31077f4..d1fc13c9796 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -56,6 +56,7 @@ public class ScalingSuggestionsMaintainerTest { new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), false, true)); + tester.clock().advance(Duration.ofHours(13)); addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository(), metricsDb); addMeasurements(0.99f, 0.99f, 0.99f, 0, 500, app2, tester.nodeRepository(), metricsDb); |