diff options
Diffstat (limited to 'node-repository/src/test/java/com')
10 files changed, 96 insertions, 347 deletions
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 8c6c116a225..87b8ccdc348 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -46,7 +46,7 @@ public class AutoscalingIntegrationTest { for (int i = 0; i < 1000; i++) { tester.clock().advance(Duration.ofSeconds(10)); - fetcher.fetchMetrics(application1).whenComplete((r, e) -> tester.nodeMetricsDb().addNodeMetrics(r.nodeMetrics())); + fetcher.fetchMetrics(application1).whenComplete((r, e) -> tester.nodeMetricsDb().add(r.metrics())); tester.clock().advance(Duration.ofSeconds(10)); tester.nodeMetricsDb().gc(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 708316802bd..3fef1d9746b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -56,7 +56,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 14, 1, 1.4, 30.8, 30.8, + 15, 1, 1.3, 28.6, 28.6, tester.autoscale(application1, cluster1.id(), min, max).target()); tester.deploy(application1, cluster1, scaledResources); @@ -74,7 +74,7 @@ public class AutoscalingTest { tester.addCpuMeasurements(0.1f, 1f, 120, application1); tester.assertResources("Scaling down to minimum since usage has gone down significantly", - 15, 1, 1.0, 28.6, 28.6, + 14, 1, 1.0, 30.8, 30.8, tester.autoscale(application1, cluster1.id(), min, max).target()); var events = tester.nodeRepository().applications().get(application1).get().cluster(cluster1.id()).get().scalingEvents(); @@ -129,7 +129,7 @@ public class AutoscalingTest { ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 14, 1, 1.4, 30.8, 30.8, + 15, 1, 1.3, 28.6, 28.6, tester.autoscale(application1, cluster1.id(), min, max).target()); assertEquals("Disk speed from min/max is used", NodeResources.DiskSpeed.any, scaledResources.nodeResources().diskSpeed()); @@ -343,7 +343,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(1.0f, 1f, 1000, application1); tester.assertResources("Increase group size to reduce memory load", - 8, 2, 13.6, 89.3, 62.5, + 8, 2, 12.9, 89.3, 62.5, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -362,7 +362,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", - 6, 1, 2.9, 4.0, 95.0, + 6, 1, 2.8, 4.0, 95.0, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -386,7 +386,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", - 6, 1, 2.9, 4.0, 95.0, + 6, 1, 2.8, 4.0, 95.0, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -405,7 +405,7 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, min); tester.addMeasurements(1.0f, 1.0f, 0.7f, 0, 1000, application1); tester.assertResources("Scaling up", - 4, 1, 7.4, 20, 200, + 4, 1, 7.0, 20, 200, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -418,7 +418,7 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, min); tester.addMeasurements(1.0f, 1.0f, 0.7f, 0, 1000, application1); tester.assertResources("Scaling up", - 4, 1, 7.4, 34, 200, + 4, 1, 7.0, 34, 200, tester.autoscale(application1, cluster1.id(), min, max).target()); } } @@ -457,7 +457,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofDays(2)); tester.addMemMeasurements(0.3f, 0.6f, 1000, application1); tester.assertResources("Scaling down since resource usage has gone down", - 6, 1, 3, 83, 28.8, + 5, 1, 3, 83, 36, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -491,44 +491,6 @@ public class AutoscalingTest { } - @Test - public void test_autoscaling_considers_growth_rate() { - NodeResources resources = new NodeResources(3, 100, 100, 1); - ClusterResources min = new ClusterResources( 1, 1, resources); - ClusterResources max = new ClusterResources(10, 1, resources); - AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); - - ApplicationId application1 = tester.applicationId("application1"); - ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); - - tester.deploy(application1, cluster1, 5, 1, resources); - tester.addCpuMeasurements(0.25f, 1f, 120, application1); - - // (no query rate data) - tester.assertResources("Advice to scale up since we assume we need 2x cpu for growth when no data", - 7, 1, 3, 100, 100, - tester.autoscale(application1, cluster1.id(), min, max).target()); - - tester.setScalingDuration(application1, cluster1.id(), Duration.ofMinutes(5)); - tester.addQueryRateMeasurements(application1, cluster1.id(), - 100, - t -> 10.0 + (t < 50 ? t : 100 - t)); - tester.assertResources("Advice to scale down since observed growth is much slower than scaling time", - 4, 1, 3, 100, 100, - tester.autoscale(application1, cluster1.id(), min, max).target()); - - tester.clearQueryRateMeasurements(application1, cluster1.id()); - - System.out.println("The fast growth one"); - tester.setScalingDuration(application1, cluster1.id(), Duration.ofMinutes(60)); - tester.addQueryRateMeasurements(application1, cluster1.id(), - 100, - t -> 10.0 + (t < 50 ? t * t * t : 125000 - (t - 49) * (t - 49) * (t - 49))); - tester.assertResources("Advice to scale up since observed growth is much faster than scaling time", - 10, 1, 3, 100, 100, - tester.autoscale(application1, cluster1.id(), min, max).target()); - } - /** * This calculator subtracts the memory tax when forecasting overhead, but not when actually * returning information about nodes. This is allowed because the forecast is a *worst case*. diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index ce3293aa518..156542ef1d4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -20,21 +20,15 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.IntFunction; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -46,7 +40,7 @@ class AutoscalingTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; - private final MemoryMetricsDb db; + private final MetricsDb db; private final MockHostResourcesCalculator hostResourcesCalculator; /** Creates an autoscaling tester with a single host type ready */ @@ -143,14 +137,14 @@ class AutoscalingTester { float cpu = value * oneExtraNodeFactor; float memory = (float) Resource.memory.idealAverageLoad() * otherResourcesLoad * oneExtraNodeFactor; float disk = (float) Resource.disk.idealAverageLoad() * otherResourcesLoad * oneExtraNodeFactor; - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - 0, - true, - true, - 0.0)))); + db.add(List.of(new Pair<>(node.hostname(), new MetricSnapshot(clock().instant(), + cpu, + memory, + disk, + 0, + true, + true, + 0.0)))); } } } @@ -175,14 +169,14 @@ class AutoscalingTester { float cpu = (float) 0.2 * otherResourcesLoad * oneExtraNodeFactor; float memory = value * oneExtraNodeFactor; float disk = (float) Resource.disk.idealAverageLoad() * otherResourcesLoad * oneExtraNodeFactor; - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - 0, - true, - true, - 0.0)))); + db.add(List.of(new Pair<>(node.hostname(), new MetricSnapshot(clock().instant(), + cpu, + memory, + disk, + 0, + true, + true, + 0.0)))); } } } @@ -197,14 +191,14 @@ class AutoscalingTester { for (int i = 0; i < count; i++) { clock().advance(Duration.ofMinutes(1)); for (Node node : nodes) { - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - generation, - inService, - stable, - 0.0)))); + db.add(List.of(new Pair<>(node.hostname(), new MetricSnapshot(clock().instant(), + cpu, + memory, + disk, + generation, + inService, + stable, + 0.0)))); } } } @@ -216,41 +210,6 @@ class AutoscalingTester { nodeRepository().applications().put(application, nodeRepository().nodes().lock(applicationId)); } - /** Creates a single redeployment event with bogus data except for the given duration */ - public void setScalingDuration(ApplicationId applicationId, ClusterSpec.Id clusterId, Duration duration) { - Application application = nodeRepository().applications().require(applicationId); - Cluster cluster = application.cluster(clusterId).get(); - cluster = new Cluster(clusterId, - cluster.exclusive(), - cluster.minResources(), - cluster.maxResources(), - cluster.suggestedResources(), - cluster.targetResources(), - List.of(), // Remove scaling events - cluster.autoscalingStatus()); - cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(), - 0, - clock().instant().minus(Duration.ofDays(1).minus(duration))).withCompletion(clock().instant().minus(Duration.ofDays(1)))); - application = application.with(cluster); - nodeRepository().applications().put(application, nodeRepository().nodes().lock(applicationId)); - } - - /** Creates the given number of measurements, spaced 5 minutes between, using the given function */ - public void addQueryRateMeasurements(ApplicationId application, - ClusterSpec.Id cluster, - int measurements, - IntFunction<Double> queryRate) { - Instant time = clock().instant(); - for (int i = 0; i < measurements; i++) { - db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i)))); - time = time.plus(Duration.ofMinutes(5)); - } - } - - public void clearQueryRateMeasurements(ApplicationId application, ClusterSpec.Id cluster) { - db.clearClusterMetrics(application, cluster); - } - public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java deleted file mode 100644 index 89fe2d76159..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.test.ManualClock; -import org.junit.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.function.IntFunction; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ClusterTimeseriesTest { - - private static final double delta = 0.001; - private static final ClusterSpec.Id cluster = new ClusterSpec.Id("test"); - - @Test - public void test_empty() { - var timeseries = new ClusterTimeseries(cluster, List.of()); - assertEquals(0.1, timeseries.maxQueryGrowthRate(), delta); - } - - @Test - public void test_constant_rate_short() { - var clock = new ManualClock(); - var timeseries = new ClusterTimeseries(cluster, rate(10, clock, t -> 50.0)); - assertEquals(0.1, timeseries.maxQueryGrowthRate(), delta); - } - - @Test - public void test_constant_rate_long() { - var clock = new ManualClock(); - var timeseries = new ClusterTimeseries(cluster, rate(10000, clock, t -> 50.0)); - assertEquals(0.0, timeseries.maxQueryGrowthRate(), delta); - } - - @Test - public void test_single_spike() { - var clock = new ManualClock(); - var snapshots = new ArrayList<ClusterMetricSnapshot>(); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - snapshots.addAll(rate( 10, clock, t -> 400.0)); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - assertEquals((400-50)/5.0/50.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta); - } - - @Test - public void test_three_spikes() { - var clock = new ManualClock(); - var snapshots = new ArrayList<ClusterMetricSnapshot>(); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - snapshots.addAll(rate( 10, clock, t -> 400.0)); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - snapshots.addAll(rate( 10, clock, t -> 600.0)); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - snapshots.addAll(rate( 10, clock, t -> 800.0)); - snapshots.addAll(rate(1000, clock, t -> 50.0)); - assertEquals((800-50)/5.0/50.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta); - } - - @Test - public void test_single_hill() { - var clock = new ManualClock(); - var snapshots = new ArrayList<ClusterMetricSnapshot>(); - snapshots.addAll(rate(100, clock, t -> (double)t)); - snapshots.addAll(rate(100, clock, t -> 100.0 - t)); - assertEquals(1/5.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta); - } - - @Test - public void test_smooth_curve() { - var clock = new ManualClock(); - var timeseries = new ClusterTimeseries(cluster, rate(10000, clock, - t -> 10.0 + 100.0 * Math.sin(t))); - assertEquals(0.26, timeseries.maxQueryGrowthRate(), delta); - } - - @Test - public void test_smooth_curve_small_variation() { - var clock = new ManualClock(); - var timeseries = new ClusterTimeseries(cluster, rate(10000, clock, - t -> 1000.0 + 10.0 * Math.sin(t))); - assertEquals(0.0, timeseries.maxQueryGrowthRate(), delta); - } - - @Test - public void test_two_periods() { - var clock = new ManualClock(); - var timeseries = new ClusterTimeseries(cluster, rate(10000, clock, - t -> 10.0 + 100.0 * Math.sin(t) + 80.0 * Math.sin(10 * t)) ); - assertEquals(1.765, timeseries.maxQueryGrowthRate(), delta); - } - - private List<ClusterMetricSnapshot> rate(int count, ManualClock clock, IntFunction<Double> rate) { - List<ClusterMetricSnapshot> snapshots = new ArrayList<>(); - for (int i = 0; i < count; i++) { - snapshots.add(new ClusterMetricSnapshot(clock.instant(), rate.apply(i))); - clock.advance(Duration.ofMinutes(5)); - } - return snapshots; - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index 14a9a596e78..384e8dd8439 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -45,7 +45,7 @@ public class MetricsV2MetricsFetcherTest { { httpClient.cannedResponse = cannedResponseForApplication1; - List<Pair<String, NodeMetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application1).get().nodeMetrics()); + List<Pair<String, MetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application1).get().metrics()); assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=autoscaling", httpClient.requestsReceived.get(0)); assertEquals(2, values.size()); @@ -63,7 +63,7 @@ public class MetricsV2MetricsFetcherTest { { httpClient.cannedResponse = cannedResponseForApplication2; - List<Pair<String, NodeMetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application2).get().nodeMetrics()); + List<Pair<String, MetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application2).get().metrics()); assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=autoscaling", httpClient.requestsReceived.get(1)); assertEquals(1, values.size()); @@ -81,7 +81,7 @@ public class MetricsV2MetricsFetcherTest { tester.nodeRepository().nodes().write(tester.nodeRepository().nodes().list(Node.State.active).owner(application2) .first().get().retire(tester.clock().instant()), lock); } - List<Pair<String, NodeMetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application2).get().nodeMetrics()); + List<Pair<String, MetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application2).get().metrics()); assertFalse(values.get(0).getSecond().stable()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 76e56004871..c1c94c7dd24 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -38,19 +38,19 @@ public class NodeMetricsDbTest { ManualClock clock = tester.clock(); MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository()); - Collection<Pair<String, NodeMetricSnapshot>> values = new ArrayList<>(); + Collection<Pair<String, MetricSnapshot>> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { - values.add(new Pair<>(node0, new NodeMetricSnapshot(clock.instant(), - 0.9f, - 0.6f, - 0.6f, - 0, - true, - false, - 0.0))); + values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), + 0.9f, + 0.6f, + 0.6f, + 0, + true, + false, + 0.0))); clock.advance(Duration.ofMinutes(120)); } - db.addNodeMetrics(values); + db.add(values); // Avoid off-by-one bug when the below windows starts exactly on one of the above getEpochSecond() timestamps. clock.advance(Duration.ofMinutes(1)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index 18b92fa6b0f..70f9d581816 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.io.IOUtils; import com.yahoo.test.ManualClock; import org.junit.Ignore; @@ -14,9 +12,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -33,7 +29,7 @@ public class QuestMetricsDbTest { private static final double delta = 0.0000001; @Test - public void testNodeMetricsReadWrite() { + public void testReadWrite() { String dataDir = "data/QuestMetricsDbReadWrite"; IOUtils.recursiveDeleteDir(new File(dataDir)); IOUtils.createDirectory(dataDir + "/metrics"); @@ -42,7 +38,7 @@ public class QuestMetricsDbTest { Instant startTime = clock.instant(); clock.advance(Duration.ofSeconds(1)); - db.addNodeMetrics(nodeTimeseries(1000, Duration.ofSeconds(1), clock, "host1", "host2", "host3")); + db.add(timeseries(1000, Duration.ofSeconds(1), clock, "host1", "host2", "host3")); clock.advance(Duration.ofSeconds(1)); @@ -52,7 +48,7 @@ public class QuestMetricsDbTest { assertEquals(1, nodeTimeSeries1.size()); assertEquals("host1", nodeTimeSeries1.get(0).hostname()); assertEquals(1000, nodeTimeSeries1.get(0).size()); - NodeMetricSnapshot snapshot = nodeTimeSeries1.get(0).asList().get(0); + MetricSnapshot snapshot = nodeTimeSeries1.get(0).asList().get(0); assertEquals(startTime.plus(Duration.ofSeconds(1)), snapshot.at()); assertEquals(0.1, snapshot.cpu(), delta); assertEquals(0.2, snapshot.memory(), delta); @@ -79,56 +75,6 @@ public class QuestMetricsDbTest { } @Test - public void testClusterMetricsReadWrite() { - String dataDir = "data/QuestMetricsDbReadWrite"; - IOUtils.recursiveDeleteDir(new File(dataDir)); - IOUtils.createDirectory(dataDir + "/clusterMetrics"); - ManualClock clock = new ManualClock("2020-10-01T00:00:00"); - QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); - Instant startTime = clock.instant(); - - var application1 = ApplicationId.from("t1", "a1", "i1"); - var application2 = ApplicationId.from("t1", "a2", "i1"); - var cluster1 = new ClusterSpec.Id("cluster1"); - var cluster2 = new ClusterSpec.Id("cluster2"); - db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0))); - db.addClusterMetrics(application1, Map.of(cluster2, new ClusterMetricSnapshot(clock.instant(), 60.0))); - clock.advance(Duration.ofMinutes(1)); - db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 45.0))); - clock.advance(Duration.ofMinutes(1)); - db.addClusterMetrics(application2, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 90.0))); - - ClusterTimeseries clusterTimeseries11 = db.getClusterTimeseries(application1, cluster1); - assertEquals(cluster1, clusterTimeseries11.cluster()); - assertEquals(2, clusterTimeseries11.asList().size()); - - ClusterMetricSnapshot snapshot111 = clusterTimeseries11.get(0); - assertEquals(startTime, snapshot111.at()); - assertEquals(30, snapshot111.queryRate(), delta); - ClusterMetricSnapshot snapshot112 = clusterTimeseries11.get(1); - assertEquals(startTime.plus(Duration.ofMinutes(1)), snapshot112.at()); - assertEquals(45, snapshot112.queryRate(), delta); - - - ClusterTimeseries clusterTimeseries12 = db.getClusterTimeseries(application1, cluster2); - assertEquals(cluster2, clusterTimeseries12.cluster()); - assertEquals(1, clusterTimeseries12.asList().size()); - - ClusterMetricSnapshot snapshot121 = clusterTimeseries12.get(0); - assertEquals(startTime, snapshot121.at()); - assertEquals(60, snapshot121.queryRate(), delta); - - - ClusterTimeseries clusterTimeseries21 = db.getClusterTimeseries(application2, cluster1); - assertEquals(cluster1, clusterTimeseries21.cluster()); - assertEquals(1, clusterTimeseries21.asList().size()); - - ClusterMetricSnapshot snapshot211 = clusterTimeseries21.get(0); - assertEquals(startTime.plus(Duration.ofMinutes(2)), snapshot211.at()); - assertEquals(90, snapshot211.queryRate(), delta); - } - - @Test public void testWriteOldData() { String dataDir = "data/QuestMetricsDbWriteOldData"; IOUtils.recursiveDeleteDir(new File(dataDir)); @@ -137,19 +83,19 @@ public class QuestMetricsDbTest { QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); Instant startTime = clock.instant(); clock.advance(Duration.ofSeconds(300)); - db.addNodeMetrics(timeseriesAt(10, clock.instant(), "host1", "host2", "host3")); + db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3")); clock.advance(Duration.ofSeconds(1)); List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")); assertEquals(10, nodeTimeSeries1.get(0).size()); - db.addNodeMetrics(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3")); + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3")); List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")); assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size()); - db.addNodeMetrics(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3")); + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3")); List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")); assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size()); @@ -165,15 +111,15 @@ public class QuestMetricsDbTest { Instant startTime = clock.instant(); int dayOffset = 3; clock.advance(Duration.ofHours(dayOffset)); - db.addNodeMetrics(nodeTimeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3")); + db.add(timeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3")); assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size()); db.gc(); - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size()); db.gc(); // no-op - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size()); } @@ -200,7 +146,7 @@ public class QuestMetricsDbTest { System.out.println(" " + snapshot); clock.advance(Duration.ofSeconds(1)); - db.addNodeMetrics(nodeTimeseries(2, Duration.ofSeconds(1), clock, "host1")); + db.add(timeseries(2, Duration.ofSeconds(1), clock, "host1")); System.out.println("New data written and read:"); timeseries = db.getNodeTimeseries(Duration.ofSeconds(2), Set.of("host1")); for (var snapshot : timeseries.get(0).asList()) @@ -217,7 +163,7 @@ public class QuestMetricsDbTest { ManualClock clock = new ManualClock("2020-10-01T00:00:00"); QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); Instant startTime = clock.instant(); - db.addNodeMetrics(nodeTimeseries(10, Duration.ofSeconds(1), clock, "host1")); + db.add(timeseries(10, Duration.ofSeconds(1), clock, "host1")); int added = db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).asList().size(); @@ -225,46 +171,36 @@ public class QuestMetricsDbTest { db.close(); } - private Collection<Pair<String, NodeMetricSnapshot>> nodeTimeseries(int countPerHost, Duration sampleRate, ManualClock clock, - String ... hosts) { - Collection<Pair<String, NodeMetricSnapshot>> timeseries = new ArrayList<>(); + private Collection<Pair<String, MetricSnapshot>> timeseries(int countPerHost, Duration sampleRate, ManualClock clock, + String ... hosts) { + Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>(); for (int i = 1; i <= countPerHost; i++) { for (String host : hosts) - timeseries.add(new Pair<>(host, new NodeMetricSnapshot(clock.instant(), + timeseries.add(new Pair<>(host, new MetricSnapshot(clock.instant(), i * 0.1, i * 0.2, i * 0.4, i % 100, - true, - true, - 30.0))); - clock.advance(sampleRate); - } - return timeseries; - } - - private List<ClusterMetricSnapshot> clusterTimeseries(int count, Duration sampleRate, ManualClock clock, - ClusterSpec.Id cluster) { - List<ClusterMetricSnapshot> timeseries = new ArrayList<>(); - for (int i = 1; i <= count; i++) { - timeseries.add(new ClusterMetricSnapshot(clock.instant(), 30.0)); + true, + true, + 30.0))); clock.advance(sampleRate); } return timeseries; } - private Collection<Pair<String, NodeMetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) { - Collection<Pair<String, NodeMetricSnapshot>> timeseries = new ArrayList<>(); + private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) { + Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>(); for (int i = 1; i <= countPerHost; i++) { for (String host : hosts) - timeseries.add(new Pair<>(host, new NodeMetricSnapshot(at, + timeseries.add(new Pair<>(host, new MetricSnapshot(at, i * 0.1, i * 0.2, i * 0.4, i % 100, - true, - false, - 0.0))); + true, + false, + 0.0))); } return timeseries; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java index e8cfe6a2310..1b531fd3237 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java @@ -16,7 +16,8 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; +import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; @@ -74,14 +75,14 @@ public class AutoscalingMaintainerTester { NodeList nodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId); for (int i = 0; i < count; i++) { for (Node node : nodes) - metricsDb.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - mem, - disk, - generation, - true, - true, - 0.0)))); + metricsDb.add(List.of(new Pair<>(node.hostname(), new MetricSnapshot(clock().instant(), + cpu, + mem, + disk, + generation, + true, + true, + 0.0)))); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index 5af787092d5..e99f7740c29 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; +import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.MetricsV2MetricsFetcher; import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; @@ -49,9 +49,9 @@ public class NodeMetricsDbMaintainerTest { List<NodeTimeseries> timeseriesList = db.getNodeTimeseries(Duration.ofDays(1), Set.of("host-1.yahoo.com", "host-2.yahoo.com")); assertEquals(2, timeseriesList.size()); - List<NodeMetricSnapshot> allSnapshots = timeseriesList.stream() - .flatMap(timeseries -> timeseries.asList().stream()) - .collect(Collectors.toList()); + List<MetricSnapshot> allSnapshots = timeseriesList.stream() + .flatMap(timeseries -> timeseries.asList().stream()) + .collect(Collectors.toList()); assertTrue(allSnapshots.stream().anyMatch(snapshot -> snapshot.inService())); assertTrue(allSnapshots.stream().anyMatch(snapshot -> ! snapshot.inService())); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 88d39e887d3..d5b7903b94c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; +import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.Resource; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; @@ -74,7 +74,7 @@ public class ScalingSuggestionsMaintainerTest { assertEquals("14 nodes with [vcpu: 6.9, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", suggestionOf(app1, cluster1, tester).get().resources().toString()); - assertEquals("9 nodes with [vcpu: 13.8, memory: 4.0 Gb, disk 10.3 Gb, bandwidth: 0.1 Gbps]", + assertEquals("8 nodes with [vcpu: 14.7, memory: 4.0 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps]", suggestionOf(app2, cluster2, tester).get().resources().toString()); // Utilization goes way down @@ -125,14 +125,14 @@ public class ScalingSuggestionsMaintainerTest { NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(applicationId); for (int i = 0; i < count; i++) { for (Node node : nodes) - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(nodeRepository.clock().instant(), - cpu, - memory, - disk, - generation, - true, - true, - 0.0)))); + db.add(List.of(new Pair<>(node.hostname(), new MetricSnapshot(nodeRepository.clock().instant(), + cpu, + memory, + disk, + generation, + true, + true, + 0.0)))); } } |