diff options
16 files changed, 188 insertions, 48 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index d2c943794fe..023eb5860ee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -15,8 +15,7 @@ import java.util.Objects; import java.util.Optional; /** - * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster - * based on observed behavior. + * The autoscaler gives advice about what resources should be allocated to a cluster based on observed behavior. * * @author bratseth */ @@ -61,7 +60,7 @@ public class Autoscaler { private Advice autoscale(Cluster cluster, List<Node> clusterNodes, Limits limits, boolean exclusive) { ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - if (unstable(clusterNodes, nodeRepository)) + if ( ! stable(clusterNodes, nodeRepository)) return Advice.none("Cluster change in progress"); AllocatableClusterResources currentAllocation = @@ -72,7 +71,11 @@ public class Autoscaler { int measurementsPerNode = clusterTimeseries.measurementsPerNode(); if (measurementsPerNode < minimumMeasurementsPerNode(clusterType)) return Advice.none("Collecting more data before making new scaling decisions" + - ": Has " + measurementsPerNode + " data points per node"); + ": Has " + measurementsPerNode + " data points per node" + + "(all: " + clusterTimeseries.measurementCount + + ", without stale: " + clusterTimeseries.measurementCountWithoutStale + + ", without out of service: " + clusterTimeseries.measurementCountWithoutStaleOutOfService + + ", without unstable: " + clusterTimeseries.measurementCountWithoutStaleOutOfServiceUnstable); int nodesMeasured = clusterTimeseries.nodesMeasured(); if (nodesMeasured != clusterNodes.size()) @@ -151,18 +154,18 @@ public class Autoscaler { return Duration.ofHours(1); } - public static boolean unstable(List<Node> nodes, NodeRepository nodeRepository) { + public static boolean stable(List<Node> nodes, NodeRepository nodeRepository) { // The cluster is processing recent changes if (nodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || node.allocation().get().isRemovable())) - return true; + return false; // A deployment is ongoing if (nodeRepository.getNodes(nodes.get(0).allocation().get().owner(), Node.State.reserved).size() > 0) - return true; + return false; - return false; + return true; } public static class Advice { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java index e325e797ca5..1983162f121 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java @@ -7,9 +7,11 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -21,16 +23,33 @@ public class ClusterTimeseries { private final List<Node> clusterNodes; + final int measurementCount; + final int measurementCountWithoutStale; + final int measurementCountWithoutStaleOutOfService; + final int measurementCountWithoutStaleOutOfServiceUnstable; + /** The measurements for all hosts in this snapshot */ private final List<NodeTimeseries> nodeTimeseries; public ClusterTimeseries(Cluster cluster, List<Node> clusterNodes, MetricsDb db, NodeRepository nodeRepository) { this.clusterNodes = clusterNodes; ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - var allTimeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterType)), - clusterNodes.stream().map(Node::hostname).collect(Collectors.toSet())); - Map<String, Instant> startTimePerNode = metricStartTimes(cluster, clusterNodes, allTimeseries, nodeRepository); - nodeTimeseries = filterStale(allTimeseries, startTimePerNode); + var timeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterType)), + clusterNodes.stream().map(Node::hostname).collect(Collectors.toSet())); + Map<String, Instant> startTimePerNode = metricStartTimes(cluster, clusterNodes, timeseries, nodeRepository); + + measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); + + timeseries = filterStale(timeseries, startTimePerNode); + measurementCountWithoutStale = timeseries.stream().mapToInt(m -> m.size()).sum(); + + timeseries = filter(timeseries, snapshot -> snapshot.inService()); + measurementCountWithoutStaleOutOfService = timeseries.stream().mapToInt(m -> m.size()).sum(); + + timeseries = filter(timeseries, snapshot -> snapshot.stable()); + measurementCountWithoutStaleOutOfServiceUnstable = timeseries.stream().mapToInt(m -> m.size()).sum(); + + this.nodeTimeseries = timeseries; } /** @@ -95,4 +114,8 @@ public class ClusterTimeseries { return timeseries.stream().map(m -> m.justAfter(startTimePerHost.get(m.hostname()))).collect(Collectors.toList()); } + private List<NodeTimeseries> filter(List<NodeTimeseries> timeseries, Predicate<MetricSnapshot> filter) { + return timeseries.stream().map(nodeTimeseries -> nodeTimeseries.filter(filter)).collect(Collectors.toList()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java index 6f6cd862c33..7fb2de5d958 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java @@ -18,14 +18,17 @@ public class MetricSnapshot { private final double disk; private final long generation; private final boolean inService; + private final boolean stable; - public MetricSnapshot(Instant at, double cpu, double memory, double disk, long generation, boolean inService) { + public MetricSnapshot(Instant at, double cpu, double memory, double disk, long generation, + boolean inService, boolean stable) { this.at = at; this.cpu = cpu; this.memory = memory; this.disk = disk; this.generation = generation; this.inService = inService; + this.stable = stable; } public Instant at() { return at; } @@ -34,12 +37,16 @@ public class MetricSnapshot { public double disk() { return disk; } public long generation() { return generation; } public boolean inService() { return inService; } + public boolean stable() { return stable; } @Override public String toString() { return "metrics at " + at + ":" + " cpu: " + cpu + " memory: " + memory + " disk: " + disk + - " generation: " + generation; } + " generation: " + generation + + " inService: " + inService + + " stable: " + stable; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 4471d267416..59e8b3e10ed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -2,17 +2,24 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; /** * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values @@ -23,33 +30,41 @@ public class MetricsResponse { private final Collection<Pair<String, MetricSnapshot>> nodeMetrics = new ArrayList<>(); - public MetricsResponse(String response) { - this(SlimeUtils.jsonToSlime(response)); + public MetricsResponse(String response, NodeList applicationNodes, NodeRepository nodeRepository) { + this(SlimeUtils.jsonToSlime(response), applicationNodes, nodeRepository); } public Collection<Pair<String, MetricSnapshot>> metrics() { return nodeMetrics; } - private MetricsResponse(Slime response) { + private MetricsResponse(Slime response, NodeList applicationNodes, NodeRepository nodeRepository) { Inspector root = response.get(); Inspector nodes = root.field("nodes"); - nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node)); + nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node, applicationNodes, nodeRepository)); } - private void consumeNode(Inspector node) { + private void consumeNode(Inspector node, NodeList applicationNodes, NodeRepository nodeRepository) { String hostname = node.field("hostname").asString(); - consumeNodeMetrics(hostname, node.field("node")); + consumeNodeMetrics(hostname, node.field("node"), applicationNodes, nodeRepository); // consumeServiceMetrics(hostname, node.field("services")); } - private void consumeNodeMetrics(String hostname, Inspector node) { - long timestampSecond = node.field("timestamp").asLong(); - Map<String, Double> values = consumeMetrics(node.field("metrics")); + private void consumeNodeMetrics(String hostname, Inspector nodeData, NodeList applicationNodes, NodeRepository nodeRepository) { + Optional<Node> node = applicationNodes.stream().filter(n -> n.hostname().equals(hostname)).findAny(); + if (node.isEmpty()) return; // Node is not part of this cluster any more + long timestampSecond = nodeData.field("timestamp").asLong(); + Map<String, Double> values = consumeMetrics(nodeData.field("metrics")); nodeMetrics.add(new Pair<>(hostname, new MetricSnapshot(Instant.ofEpochMilli(timestampSecond * 1000), Metric.cpu.from(values), Metric.memory.from(values), Metric.disk.from(values), (long)Metric.generation.from(values), - Metric.inService.from(values) > 0))); + Metric.inService.from(values) > 0, + clusterIsStable(node.get(), applicationNodes, nodeRepository)))); + } + + private boolean clusterIsStable(Node node, NodeList applicationNodes, NodeRepository nodeRepository) { + ClusterSpec cluster = node.allocation().get().membership().cluster(); + return Autoscaler.stable(applicationNodes.cluster(cluster.id()).asList(), nodeRepository); } private void consumeServiceMetrics(String hostname, Inspector node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java index 33bdc746678..4afc876056a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java @@ -55,9 +55,6 @@ public class MetricsV2MetricsFetcher extends AbstractComponent implements Metric public Collection<Pair<String, MetricSnapshot>> fetchMetrics(ApplicationId application) { NodeList applicationNodes = nodeRepository.list(application).state(Node.State.active); - // Do not try to draw conclusions from utilization while unstable - if (Autoscaler.unstable(applicationNodes.asList(), nodeRepository)) return Collections.emptyList(); - Optional<Node> metricsV2Container = applicationNodes.container() .matching(node -> expectedUp(node)) .stream() @@ -65,8 +62,7 @@ public class MetricsV2MetricsFetcher extends AbstractComponent implements Metric if (metricsV2Container.isEmpty()) return Collections.emptyList(); // Consumer 'autoscaling' defined in com.yahoo.vespa.model.admin.monitoring.MetricConsumer String url = "http://" + metricsV2Container.get().hostname() + ":" + 4080 + apiPath + "?consumer=autoscaling"; - String response = httpClient.get(url); - return new MetricsResponse(response).metrics(); + return new MetricsResponse(httpClient.get(url), applicationNodes, nodeRepository).metrics(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index 6cba3928b8f..bebe87929ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -7,6 +7,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -41,6 +42,10 @@ public class NodeTimeseries { return new NodeTimeseries(hostname(), list); } + public NodeTimeseries filter(Predicate<MetricSnapshot> filter) { + return new NodeTimeseries(hostname, snapshots.stream().filter(filter).collect(Collectors.toList())); + } + public NodeTimeseries justAfter(Instant oldestTime) { return new NodeTimeseries(hostname, snapshots.stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index bf013810b2f..b3632a4e82a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -89,6 +89,8 @@ public class QuestMetricsDb implements MetricsDb { row.putFloat(3, (float)snapshot.getSecond().memory()); row.putFloat(4, (float)snapshot.getSecond().disk()); row.putLong(5, snapshot.getSecond().generation()); + row.putBool(6, snapshot.getSecond().inService()); + row.putBool(7, snapshot.getSecond().stable()); row.append(); } writer.commit(); @@ -152,7 +154,8 @@ public class QuestMetricsDb implements MetricsDb { try (SqlCompiler compiler = new SqlCompiler(engine)) { compiler.compile("create table " + tableName + - " (hostname string, at timestamp, cpu_util float, mem_total_util float, disk_util float, application_generation long)" + + " (hostname string, at timestamp, cpu_util float, mem_total_util float, disk_util float," + + " application_generation long, inService boolean, stable boolean)" + " timestamp(at)" + "PARTITION BY DAY;", context); @@ -200,7 +203,8 @@ public class QuestMetricsDb implements MetricsDb { record.getFloat(3), record.getFloat(4), record.getLong(5), - true)); + record.getBool(6), + record.getBool(7))); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index ff7bc9393bd..0d5ff23f32b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -43,7 +43,7 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { int warnings = 0; for (ApplicationId application : activeNodesByApplication().keySet()) { try { - metricsDb.add(filter(metricsFetcher.fetchMetrics(application))); + metricsDb.add(metricsFetcher.fetchMetrics(application)); } catch (Exception e) { // TODO: Don't warn if this only happens occasionally @@ -59,11 +59,4 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { return warnings == 0; } - /** Filter out uninformative snapshots before storing */ - private Collection<Pair<String, MetricSnapshot>> filter(Collection<Pair<String, MetricSnapshot>> snapshots) { - return snapshots.stream() - .filter(snapshot -> snapshot.getSecond().inService()) - .collect(Collectors.toList()); - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 5393aa7cfb8..a821bde5b26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -225,6 +225,40 @@ public class AutoscalingTest { } @Test + public void not_using_out_of_service_measurements() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources(2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(5, 1, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 2, 1, resources); + tester.addMeasurements(0.5f, 0.6f, 0.7f, 1, false, true, 120, application1); + assertTrue("Not scaling up since nodes were measured while cluster was unstable", + tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + } + + @Test + public void not_using_unstable_measurements() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources(2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(5, 1, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 2, 1, resources); + tester.addMeasurements(0.5f, 0.6f, 0.7f, 1, true, false, 120, application1); + assertTrue("Not scaling up since nodes were measured while cluster was unstable", + tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + } + + @Test public void test_autoscaling_group_size_1() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 3faa4c244ee..e6a921d055e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -138,6 +138,7 @@ class AutoscalingTester { memory, disk, 0, + true, true)))); } } @@ -168,12 +169,18 @@ class AutoscalingTester { memory, disk, 0, + true, true)))); } } } - public void addMeasurements(float cpu, float memory, float disk, int generation, int count, ApplicationId applicationId) { + public void addMeasurements(float cpu, float memory, float disk, int generation, int count, ApplicationId applicationId) { + addMeasurements(cpu, memory, disk, generation, true, true, count, applicationId); + } + + public void addMeasurements(float cpu, float memory, float disk, int generation, boolean inService, boolean stable, + int count, ApplicationId applicationId) { List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active); for (int i = 0; i < count; i++) { clock().advance(Duration.ofMinutes(1)); @@ -183,7 +190,8 @@ class AutoscalingTester { memory, disk, generation, - true)))); + inService, + stable)))); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index dd991f15087..0a62e5b6a68 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class MetricsV2MetricsFetcherTest { @@ -70,14 +71,17 @@ public class MetricsV2MetricsFetcherTest { assertEquals(0.15, values.get(0).getSecond().memory(), delta); assertEquals(0.20, values.get(0).getSecond().disk(), delta); assertEquals(3, values.get(0).getSecond().generation(), delta); + assertTrue(values.get(0).getSecond().stable()); } { + httpClient.cannedResponse = cannedResponseForApplication2; try (Mutex lock = tester.nodeRepository().lock(application1)) { - tester.nodeRepository().write(tester.nodeRepository().getNodes(application1, Node.State.active) + tester.nodeRepository().write(tester.nodeRepository().getNodes(application2, Node.State.active) .get(0).retire(tester.clock().instant()), lock); } - assertTrue("No metrics fetching while unstable", fetcher.fetchMetrics(application1).isEmpty()); + List<Pair<String, MetricSnapshot>> values = new ArrayList<>(fetcher.fetchMetrics(application2)); + assertFalse(values.get(0).getSecond().stable()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index dba19d675dd..681fc082019 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -40,7 +40,7 @@ public class NodeMetricsDbTest { MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository()); Collection<Pair<String, MetricSnapshot>> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { - values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), 0.9f, 0.6f, 0.6f, 0, true))); + values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), 0.9f, 0.6f, 0.6f, 0, true, false))); clock.advance(Duration.ofMinutes(120)); } db.add(values); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index b97d5136485..b2c9da4d22c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; import com.yahoo.io.IOUtils; import com.yahoo.test.ManualClock; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -16,6 +17,7 @@ import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * Tests the Quest metrics db. @@ -34,6 +36,7 @@ public class QuestMetricsDbTest { ManualClock clock = new ManualClock("2020-10-01T00:00:00"); QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); Instant startTime = clock.instant(); + clock.advance(Duration.ofSeconds(1)); db.add(timeseries(1000, Duration.ofSeconds(1), clock, "host1", "host2", "host3")); @@ -111,6 +114,46 @@ public class QuestMetricsDbTest { assertEquals(24 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size()); } + /** To manually test that we can read existing data */ + @Ignore + @Test + public void testReadingExistingData() { + String dataDir = "data/QuestMetricsDbExistingData"; + if ( ! new File(dataDir).exists()) { + System.out.println("No existing data to check"); + return; + } + IOUtils.createDirectory(dataDir + "/metrics"); + ManualClock clock = new ManualClock("2020-10-01T00:00:00"); + clock.advance(Duration.ofSeconds(10)); // Adjust to end time of data written + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + + List<NodeTimeseries> timeseries = db.getNodeTimeseries(clock.instant().minus(Duration.ofSeconds(10)), Set.of("host1")); + assertFalse("Could read existing data", timeseries.isEmpty()); + assertEquals(10, timeseries.get(0).size()); + + System.out.println("Existing data:"); + for (var snapshot : timeseries.get(0).asList()) + System.out.println(" " + snapshot); + } + + /** To update data for the manual test above */ + @Ignore + @Test + public void updateExistingData() { + String dataDir = "data/QuestMetricsDbExistingData"; + IOUtils.recursiveDeleteDir(new File(dataDir)); + IOUtils.createDirectory(dataDir + "/metrics"); + ManualClock clock = new ManualClock("2020-10-01T00:00:00"); + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + Instant startTime = clock.instant(); + db.add(timeseries(10, Duration.ofSeconds(1), clock, "host1")); + + int added = db.getNodeTimeseries(startTime, Set.of("host1")).get(0).asList().size(); + System.out.println("Added " + added + " rows of data"); + db.close(); + } + private Collection<Pair<String, MetricSnapshot>> timeseries(int countPerHost, Duration sampleRate, ManualClock clock, String ... hosts) { Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>(); @@ -121,6 +164,7 @@ public class QuestMetricsDbTest { i * 0.2, i * 0.4, i % 100, + true, true))); clock.advance(sampleRate); } @@ -136,8 +180,10 @@ public class QuestMetricsDbTest { i * 0.2, i * 0.4, i % 100, - true))); + true, + false))); } return timeseries; } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java index fadcd40ad0a..f47202e1580 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java @@ -77,6 +77,7 @@ public class AutoscalingMaintainerTester { mem, disk, generation, + true, true)))); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index 722911569de..4f0b0d55742 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -51,9 +52,8 @@ public class NodeMetricsDbMaintainerTest { List<MetricSnapshot> allSnapshots = timeseriesList.stream() .flatMap(timeseries -> timeseries.asList().stream()) .collect(Collectors.toList()); - assertEquals("Snapshot from the node not in service is filtered out", - 1, allSnapshots.size()); - assertEquals(0.14, allSnapshots.get(0).cpu(), 0.000001); + assertTrue(allSnapshots.stream().anyMatch(snapshot -> snapshot.inService())); + assertTrue(allSnapshots.stream().anyMatch(snapshot -> ! snapshot.inService())); } private static class MockHttpClient implements MetricsV2MetricsFetcher.HttpClient { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 15966a4c44b..35da31077f4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -81,6 +81,7 @@ public class ScalingSuggestionsMaintainerTest { memory, disk, generation, + true, true)))); } } |