From f3b9ba770300b7a93e2e71e8a6c073da43a9d6d5 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 3 Mar 2021 18:59:14 +0100 Subject: Revert "Revert "Add query.rate to the autoscaling metrics"" --- .../model/admin/monitoring/AutoscalingMetrics.java | 3 +- .../hosted/provision/autoscale/MetricSnapshot.java | 13 +++++++-- .../provision/autoscale/MetricsResponse.java | 8 +++++- .../hosted/provision/autoscale/QuestMetricsDb.java | 33 +++++++++++----------- .../provision/autoscale/AutoscalingTester.java | 9 ++++-- .../provision/autoscale/NodeMetricsDbTest.java | 9 +++++- .../provision/autoscale/QuestMetricsDbTest.java | 7 +++-- .../maintenance/AutoscalingMaintainerTester.java | 3 +- .../ScalingSuggestionsMaintainerTest.java | 3 +- 9 files changed, 59 insertions(+), 29 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 347d0ebce6c..6bb3a62f234 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -20,7 +20,8 @@ public class AutoscalingMetrics { "mem.util", "disk.util", "application_generation", - "in_service")); + "in_service", + "queries.rate")); } private static Set metrics(String ... metrics) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java index d5072475cd9..82812592809 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java @@ -18,9 +18,11 @@ public class MetricSnapshot implements Comparable { private final long generation; private final boolean inService; private final boolean stable; + private final double queryRate; - public MetricSnapshot(Instant at, double cpu, double memory, double disk, long generation, - boolean inService, boolean stable) { + public MetricSnapshot(Instant at, double cpu, double memory, double disk, + long generation, boolean inService, boolean stable, + double queryRate) { this.at = at; this.cpu = cpu; this.memory = memory; @@ -28,6 +30,7 @@ public class MetricSnapshot implements Comparable { this.generation = generation; this.inService = inService; this.stable = stable; + this.queryRate = queryRate; } public Instant at() { return at; } @@ -35,6 +38,9 @@ public class MetricSnapshot implements Comparable { public double memory() { return memory; } public double disk() { return disk; } + /** Queries per second */ + public double queryRate() { return queryRate; } + /** The configuration generation at the time of this measurement, or -1 if not known */ public long generation() { return generation; } @@ -53,7 +59,8 @@ public class MetricSnapshot implements Comparable { " disk: " + disk + " generation: " + generation + " inService: " + inService + - " stable: " + stable; + " stable: " + stable + + " queryRate: " + queryRate; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 341a2028f05..d6661b89536 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -65,7 +65,8 @@ public class MetricsResponse { Metric.disk.from(values), (long)Metric.generation.from(values), Metric.inService.from(values) > 0, - clusterIsStable(node.get(), applicationNodes, nodeRepository)))); + clusterIsStable(node.get(), applicationNodes, nodeRepository), + Metric.queryRate.from(values)))); } private boolean clusterIsStable(Node node, NodeList applicationNodes, NodeRepository nodeRepository) { @@ -115,6 +116,11 @@ public class MetricsResponse { public String metricResponseName() { return "in_service"; } double convertValue(double metricValue) { return (float)metricValue; } // Really a boolean double defaultValue() { return 1.0; } + }, + queryRate { // queries per second + public String metricResponseName() { return "queries.rate"; } + double convertValue(double metricValue) { return (float)metricValue; } + double defaultValue() { return 0.0; } }; /** The name of this metric as emitted from its source */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 81401f9f743..37e70e3539a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -110,6 +110,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { row.putLong(5, snapshot.getSecond().generation()); row.putBool(6, snapshot.getSecond().inService()); row.putBool(7, snapshot.getSecond().stable()); + row.putFloat(8, (float)snapshot.getSecond().queryRate()); row.append(); } writer.commit(); @@ -183,28 +184,17 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private void ensureExists(String table) { SqlExecutionContext context = newContext(); if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) { // table exists - ensureUpdated(table, context); + ensureTableIsUpdated(table, context); } else { - create(table, context); + createTable(table, context); } } - private void ensureUpdated(String table, SqlExecutionContext context) { - try (SqlCompiler compiler = new SqlCompiler(engine)) { - if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) { - ensureColumnExists("inService", "boolean", table, compiler, context); // TODO: Remove after December 2020 - ensureColumnExists("stable", "boolean", table, compiler, context); // TODO: Remove after December 2020 - } - } catch (SqlException e) { - repair(e); - } - } - - private void create(String table, SqlExecutionContext context) { + private void createTable(String table, SqlExecutionContext context) { try (SqlCompiler compiler = new SqlCompiler(engine)) { compiler.compile("create table " + table + " (hostname string, at timestamp, cpu_util float, mem_total_util float, disk_util float," + - " application_generation long, inService boolean, stable boolean)" + + " application_generation long, inService boolean, stable boolean, queries_rate float)" + " timestamp(at)" + "PARTITION BY DAY;", context); @@ -216,6 +206,16 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } } + private void ensureTableIsUpdated(String table, SqlExecutionContext context) { + try (SqlCompiler compiler = new SqlCompiler(engine)) { + if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) { + ensureColumnExists("queries_rate", "float", table, compiler, context); // TODO: Remove after March 2021 + } + } catch (SqlException e) { + repair(e); + } + } + private void ensureColumnExists(String column, String columnType, String table, SqlCompiler compiler, SqlExecutionContext context) throws SqlException { if (columnNamesOf(table, compiler, context).contains(column)) return; @@ -272,7 +272,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { record.getFloat(4), record.getLong(5), record.getBool(6), - record.getBool(7))); + record.getBool(7), + record.getFloat(8))); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 77c3a5209e2..156542ef1d4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -143,7 +143,8 @@ class AutoscalingTester { disk, 0, true, - true)))); + true, + 0.0)))); } } } @@ -174,7 +175,8 @@ class AutoscalingTester { disk, 0, true, - true)))); + true, + 0.0)))); } } } @@ -195,7 +197,8 @@ class AutoscalingTester { disk, generation, inService, - stable)))); + stable, + 0.0)))); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 02d258313db..c1c94c7dd24 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -40,7 +40,14 @@ public class NodeMetricsDbTest { MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository()); Collection> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { - values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), 0.9f, 0.6f, 0.6f, 0, true, false))); + values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), + 0.9f, + 0.6f, + 0.6f, + 0, + true, + false, + 0.0))); clock.advance(Duration.ofMinutes(120)); } db.add(values); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index 6090d2d5f34..70f9d581816 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -54,6 +54,7 @@ public class QuestMetricsDbTest { assertEquals(0.2, snapshot.memory(), delta); assertEquals(0.4, snapshot.disk(), delta); assertEquals(1, snapshot.generation(), delta); + assertEquals(30, snapshot.queryRate(), delta); // Read all from 2 hosts List nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()), @@ -181,7 +182,8 @@ public class QuestMetricsDbTest { i * 0.4, i % 100, true, - true))); + true, + 30.0))); clock.advance(sampleRate); } return timeseries; @@ -197,7 +199,8 @@ public class QuestMetricsDbTest { i * 0.4, i % 100, true, - false))); + false, + 0.0))); } return timeseries; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java index 887ce158e09..1b531fd3237 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java @@ -81,7 +81,8 @@ public class AutoscalingMaintainerTester { disk, generation, true, - true)))); + true, + 0.0)))); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 78965f0b902..d5b7903b94c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -131,7 +131,8 @@ public class ScalingSuggestionsMaintainerTest { disk, generation, true, - true)))); + true, + 0.0)))); } } -- cgit v1.2.3