Merge pull request #16771 from vespa-engine/revert-16753-revert-16745-bratseth/query-rate-autoscaling-metric

Revert "Revert "Add query.rate to the autoscaling metrics""

9 files changed, 59 insertions, 29 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
index 347d0ebce6c..6bb3a62f234 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
@@ -20,7 +20,8 @@ public class AutoscalingMetrics {
                                      "mem.util",
                                      "disk.util",
                                      "application_generation",
-                                     "in_service"));
+                                     "in_service",
+                                     "queries.rate"));
     }
 
     private static Set<Metric> metrics(String ... metrics) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java
index d5072475cd9..82812592809 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricSnapshot.java
@@ -18,9 +18,11 @@ public class MetricSnapshot implements Comparable<MetricSnapshot> {
     private final long generation;
     private final boolean inService;
     private final boolean stable;
+    private final double queryRate;
 
-    public MetricSnapshot(Instant at, double cpu, double memory, double disk, long generation,
-                          boolean inService, boolean stable) {
+    public MetricSnapshot(Instant at, double cpu, double memory, double disk,
+                          long generation, boolean inService, boolean stable,
+                          double queryRate) {
         this.at = at;
         this.cpu = cpu;
         this.memory = memory;
@@ -28,6 +30,7 @@ public class MetricSnapshot implements Comparable<MetricSnapshot> {
         this.generation = generation;
         this.inService = inService;
         this.stable = stable;
+        this.queryRate = queryRate;
     }
 
     public Instant at() { return at; }
@@ -35,6 +38,9 @@ public class MetricSnapshot implements Comparable<MetricSnapshot> {
     public double memory() { return memory; }
     public double disk() { return disk; }
 
+    /** Queries per second */
+    public double queryRate() { return queryRate; }
+
     /** The configuration generation at the time of this measurement, or -1 if not known */
     public long generation() { return generation; }
 
@@ -53,7 +59,8 @@ public class MetricSnapshot implements Comparable<MetricSnapshot> {
                                       " disk: " + disk +
                                       " generation: " + generation +
                                       " inService: " + inService +
-                                      " stable: " + stable;
+                                      " stable: " + stable +
+                                      " queryRate: " + queryRate;
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
index 341a2028f05..d6661b89536 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -65,7 +65,8 @@ public class MetricsResponse {
                                                                 Metric.disk.from(values),
                                                                 (long)Metric.generation.from(values),
                                                                 Metric.inService.from(values) > 0,
-                                                                 clusterIsStable(node.get(), applicationNodes, nodeRepository))));
+                                                                clusterIsStable(node.get(), applicationNodes, nodeRepository),
+                                                                Metric.queryRate.from(values))));
     }
 
     private boolean clusterIsStable(Node node, NodeList applicationNodes, NodeRepository nodeRepository) {
@@ -115,6 +116,11 @@ public class MetricsResponse {
             public String metricResponseName() { return "in_service"; }
             double convertValue(double metricValue) { return (float)metricValue; } // Really a boolean
             double defaultValue() { return 1.0; }
+        },
+        queryRate { // queries per second
+            public String metricResponseName() { return "queries.rate"; }
+            double convertValue(double metricValue) { return (float)metricValue; }
+            double defaultValue() { return 0.0; }
         };
 
         /** The name of this metric as emitted from its source */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 81401f9f743..37e70e3539a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -110,6 +110,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
             row.putLong(5, snapshot.getSecond().generation());
             row.putBool(6, snapshot.getSecond().inService());
             row.putBool(7, snapshot.getSecond().stable());
+            row.putFloat(8, (float)snapshot.getSecond().queryRate());
             row.append();
         }
         writer.commit();
@@ -183,28 +184,17 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     private void ensureExists(String table) {
         SqlExecutionContext context = newContext();
         if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) { // table exists
-            ensureUpdated(table, context);
+            ensureTableIsUpdated(table, context);
         } else {
-            create(table, context);
+            createTable(table, context);
         }
     }
 
-    private void ensureUpdated(String table, SqlExecutionContext context) {
-        try (SqlCompiler compiler = new SqlCompiler(engine)) {
-            if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) {
-                ensureColumnExists("inService", "boolean", table, compiler, context); // TODO: Remove after December 2020
-                ensureColumnExists("stable", "boolean", table, compiler, context); // TODO: Remove after December 2020
-            }
-        } catch (SqlException e) {
-            repair(e);
-        }
-    }
-
-    private void create(String table, SqlExecutionContext context) {
+    private void createTable(String table, SqlExecutionContext context) {
         try (SqlCompiler compiler = new SqlCompiler(engine)) {
             compiler.compile("create table " + table +
                              " (hostname string, at timestamp, cpu_util float, mem_total_util float, disk_util float," +
-                             "  application_generation long, inService boolean, stable boolean)" +
+                             "  application_generation long, inService boolean, stable boolean, queries_rate float)" +
                              " timestamp(at)" +
                              "PARTITION BY DAY;",
                              context);
@@ -216,6 +206,16 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
         }
     }
 
+    private void ensureTableIsUpdated(String table, SqlExecutionContext context) {
+        try (SqlCompiler compiler = new SqlCompiler(engine)) {
+            if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), table)) {
+                ensureColumnExists("queries_rate", "float", table, compiler, context); // TODO: Remove after March 2021
+            }
+        } catch (SqlException e) {
+            repair(e);
+        }
+    }
+
     private void ensureColumnExists(String column, String columnType,
                                     String table, SqlCompiler compiler, SqlExecutionContext context) throws SqlException {
         if (columnNamesOf(table, compiler, context).contains(column)) return;
@@ -272,7 +272,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
                                                          record.getFloat(4),
                                                          record.getLong(5),
                                                          record.getBool(6),
-                                                         record.getBool(7)));
+                                                         record.getBool(7),
+                                                         record.getFloat(8)));
                     }
                 }
             }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index 77c3a5209e2..156542ef1d4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -143,7 +143,8 @@ class AutoscalingTester {
                                                                               disk,
                                                                               0,
                                                                               true,
-                                                                              true))));
+                                                                              true,
+                                                                              0.0))));
             }
         }
     }
@@ -174,7 +175,8 @@ class AutoscalingTester {
                                                                               disk,
                                                                               0,
                                                                               true,
-                                                                              true))));
+                                                                              true,
+                                                                              0.0))));
             }
         }
     }
@@ -195,7 +197,8 @@ class AutoscalingTester {
                                                                               disk,
                                                                               generation,
                                                                               inService,
-                                                                              stable))));
+                                                                              stable,
+                                                                              0.0))));
             }
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
index 02d258313db..c1c94c7dd24 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
@@ -40,7 +40,14 @@ public class NodeMetricsDbTest {
         MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository());
         Collection<Pair<String, MetricSnapshot>> values = new ArrayList<>();
         for (int i = 0; i < 40; i++) {
-            values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(), 0.9f, 0.6f, 0.6f, 0, true, false)));
+            values.add(new Pair<>(node0, new MetricSnapshot(clock.instant(),
+                                                            0.9f,
+                                                            0.6f,
+                                                            0.6f,
+                                                            0,
+                                                            true,
+                                                            false,
+                                                            0.0)));
             clock.advance(Duration.ofMinutes(120));
         }
         db.add(values);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index 6090d2d5f34..70f9d581816 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -54,6 +54,7 @@ public class QuestMetricsDbTest {
         assertEquals(0.2, snapshot.memory(), delta);
         assertEquals(0.4, snapshot.disk(), delta);
         assertEquals(1, snapshot.generation(), delta);
+        assertEquals(30, snapshot.queryRate(), delta);
 
         // Read all from 2 hosts
         List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
@@ -181,7 +182,8 @@ public class QuestMetricsDbTest {
                                                                    i * 0.4,
                                                                    i % 100,
                                                                    true,
-                                                                   true)));
+                                                                   true,
+                                                                   30.0)));
             clock.advance(sampleRate);
         }
         return timeseries;
@@ -197,7 +199,8 @@ public class QuestMetricsDbTest {
                                                                    i * 0.4,
                                                                    i % 100,
                                                                    true,
-                                                                   false)));
+                                                                   false,
+                                                                   0.0)));
         }
         return timeseries;
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java
index 887ce158e09..1b531fd3237 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java
@@ -81,7 +81,8 @@ public class AutoscalingMaintainerTester {
                                                                                      disk,
                                                                                      generation,
                                                                                      true,
-                                                                                     true))));
+                                                                                     true,
+                                                                                     0.0))));
         }
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
index 78965f0b902..d5b7903b94c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java
@@ -131,7 +131,8 @@ public class ScalingSuggestionsMaintainerTest {
                                                                               disk,
                                                                               generation,
                                                                               true,
-                                                                              true))));
+                                                                              true,
+                                                                              0.0))));
         }
     }