Add writeRate cluster metric

8 files changed, 81 insertions, 46 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
index 6bb3a62f234..f5e79ca2723 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java
@@ -1,7 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.admin.monitoring;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -15,17 +18,19 @@ public class AutoscalingMetrics {
     public static final MetricSet autoscalingMetricSet = create();
 
     private static MetricSet create() {
-        return new MetricSet("autoscaling",
-                             metrics("cpu.util",
-                                     "mem.util",
-                                     "disk.util",
-                                     "application_generation",
-                                     "in_service",
-                                     "queries.rate"));
+        List<String> metrics = new ArrayList<>();
+        metrics.add("cpu.util");
+        metrics.add("mem.util");
+        metrics.add("disk.util");
+        metrics.add("application_generation");
+        metrics.add("in_service");
+        metrics.add("queries.rate");
+        metrics.add("feed.http-requests.rate");
+        return new MetricSet("autoscaling", toMetrics(metrics));
     }
 
-    private static Set<Metric> metrics(String ... metrics) {
-        return Arrays.stream(metrics).map(Metric::new).collect(Collectors.toSet());
+    private static Set<Metric> toMetrics(List<String> metrics) {
+        return metrics.stream().map(Metric::new).collect(Collectors.toSet());
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 869738223d4..aa2a43e0d60 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -140,10 +140,10 @@ public class VespaMetricSet {
         metrics.add(new Metric("serverNumConnections.last"));
 
         {
-            List<String> suffices = List.of("sum", "count", "last", "min", "max");
-            addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffices);
-            addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffices);
-            addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffices);
+            List<String> suffixes = List.of("sum", "count", "last", "min", "max");
+            addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffixes);
+            addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffixes);
         }
 
         metrics.add(new Metric("httpapi_latency.max"));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java
index fd8e91584c4..f13d49e35a9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java
@@ -12,12 +12,13 @@ import java.time.Instant;
 public class ClusterMetricSnapshot implements Comparable<ClusterMetricSnapshot> {
 
     private final Instant at;
-
     private final double queryRate;
+    private final double writeRate;
 
-    public ClusterMetricSnapshot(Instant at, double queryRate) {
+    public ClusterMetricSnapshot(Instant at, double queryRate, double writeRate) {
         this.at = at;
         this.queryRate = queryRate;
+        this.writeRate = writeRate;
     }
 
     public Instant at() { return at; }
@@ -25,8 +26,15 @@ public class ClusterMetricSnapshot implements Comparable<ClusterMetricSnapshot>
     /** Queries per second */
     public double queryRate() { return queryRate; }
 
+    /** Write operations per second */
+    public double writeRate() { return writeRate; }
+
     public ClusterMetricSnapshot withQueryRate(double queryRate) {
-        return new ClusterMetricSnapshot(at, queryRate);
+        return new ClusterMetricSnapshot(at, queryRate, writeRate);
+    }
+
+    public ClusterMetricSnapshot withWriteRate(double writeRate) {
+        return new ClusterMetricSnapshot(at, queryRate, writeRate);
     }
 
     @Override
@@ -36,7 +44,12 @@ public class ClusterMetricSnapshot implements Comparable<ClusterMetricSnapshot>
 
     @Override
     public String toString() { return "metrics at " + at + ":" +
-                                      " queryRate: " + queryRate;
+                                      " queryRate: " + queryRate +
+                                      " writeRate: " + writeRate;
+    }
+
+    public static ClusterMetricSnapshot empty(Instant instant) {
+        return new ClusterMetricSnapshot(instant, 0.0, 0.0);
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
index 0fa7a0e0bb1..9c50684c6a4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -79,8 +79,9 @@ public class MetricsResponse {
                                                                     Metric.queryRate.from(values))));
 
         var cluster = node.get().allocation().get().membership().cluster().id();
-        var metrics = clusterMetrics.getOrDefault(cluster, new ClusterMetricSnapshot(at, 0.0));
+        var metrics = clusterMetrics.getOrDefault(cluster, ClusterMetricSnapshot.empty(at));
         metrics = metrics.withQueryRate(metrics.queryRate() + Metric.queryRate.from(values));
+        metrics = metrics.withWriteRate(metrics.queryRate() + Metric.writeRate.from(values));
         clusterMetrics.put(cluster, metrics);
     }
 
@@ -136,6 +137,11 @@ public class MetricsResponse {
             public String metricResponseName() { return "queries.rate"; }
             double convertValue(double metricValue) { return (float)metricValue; }
             double defaultValue() { return 0.0; }
+        },
+        writeRate { // writes per second
+            public String metricResponseName() { return "feed.http-requests.rate"; }
+            double convertValue(double metricValue) { return (float) metricValue; }
+            double defaultValue() { return 0.0; }
         };
 
         /** The name of this metric as emitted from its source */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index efa1de6bb97..1aecd808e2d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -151,6 +151,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
             row.putStr(1, snapshot.getKey().value());
             // (2 is timestamp)
             row.putFloat(3, (float)snapshot.getValue().queryRate());
+            row.putFloat(4, (float)snapshot.getValue().writeRate());
             row.append();
         }
         writer.commit();
@@ -273,7 +274,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     private void createClusterTable(SqlExecutionContext context) {
         try (SqlCompiler compiler = new SqlCompiler(engine)) {
             compiler.compile("create table " + clusterTable +
-                             " (application string, cluster string, at timestamp, queries_rate float)" +
+                             " (application string, cluster string, at timestamp, queries_rate float, write_rate float)" +
                              " timestamp(at)" +
                              "PARTITION BY DAY;",
                              context);
@@ -296,7 +297,13 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     }
 
     private void ensureClusterTableIsUpdated(SqlExecutionContext context) {
-        // Nothing to do for now
+        try (SqlCompiler compiler = new SqlCompiler(engine)) {
+            if (0 == engine.getStatus(context.getCairoSecurityContext(), new Path(), nodeTable)) {
+                ensureColumnExists("write_rate", "float", nodeTable, compiler, context); // TODO: Remove after March 2021
+            }
+        } catch (SqlException e) {
+            repair(e);
+        }
     }
 
     private void ensureColumnExists(String column, String columnType,
@@ -379,7 +386,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
                     String clusterId = record.getStr(1).toString();
                     if (cluster.value().equals(clusterId)) {
                         snapshots.add(new ClusterMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(2) / 1000),
-                                                                record.getFloat(3)));
+                                                                record.getFloat(3),
+                                                                record.getFloat(4)));
                     }
                 }
             }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index ce3293aa518..1949a6116d8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -242,7 +242,7 @@ class AutoscalingTester {
                                          IntFunction<Double> queryRate) {
         Instant time = clock().instant();
         for (int i = 0; i < measurements; i++) {
-            db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i))));
+            db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i), 0.0)));
             time = time.plus(Duration.ofMinutes(5));
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java
index 89fe2d76159..9a08e7b3279 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseriesTest.java
@@ -29,14 +29,14 @@ public class ClusterTimeseriesTest {
     @Test
     public void test_constant_rate_short() {
         var clock = new ManualClock();
-        var timeseries = new ClusterTimeseries(cluster, rate(10, clock, t -> 50.0));
+        var timeseries = new ClusterTimeseries(cluster, queryRate(10, clock, t -> 50.0));
         assertEquals(0.1, timeseries.maxQueryGrowthRate(), delta);
     }
 
     @Test
     public void test_constant_rate_long() {
         var clock = new ManualClock();
-        var timeseries = new ClusterTimeseries(cluster, rate(10000, clock, t -> 50.0));
+        var timeseries = new ClusterTimeseries(cluster, queryRate(10000, clock, t -> 50.0));
         assertEquals(0.0, timeseries.maxQueryGrowthRate(), delta);
     }
 
@@ -44,9 +44,9 @@ public class ClusterTimeseriesTest {
     public void test_single_spike() {
         var clock = new ManualClock();
         var snapshots = new ArrayList<ClusterMetricSnapshot>();
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
-        snapshots.addAll(rate(  10, clock, t -> 400.0));
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(10, clock, t -> 400.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
         assertEquals((400-50)/5.0/50.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta);
     }
 
@@ -54,13 +54,13 @@ public class ClusterTimeseriesTest {
     public void test_three_spikes() {
         var clock = new ManualClock();
         var snapshots = new ArrayList<ClusterMetricSnapshot>();
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
-        snapshots.addAll(rate(  10, clock, t -> 400.0));
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
-        snapshots.addAll(rate(  10, clock, t -> 600.0));
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
-        snapshots.addAll(rate(  10, clock, t -> 800.0));
-        snapshots.addAll(rate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(10, clock, t -> 400.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(10, clock, t -> 600.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
+        snapshots.addAll(queryRate(10, clock, t -> 800.0));
+        snapshots.addAll(queryRate(1000, clock, t ->  50.0));
         assertEquals((800-50)/5.0/50.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta);
     }
 
@@ -68,15 +68,15 @@ public class ClusterTimeseriesTest {
     public void test_single_hill() {
         var clock = new ManualClock();
         var snapshots = new ArrayList<ClusterMetricSnapshot>();
-        snapshots.addAll(rate(100, clock, t ->  (double)t));
-        snapshots.addAll(rate(100, clock, t -> 100.0 - t));
+        snapshots.addAll(queryRate(100, clock, t ->  (double)t));
+        snapshots.addAll(queryRate(100, clock, t -> 100.0 - t));
         assertEquals(1/5.0, new ClusterTimeseries(cluster, snapshots).maxQueryGrowthRate(), delta);
     }
 
     @Test
     public void test_smooth_curve() {
         var clock = new ManualClock();
-        var timeseries = new ClusterTimeseries(cluster, rate(10000, clock,
+        var timeseries = new ClusterTimeseries(cluster, queryRate(10000, clock,
                                                              t -> 10.0 + 100.0 * Math.sin(t)));
         assertEquals(0.26, timeseries.maxQueryGrowthRate(), delta);
     }
@@ -84,7 +84,7 @@ public class ClusterTimeseriesTest {
     @Test
     public void test_smooth_curve_small_variation() {
         var clock = new ManualClock();
-        var timeseries = new ClusterTimeseries(cluster, rate(10000, clock,
+        var timeseries = new ClusterTimeseries(cluster, queryRate(10000, clock,
                                                                            t -> 1000.0 + 10.0 * Math.sin(t)));
         assertEquals(0.0, timeseries.maxQueryGrowthRate(), delta);
     }
@@ -92,15 +92,15 @@ public class ClusterTimeseriesTest {
     @Test
     public void test_two_periods() {
         var clock = new ManualClock();
-        var timeseries = new ClusterTimeseries(cluster, rate(10000, clock,
+        var timeseries = new ClusterTimeseries(cluster, queryRate(10000, clock,
                                                              t -> 10.0 + 100.0 * Math.sin(t) + 80.0 * Math.sin(10 * t)) );
         assertEquals(1.765, timeseries.maxQueryGrowthRate(), delta);
     }
 
-    private List<ClusterMetricSnapshot> rate(int count, ManualClock clock, IntFunction<Double> rate) {
+    private List<ClusterMetricSnapshot> queryRate(int count, ManualClock clock, IntFunction<Double> rate) {
         List<ClusterMetricSnapshot> snapshots = new ArrayList<>();
         for (int i = 0; i < count; i++) {
-            snapshots.add(new ClusterMetricSnapshot(clock.instant(), rate.apply(i)));
+            snapshots.add(new ClusterMetricSnapshot(clock.instant(), rate.apply(i), 0.0));
             clock.advance(Duration.ofMinutes(5));
         }
         return snapshots;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index 18b92fa6b0f..f465a57d76a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -91,12 +91,12 @@ public class QuestMetricsDbTest {
         var application2 = ApplicationId.from("t1", "a2", "i1");
         var cluster1 = new ClusterSpec.Id("cluster1");
         var cluster2 = new ClusterSpec.Id("cluster2");
-        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0)));
-        db.addClusterMetrics(application1, Map.of(cluster2, new ClusterMetricSnapshot(clock.instant(), 60.0)));
+        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0, 15.0)));
+        db.addClusterMetrics(application1, Map.of(cluster2, new ClusterMetricSnapshot(clock.instant(), 60.0, 30.0)));
         clock.advance(Duration.ofMinutes(1));
-        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 45.0)));
+        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 45.0, 22.5)));
         clock.advance(Duration.ofMinutes(1));
-        db.addClusterMetrics(application2, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 90.0)));
+        db.addClusterMetrics(application2, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 90.0, 45.0)));
 
         ClusterTimeseries clusterTimeseries11 = db.getClusterTimeseries(application1, cluster1);
         assertEquals(cluster1, clusterTimeseries11.cluster());
@@ -105,9 +105,11 @@ public class QuestMetricsDbTest {
         ClusterMetricSnapshot snapshot111 = clusterTimeseries11.get(0);
         assertEquals(startTime, snapshot111.at());
         assertEquals(30, snapshot111.queryRate(), delta);
+        assertEquals(15, snapshot111.writeRate(), delta);
         ClusterMetricSnapshot snapshot112 = clusterTimeseries11.get(1);
         assertEquals(startTime.plus(Duration.ofMinutes(1)), snapshot112.at());
         assertEquals(45, snapshot112.queryRate(), delta);
+        assertEquals(22.5, snapshot112.writeRate(), delta);
 
 
         ClusterTimeseries clusterTimeseries12 = db.getClusterTimeseries(application1, cluster2);
@@ -117,6 +119,7 @@ public class QuestMetricsDbTest {
         ClusterMetricSnapshot snapshot121 = clusterTimeseries12.get(0);
         assertEquals(startTime, snapshot121.at());
         assertEquals(60, snapshot121.queryRate(), delta);
+        assertEquals(30, snapshot121.writeRate(), delta);
 
 
         ClusterTimeseries clusterTimeseries21 = db.getClusterTimeseries(application2, cluster1);
@@ -247,7 +250,7 @@ public class QuestMetricsDbTest {
                                                           ClusterSpec.Id cluster) {
         List<ClusterMetricSnapshot> timeseries = new ArrayList<>();
         for (int i = 1; i <= count; i++) {
-            timeseries.add(new ClusterMetricSnapshot(clock.instant(), 30.0));
+            timeseries.add(new ClusterMetricSnapshot(clock.instant(), 30.0, 0.0));
             clock.advance(sampleRate);
         }
         return timeseries;