Adding vds.distributor.gets.latency to DeploymentMetrics

5 files changed, 31 insertions, 3 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java
index 154c24cb684..3b9e713d425 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java
@@ -28,9 +28,11 @@ public class DeploymentMetricsResponse extends SlimeJsonResponse {
             DeploymentMetricsAggregator aggregator = entry.getValue();
             Cursor metrics = cluster.setObject("metrics");
             aggregator.aggregateQueryRate().ifPresent(queryRate -> metrics.setDouble("queriesPerSecond", queryRate));
+            aggregator.aggregateReadRate().ifPresent(readRate -> metrics.setDouble("readsPerSecond", readRate));
             aggregator.aggregateFeedRate().ifPresent(feedRate -> metrics.setDouble("feedPerSecond", feedRate));
             aggregator.aggregateDocumentCount().ifPresent(documentCount -> metrics.setDouble("documentCount", documentCount));
             aggregator.aggregateQueryLatency().ifPresent(queryLatency -> metrics.setDouble("queryLatency",queryLatency));
+            aggregator.aggregateReadLatency().ifPresent(readLatency -> metrics.setDouble("readLatency", readLatency));
             aggregator.aggregateFeedLatency().ifPresent(feedLatency -> metrics.setDouble("feedLatency", feedLatency));
 
             aggregator.memoryUsage().ifPresent(memory -> {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
index e453079cffa..907928b1d40 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java
@@ -134,8 +134,14 @@ public class ClusterDeploymentMetricsRetriever {
             }
             case VESPA_QRSERVER -> optionalDouble(values.field("query_latency.sum")).ifPresent(qlSum ->
                     aggregator.get().addQrLatency(qlSum, values.field("query_latency.count").asDouble()));
-            case VESPA_DISTRIBUTOR -> optionalDouble(values.field("vds.distributor.docsstored.average"))
-                    .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount));
+            case VESPA_DISTRIBUTOR -> {
+                optionalDouble(values.field("vds.distributor.docsstored.average"))
+                        .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount));
+                // TODO: Replace with container metrics when available. Using distributor get metrics as a proxy for container metrics
+                optionalDouble(values.field("vds.distributor.gets.latency.sum")).ifPresent(rlSum ->
+                        aggregator.get().addReadLatency(rlSum, values.field("vds.distributor.gets.latency.count").asDouble()));
+            }
+
             case VESPA_CONTAINER_CLUSTERCONTROLLER ->
                     optionalDouble(values.field(ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION.max())).ifPresent(memoryUtil ->
                             aggregator.get()
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
index 50cad45889b..2e2525fa459 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java
@@ -10,6 +10,7 @@ import java.util.Optional;
 public class DeploymentMetricsAggregator {
 
     private LatencyMetrics feed;
+    private LatencyMetrics read;
     private LatencyMetrics qr;
     private LatencyMetrics container;
     private Double documentCount;
@@ -21,6 +22,11 @@ public class DeploymentMetricsAggregator {
         return this;
     }
 
+    public synchronized DeploymentMetricsAggregator addReadLatency(double sum, double count) {
+        this.read = combineLatency(this.read, sum, count);
+        return this;
+    }
+
     public synchronized DeploymentMetricsAggregator addQrLatency(double sum, double count) {
         this.qr = combineLatency(this.qr, sum, count);
         return this;
@@ -54,6 +60,14 @@ public class DeploymentMetricsAggregator {
         return Optional.ofNullable(feed).map(m -> m.count / 60);
     }
 
+    public Optional<Double> aggregateReadLatency() {
+        return Optional.ofNullable(read).map(m -> m.sum / m.count).filter(num -> !num.isNaN());
+    }
+
+    public Optional<Double> aggregateReadRate() {
+        return Optional.ofNullable(read).map(m -> m.count / 60);
+    }
+
     public Optional<Double> aggregateQueryLatency() {
         if (container == null && qr == null) return Optional.empty();
         var c = Optional.ofNullable(container).orElseGet(LatencyMetrics::new);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
index 49a395d5a9e..96456a898a8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java
@@ -68,6 +68,8 @@ public class ClusterDeploymentMetricsRetrieverTest {
         compareAggregators(
                 new DeploymentMetricsAggregator()
                         .addDocumentCount(6000.0)
+                        .addReadLatency(3000 , 43)
+                        .addReadLatency(3000 , 43) // there are 2 content nodes
                         .addMemoryUsage(0.89074, 0.8)
                         .addDiskUsage(0.83517, 0.75),
                 aggregatorMap.get(expectedContentCluster)
@@ -105,8 +107,10 @@ public class ClusterDeploymentMetricsRetrieverTest {
 
         compareOptionals(expected.aggregateDocumentCount(), actual.aggregateDocumentCount(), assertDoubles);
         compareOptionals(expected.aggregateQueryRate(), actual.aggregateQueryRate(), assertDoubles);
+        compareOptionals(expected.aggregateReadRate(), actual.aggregateReadRate(), assertDoubles);
         compareOptionals(expected.aggregateFeedRate(), actual.aggregateFeedRate(), assertDoubles);
         compareOptionals(expected.aggregateQueryLatency(), actual.aggregateQueryLatency(), assertDoubles);
+        compareOptionals(expected.aggregateReadLatency(), actual.aggregateReadLatency(), assertDoubles);
         compareOptionals(expected.aggregateFeedLatency(), actual.aggregateFeedLatency(), assertDoubles);
         compareOptionals(expected.diskUsage(), actual.diskUsage(), (a, b) -> assertDoubles.accept(a.util(), b.util()));
         compareOptionals(expected.diskUsage(), actual.diskUsage(), (a, b) -> assertDoubles.accept(a.feedBlockLimit(), b.feedBlockLimit()));
diff --git a/configserver/src/test/resources/metrics/content_metrics.json b/configserver/src/test/resources/metrics/content_metrics.json
index 2f88dc91709..c8e97dca264 100644
--- a/configserver/src/test/resources/metrics/content_metrics.json
+++ b/configserver/src/test/resources/metrics/content_metrics.json
@@ -6,7 +6,9 @@
       "metrics": [
         {
           "values": {
-            "vds.distributor.docsstored.average": 3000
+            "vds.distributor.docsstored.average": 3000,
+            "vds.distributor.gets.latency.count": 43.0,
+            "vds.distributor.gets.latency.sum": 3000
           },
           "dimensions": {
             "clustertype": "content",