Fetch and aggregate correct metrics

5 files changed, 152 insertions, 102 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
index 8c8b69d8945..237a1e6dccb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
@@ -32,11 +32,11 @@ public class MetricsResponse extends HttpResponse {
                 Cursor clusterCursor = clusterArray.addObject();
                 Metrics metrics = clusterMetrics.getValue();
                 clusterCursor.setString("clusterName", clusterMetrics.getKey());
-                clusterCursor.setDouble("queriesPerSecond", metrics.getQueriesPerSecond());
-                clusterCursor.setDouble("writesPerSecond", metrics.getWritesPerSecond());
-                clusterCursor.setDouble("documentCount", metrics.getDocumentCount());
-                clusterCursor.setDouble("queryLatencyMillis", metrics.getQueryLatencyMillis());
-                clusterCursor.setDouble("writeLatencyMillis", metrics.getWriteLatencyMills());
+                metrics.aggregateQueryRate().ifPresent(queryrate -> clusterCursor.setDouble("queriesPerSecond", queryrate));
+                metrics.aggregateFeedRate().ifPresent(feedRate -> clusterCursor.setDouble("writesPerSecond", feedRate));
+                metrics.aggregateDocumentCount().ifPresent(documentCount -> clusterCursor.setDouble("documentCount", documentCount));
+                metrics.aggregateQueryLatency().ifPresent(queryLatency -> clusterCursor.setDouble("queryLatencyMillis",queryLatency));
+                metrics.aggregateFeedLatency().ifPresent(feedLatency -> clusterCursor.setDouble("feedLatency", feedLatency));
                 clusterCursor.setLong("timestamp", metrics.getTimestamp().getEpochSecond());
             }
         }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java
index 8532888dc3d..2fcfbf013bc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java
@@ -2,74 +2,89 @@
 package com.yahoo.vespa.config.server.metrics;
 
 import java.time.Instant;
-import java.util.List;
+import java.util.Optional;
 
 /**
  * @author olaa
  */
 public class Metrics {
 
-    private final double queriesPerSecond;
-    private final double writesPerSecond;
-    private final double documentCount;
-    private final double queryLatencyMillis;
-    private final double writeLatencyMills;
-    private final Instant timestamp;
-
-    public Metrics(double queriesPerSecond, double writesPerSecond, double documentCount,
-                             double queryLatencyMillis, double writeLatencyMills, Instant timestamp) {
-        this.queriesPerSecond = queriesPerSecond;
-        this.writesPerSecond = writesPerSecond;
-        this.documentCount = documentCount;
-        this.queryLatencyMillis = queryLatencyMillis;
-        this.writeLatencyMills = writeLatencyMills;
-        this.timestamp = timestamp;
+    double feedLatencySum;
+    double feedLatencyCount;
+    double qrQueryLatencySum;
+    double qrQueryLatencyCount;
+    double containerQueryLatencySum;
+    double containerQueryLatencyCount;
+    double documentCount;
+    Instant timestamp;
+
+    public void addFeedLatencySum(double feedLatencySum) {
+        this.feedLatencySum += feedLatencySum;
     }
 
+    public void addFeedLatencyCount(double feedLatencyCount) {
+        this.feedLatencyCount += feedLatencyCount;
+    }
+
+    public void addQrQueryLatencyCount(double qrQueryLatencyCount) {
+        this.qrQueryLatencyCount += qrQueryLatencyCount;
+    }
+
+    public void addQrQueryLatencySum(double qrQueryLatencySum) {
+        this.qrQueryLatencySum += qrQueryLatencySum;
+    }
+
+    public void addContainerQueryLatencyCount(double containerQueryLatencyCount) {
+        this.containerQueryLatencyCount += containerQueryLatencyCount;
+    }
 
-    public double getQueriesPerSecond() {
-        return queriesPerSecond;
+    public void addContainerQueryLatencySum(double containerQueryLatencySum) {
+        this.containerQueryLatencySum += containerQueryLatencySum;
     }
 
-    public double getWritesPerSecond() {
-        return writesPerSecond;
+    public void addDocumentCount(double documentCount) {
+        this.documentCount += documentCount;
     }
 
-    public double getDocumentCount() {
-        return documentCount;
+    public Optional<Double> aggregateFeedLatency() {
+        if (isZero(feedLatencySum) || isZero(feedLatencyCount)) return Optional.empty();
+        return Optional.of(feedLatencySum / feedLatencyCount);
     }
 
-    public double getQueryLatencyMillis() {
-        return queryLatencyMillis;
+    public Optional<Double> aggregateFeedRate() {
+        if (isZero(feedLatencyCount)) return Optional.empty();
+        return Optional.of(feedLatencyCount / 60);
     }
 
-    public double getWriteLatencyMills() {
-        return writeLatencyMills;
+    public Optional<Double> aggregateQueryLatency() {
+        if (isZero(containerQueryLatencyCount, containerQueryLatencySum) && isZero(qrQueryLatencyCount, qrQueryLatencySum)) return Optional.empty();
+        return Optional.of((containerQueryLatencySum + qrQueryLatencySum) / (containerQueryLatencyCount + qrQueryLatencyCount));
+    }
+
+    public Optional<Double> aggregateQueryRate() {
+        if (isZero(containerQueryLatencyCount) && isZero(qrQueryLatencyCount)) return Optional.empty();
+        return Optional.of((containerQueryLatencyCount + qrQueryLatencyCount) / 60);
+    }
+
+    public Optional<Double> aggregateDocumentCount() {
+        if (isZero(documentCount)) return Optional.empty();
+        return Optional.of(documentCount);
+    }
+
+    public void setTimestamp(Instant timestamp) {
+        this.timestamp = timestamp;
     }
 
     public Instant getTimestamp() {
         return timestamp;
     }
 
-    public static Metrics averagedMetrics(List<Metrics> metrics) {
-        return new Metrics(
-                metrics.stream().mapToDouble(Metrics::getQueriesPerSecond).sum() / metrics.size(),
-                metrics.stream().mapToDouble(Metrics::getWritesPerSecond).sum() / metrics.size(),
-                metrics.stream().mapToDouble(Metrics::getDocumentCount).sum() / metrics.size(),
-                metrics.stream().mapToDouble(Metrics::getQueryLatencyMillis).sum() / metrics.size(),
-                metrics.stream().mapToDouble(Metrics::getWriteLatencyMills).sum() / metrics.size(),
-                metrics.stream().findAny().get().timestamp
-        );
+    private boolean isZero(double... values) {
+        boolean isZero = false;
+        for (double value : values) {
+            isZero |= Math.abs(value) < 0.001;
+        }
+        return isZero;
     }
 
-    public static Metrics accumulatedMetrics(List<Metrics> metrics) {
-        return new Metrics(
-                metrics.stream().mapToDouble(Metrics::getQueriesPerSecond).sum(),
-                metrics.stream().mapToDouble(Metrics::getWritesPerSecond).sum() ,
-                metrics.stream().mapToDouble(Metrics::getDocumentCount).sum(),
-                metrics.stream().mapToDouble(Metrics::getQueryLatencyMillis).sum(),
-                metrics.stream().mapToDouble(Metrics::getWriteLatencyMills).sum(),
-                metrics.stream().findAny().get().timestamp
-        );
-    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java
index 9b74e46111b..b1a3d0601d7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.metrics;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
-import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.config.server.http.v2.MetricsResponse;
@@ -18,11 +17,8 @@ import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -39,7 +35,7 @@ public class MetricsAggregator {
         Map<ApplicationId, Map<String, Metrics>> aggregatedMetrics = applicationHosts.entrySet().stream()
                 .collect(Collectors.toMap(
                         Map.Entry::getKey,
-                        e-> aggregateMetricsByCluster(e.getValue())));
+                        e -> aggregateMetricsByCluster(e.getValue())));
         return new MetricsResponse(200, aggregatedMetrics);
     }
 
@@ -53,48 +49,59 @@ public class MetricsAggregator {
     }
 
     private Metrics aggregateMetrics(List<URI> hosts) {
-        List<Metrics> metrics= hosts.stream()
-                .map(host -> getMetrics(host))
-                .collect(Collectors.toList());
-        Metrics.averagedMetrics(metrics);
-        return Metrics.accumulatedMetrics(metrics);
+        Metrics clusterMetrics = new Metrics();
+        hosts.stream()
+            .forEach(host -> accumulateMetrics(host, clusterMetrics));
+        return clusterMetrics;
     }
 
-    private Metrics getMetrics(URI hostURI) {
+    private void accumulateMetrics(URI hostURI, Metrics metrics) {
+            Slime responseBody = doMetricsRequest(hostURI);
+            Inspector services = responseBody.get().field("services");
+            services.traverse((ArrayTraverser) (i, servicesInspector) -> {
+                parseService(servicesInspector, metrics);
+
+            });
+
+    }
+
+    private Slime doMetricsRequest(URI hostURI) {
         HttpGet get = new HttpGet(hostURI);
         try {
             HttpResponse response = httpClient.execute(get);
-
             InputStream is = response.getEntity().getContent();
             Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes());
             is.close();
-
-            Inspector nodeMetrics = slime.get().field("node");
-
-            List<Metrics> metricsList = new ArrayList<>();
-            Inspector services = slime.get().field("services");
-            services.traverse((ArrayTraverser) (i, servicesInspector) -> {
-                String serviceName = servicesInspector.field("name").asString();
-
-                Instant timestamp = Instant.ofEpochSecond(servicesInspector.field("timestamp").asLong());
-                Inspector serviceMetrics = servicesInspector.field("metrics");
-                serviceMetrics.traverse((ArrayTraverser) (j, metrics) -> {
-                    Inspector values = metrics.field("values");
-                    double queryCount = values.field("queries.count").asDouble();
-                    double queryLatency = values.field("query_latency.sum").asDouble();
-                    double documentCount = values.field("document.count").asDouble();
-                    double writeCount = values.field("write.count").asDouble();
-                    double writeLatency = values.field("write_latency.sum").asDouble();
-                    logger.log(Level.WARNING, writeLatency + " write latency");
-                    Map<String, Double> map = new HashMap<>();
-                    values.traverse((ObjectTraverser) (key, value) -> map.put(key, value.asDouble()));
-                    metricsList.add(new Metrics(queryCount, writeCount, documentCount, queryLatency, writeLatency, timestamp));
-                });
-
-            });
-            return Metrics.accumulatedMetrics(metricsList);
+            return slime;
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
+
+    private void parseService(Inspector service, Metrics metrics) {
+        String serviceName = service.field("name").asString();
+        Instant timestamp = Instant.ofEpochSecond(service.field("timestamp").asLong());
+        metrics.setTimestamp(timestamp);
+        service.field("metrics").traverse((ArrayTraverser) (i, m) -> {
+            Inspector values = m.field("values");
+            switch (serviceName) {
+                case "container":
+                    metrics.addContainerQueryLatencyCount(values.field("query_latency.count").asDouble());
+                    metrics.addContainerQueryLatencySum(values.field("query_latency.sum").asDouble());
+                    metrics.addFeedLatencyCount(values.field("feed_latency.count").asDouble());
+                    metrics.addFeedLatencySum(values.field("feed_latency.sum").asDouble());
+                case "qrserver":
+                    metrics.addQrQueryLatencyCount(values.field("query_latency.count").asDouble());
+                    metrics.addQrQueryLatencySum(values.field("query_latency.sum").asDouble());
+                case "distributor":
+                    metrics.addDocumentCount(values.field("vds.distributor.docsstored.average").asDouble());
+            }
+        });
+
+    }
+
+    private void parseContainerMetrics(Inspector containerInspector, Metrics metrics) {
+
+    }
+
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java
index 9e8dc88773c..f317322343f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java
@@ -65,20 +65,20 @@ public class MetricsAggregatorTest {
                 "  \"clusters\": [\n" +
                 "   {\n" +
                 "    \"clusterName\": \"cluster1\",\n" +
-                "    \"queriesPerSecond\": 132.0,\n" +
-                "    \"writesPerSecond\": 46.0,\n" +
-                "    \"documentCount\": 600000.0,\n" +
-                "    \"queryLatencyMillis\": 10000.0,\n" +
-                "    \"writeLatencyMillis\": 4000.0,\n" +
+                "    \"queriesPerSecond\": 2.8666666666666667,\n" +
+                "    \"writesPerSecond\": 1.4333333333333333,\n" +
+                "    \"documentCount\": 6000.0,\n" +
+                "    \"queryLatencyMillis\": 116.27906976744185,\n" +
+                "    \"feedLatency\": 69.76744186046511,\n" +
                 "    \"timestamp\": 1557306075\n" +
                 "   },\n" +
                 "   {\n" +
                 "    \"clusterName\": \"cluster2\",\n" +
-                "    \"queriesPerSecond\": 66.0,\n" +
-                "    \"writesPerSecond\": 23.0,\n" +
-                "    \"documentCount\": 300000.0,\n" +
-                "    \"queryLatencyMillis\": 5000.0,\n" +
-                "    \"writeLatencyMillis\": 2000.0,\n" +
+                "    \"queriesPerSecond\": 1.4333333333333333,\n" +
+                "    \"writesPerSecond\": 0.7166666666666667,\n" +
+                "    \"documentCount\": 3000.0,\n" +
+                "    \"queryLatencyMillis\": 116.27906976744185,\n" +
+                "    \"feedLatency\": 69.76744186046511,\n" +
                 "    \"timestamp\": 1557306075\n" +
                 "   }\n" +
                 "  ]\n" +
diff --git a/configserver/src/test/resources/metrics_response b/configserver/src/test/resources/metrics_response
index 7956a73523f..ef992ba067b 100644
--- a/configserver/src/test/resources/metrics_response
+++ b/configserver/src/test/resources/metrics_response
@@ -1,22 +1,50 @@
 {
   "services": [
       {
-        "name":"searchnode",
+        "name":"container",
         "timestamp": 1557306075,
         "metrics": [
             {
               "values": {
-                  "queries.count": 23.0,
+                  "queries.rate": 23.0,
                   "query_latency.sum": 2000,
                   "document.count": 300000,
-                  "write.count": 23.0,
+                  "feed.rate": 23.0,
                   "write_latency.sum": 2000
               }
             },
             {
              "values": {
-                  "queries.count": 43.0,
-                  "query_latency.sum": 3000
+                  "query_latency.count": 43.0,
+                  "query_latency.sum": 3000,
+                  "feed_latency.count": 43.0,
+                  "feed_latency.sum": 3000
+
+              }
+            }
+          ]
+      },
+
+      {
+              "name":"qrs",
+              "timestamp": 1557306075,
+              "metrics": [
+                  {
+                   "values": {
+                        "query_latency.count": 43.0,
+                        "query_latency.sum": 3000
+                    }
+                  }
+                ]
+            },
+        {
+        "name":"distributor",
+        "timestamp": 1557306075,
+        "metrics": [
+            {
+             "values": {
+                  "vds.distributor.docsstored.average": 3000
+
               }
             }
           ]