Include cluster type in response. Other misc changes

7 files changed, 125 insertions, 68 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 7e2fa440bd5..359650df514 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -43,6 +43,7 @@ import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream;
 import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
+import com.yahoo.vespa.config.server.metrics.ClusterInfo;
 import com.yahoo.vespa.config.server.metrics.MetricsRetriever;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.LocalSession;
@@ -588,8 +589,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     public HttpResponse getMetrics() {
         MetricsRetriever metricsRetriever = new MetricsRetriever();
-        Map<ApplicationId, Map<String, List<URI>>> applicationHosts = getHostsPerApplication();
-        return metricsRetriever.retrieveAllMetrics(applicationHosts);
+        Map<ApplicationId, Collection<ClusterInfo>> applicationClusters = getApplicationClusters();
+        return metricsRetriever.retrieveAllMetrics(applicationClusters);
     }
 
     // ---------------- Misc operations ----------------------------------------------------------------
@@ -745,8 +746,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     }
 
     /** Finds all hosts, grouping them by application ID and cluster name */
-    private  Map<ApplicationId, Map<String, List<URI>>> getHostsPerApplication() {
-        Map<ApplicationId, Map<String, List<URI>>> applicationHosts = new HashMap<>();
+    private  Map<ApplicationId, Collection<ClusterInfo>> getApplicationClusters() {
+        Map<ApplicationId, Collection<ClusterInfo>> applicationHosts = new HashMap<>();
         tenantRepository.getAllTenants().stream()
                 .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream())
                 .forEach(applicationId ->{
@@ -757,20 +758,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     }
 
     /** Finds the hosts of an application, grouped by cluster name */
-    private Map<String, List<URI>> getClustersOfApplication(ApplicationId applicationId) {
+    private Collection<ClusterInfo> getClustersOfApplication(ApplicationId applicationId) {
         Application application = getApplication(applicationId);
         Map<String, List<URI>> clusterHosts = new HashMap<>();
+        Map<String, ClusterInfo> clusters = new HashMap<>();
         application.getModel().getHosts().stream()
                 .filter(host -> host.getServices().stream().noneMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver")))
                 .forEach(hostInfo -> {
                             ServiceInfo serviceInfo = hostInfo.getServices().stream().filter(service -> METRICS_PROXY_CONTAINER.serviceName.equals(service.getServiceType()))
                                     .findFirst().orElseThrow(() -> new IllegalArgumentException("Unable to find services " + METRICS_PROXY_CONTAINER.serviceName.toString()));
                             String clusterName = serviceInfo.getProperty("clusterid").orElse("");
+                            String clusterTypeString = serviceInfo.getProperty("clustertype").orElse("");
+                            ClusterInfo.ClusterType clusterType = ClusterInfo.ClusterType.valueOf(clusterTypeString);
                             URI host = URI.create("http://" + hostInfo.getHostname() + ":" + servicePort(serviceInfo) + "/metrics/v1/values");
                             clusterHosts.computeIfAbsent(clusterName, l -> new ArrayList<URI>()).add(host);
+                            clusters.computeIfAbsent(clusterName, c -> new ClusterInfo(clusterName, clusterType)).addHost(host);
                         }
                 );
-        return clusterHosts;
+        return clusters.values();
 
     }
     /** Returns version to use when deploying application in given environment */
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
index c18608f5f64..21aa7753404 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java
@@ -7,6 +7,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.server.http.HttpConfigResponse;
+import com.yahoo.vespa.config.server.metrics.ClusterInfo;
 import com.yahoo.vespa.config.server.metrics.MetricsAggregator;
 
 import java.io.IOException;
@@ -20,25 +21,26 @@ public class MetricsResponse extends HttpResponse {
 
     private final Slime slime = new Slime();
 
-    public MetricsResponse(int status, Map<ApplicationId, Map<String, MetricsAggregator>> aggregatedMetrics) {
+    public MetricsResponse(int status, Map<ApplicationId, Map<ClusterInfo, MetricsAggregator>> aggregatedMetrics) {
         super(status);
 
         Cursor array = slime.setArray();
-        for (Map.Entry<ApplicationId, Map<String, MetricsAggregator>> entry : aggregatedMetrics.entrySet()) {
+        for (Map.Entry<ApplicationId, Map<ClusterInfo, MetricsAggregator>> entry : aggregatedMetrics.entrySet()) {
             Cursor object = array.addObject();
             object.setString("applicationId", entry.getKey().serializedForm());
             Cursor clusterArray = object.setArray("clusters");
-            for (Map.Entry<String, MetricsAggregator> clusterMetrics : entry.getValue().entrySet()) {
+            for (Map.Entry<ClusterInfo, MetricsAggregator> clusterMetrics : entry.getValue().entrySet()) {
                 Cursor clusterCursor = clusterArray.addObject();
                 MetricsAggregator metrics = clusterMetrics.getValue();
-                clusterCursor.setString("clusterName", clusterMetrics.getKey());
+                clusterCursor.setString("clusterId", clusterMetrics.getKey().getClusterId());
+                clusterCursor.setString("clusterType", clusterMetrics.getKey().getClusterType().name());
                 Cursor metricsCursor = clusterCursor.setObject("metrics");
                 metrics.aggregateQueryRate().ifPresent(queryrate -> metricsCursor.setDouble("queriesPerSecond", queryrate));
-                metrics.aggregateFeedRate().ifPresent(feedRate -> metricsCursor.setDouble("writesPerSecond", feedRate));
+                metrics.aggregateFeedRate().ifPresent(feedRate -> metricsCursor.setDouble("feedPerSecond", feedRate));
                 metrics.aggregateDocumentCount().ifPresent(documentCount -> metricsCursor.setDouble("documentCount", documentCount));
-                metrics.aggregateQueryLatency().ifPresent(queryLatency -> metricsCursor.setDouble("queryLatencyMillis",queryLatency));
+                metrics.aggregateQueryLatency().ifPresent(queryLatency -> metricsCursor.setDouble("queryLatency",queryLatency));
                 metrics.aggregateFeedLatency().ifPresent(feedLatency -> metricsCursor.setDouble("feedLatency", feedLatency));
-                clusterCursor.setLong("timestamp", metrics.getTimestamp().getEpochSecond());
+//                clusterCursor.setLong("timestamp", metrics.getTimestamp().getEpochSecond());
             }
         }
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java
new file mode 100644
index 00000000000..ef9a73fedd4
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java
@@ -0,0 +1,44 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.metrics;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author olaa
+ */
+public class ClusterInfo {
+
+    private final String clusterId;
+    private final ClusterType clusterType;
+    private final List<URI> hostnames;
+
+    public ClusterInfo(String clusterId, ClusterType clusterType) {
+        this(clusterId, clusterType, new ArrayList<>());
+    }
+
+    public ClusterInfo(String clusterId, ClusterType clusterType, List<URI> hostnames) {
+        this.clusterId = clusterId;
+        this.clusterType = clusterType;
+        this.hostnames = hostnames;
+    }
+
+    public String getClusterId() {
+        return clusterId;
+    }
+
+    public ClusterType getClusterType() {
+        return clusterType;
+    }
+
+    public List<URI> getHostnames() {
+        return hostnames;
+    }
+
+    public void addHost(URI host) {
+        hostnames.add(host);
+    }
+
+    public enum ClusterType {content, container};
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
index e61623bdc07..d41606a946f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
@@ -17,6 +17,8 @@ import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.time.Instant;
+import java.util.Collection;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
@@ -31,21 +33,20 @@ public class MetricsRetriever {
     private static final Logger logger = Logger.getLogger(MetricsRetriever.class.getName());
     HttpClient httpClient = HttpClientBuilder.create().build();
 
-    public MetricsResponse retrieveAllMetrics(Map<ApplicationId, Map<String, List<URI>>> applicationHosts) {
-        Map<ApplicationId, Map<String, MetricsAggregator>> allMetrics = applicationHosts.entrySet().stream()
+    public MetricsResponse retrieveAllMetrics(Map<ApplicationId, Collection<ClusterInfo>> applicationClusters) {
+        Map<ApplicationId, Map<ClusterInfo, MetricsAggregator>> allMetrics = applicationClusters.entrySet().stream()
                 .collect(Collectors.toMap(
                         Map.Entry::getKey,
                         e -> getMetricsByCluster(e.getValue())));
         return new MetricsResponse(200, allMetrics);
     }
 
-    private Map<String, MetricsAggregator> getMetricsByCluster(Map<String, List<URI>> clusterHosts) {
-        return clusterHosts.entrySet().stream()
-                .collect(Collectors.toMap(
-                        Map.Entry::getKey,
-                        e -> getClusterMetrics(e.getValue())
-                )
-            );
+    private Map<ClusterInfo, MetricsAggregator> getMetricsByCluster(Collection<ClusterInfo> clusters) {
+        return clusters.stream()
+                .collect(LinkedHashMap::new,
+                        (map, item) -> map.put(item, getClusterMetrics(item.getHostnames())),
+                        Map::putAll
+                );
     }
 
     private MetricsAggregator getClusterMetrics(List<URI> hosts) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
index 86874a0dae9..798fecaff65 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
@@ -11,6 +11,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -35,26 +36,26 @@ public class MetricsRetrieverTest {
         MetricsRetriever metricsRetriever = new MetricsRetriever();
 
         ApplicationId applicationId = ApplicationId.from("tenant", "app", "default");
-        Map<String, List<URI>> clusterHosts = Map.of(
-                "cluster1", List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2")),
-                "cluster2", List.of(URI.create("http://localhost:8080/3"))
-        );
-        Map<ApplicationId, Map<String, List<URI>>> applications = Map.of(applicationId, clusterHosts);
+
+        List<ClusterInfo> clusters = List.of(new ClusterInfo("cluster1", ClusterInfo.ClusterType.content, List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"))),
+                new ClusterInfo("cluster2", ClusterInfo.ClusterType.container, List.of(URI.create("http://localhost:8080/3"))));
+
+        Map<ApplicationId, Collection<ClusterInfo>> applications = Map.of(applicationId, clusters);
 
         stubFor(get(urlEqualTo("/1"))
                 .willReturn(aResponse()
                         .withStatus(200)
-                        .withBody(metricsString())));
+                        .withBody(contentMetrics())));
 
         stubFor(get(urlEqualTo("/2"))
                 .willReturn(aResponse()
                         .withStatus(200)
-                        .withBody(metricsString())));
+                        .withBody(contentMetrics())));
 
         stubFor(get(urlEqualTo("/3"))
                 .willReturn(aResponse()
                         .withStatus(200)
-                        .withBody(metricsString())));
+                        .withBody(containerMetrics())));
 
         MetricsResponse metricsResponse = metricsRetriever.retrieveAllMetrics(applications);
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
@@ -64,35 +65,35 @@ public class MetricsRetrieverTest {
                 "  \"applicationId\": \"tenant:app:default\",\n" +
                 "  \"clusters\": [\n" +
                 "   {\n" +
-                "    \"clusterName\": \"cluster1\",\n" +
+                "    \"clusterId\": \"cluster1\",\n" +
+                "    \"clusterType\": \"content\",\n" +
                 "    \"metrics\": {\n" +
-                "     \"queriesPerSecond\": 2.8666666666666667,\n" +
-                "     \"writesPerSecond\": 1.4333333333333333,\n" +
-                "     \"documentCount\": 6000.0,\n" +
-                "     \"queryLatencyMillis\": 93.02325581395348,\n" +
-                "     \"feedLatency\": 69.76744186046511\n" +
-                "    },\n" +
-                "    \"timestamp\": 1557306075\n" +
+                "     \"documentCount\": 6000.0\n" +
+                "    }\n" +
                 "   },\n" +
                 "   {\n" +
-                "    \"clusterName\": \"cluster2\",\n" +
+                "    \"clusterId\": \"cluster2\",\n" +
+                "    \"clusterType\": \"container\",\n" +
                 "    \"metrics\": {\n" +
                 "     \"queriesPerSecond\": 1.4333333333333333,\n" +
-                "     \"writesPerSecond\": 0.7166666666666667,\n" +
-                "     \"documentCount\": 3000.0,\n" +
-                "     \"queryLatencyMillis\": 93.02325581395348,\n" +
+                "     \"feedPerSecond\": 0.7166666666666667,\n" +
+                "     \"queryLatency\": 93.02325581395348,\n" +
                 "     \"feedLatency\": 69.76744186046511\n" +
-                "    },\n" +
-                "    \"timestamp\": 1557306075\n" +
+                "    }\n" +
                 "   }\n" +
                 "  ]\n" +
                 " }\n" +
                 "]\n";
         assertEquals(expectedResponse, bos.toString());
         wireMock.stop();
+
+    }
+
+    private String containerMetrics() throws IOException {
+        return Files.readString(Path.of("src/test/resources/metrics/container_metrics"));
     }
 
-    private String metricsString() throws IOException {
-        return Files.readString(Path.of("src/test/resources/metrics_response"));
+    private String contentMetrics() throws IOException {
+        return Files.readString(Path.of("src/test/resources/metrics/content_metrics"));
     }
 }
\ No newline at end of file
diff --git a/configserver/src/test/resources/metrics_response b/configserver/src/test/resources/metrics/container_metrics
index 19fa2c97799..09232fe0f93 100644
--- a/configserver/src/test/resources/metrics_response
+++ b/configserver/src/test/resources/metrics/container_metrics
@@ -26,28 +26,16 @@
       },
 
       {
-              "name":"qrserver",
-              "timestamp": 1557306075,
-              "metrics": [
-                  {
-                   "values": {
-                        "query_latency.count": 43.0,
-                        "query_latency.sum": 3000
-                    }
-                  }
-                ]
-            },
-        {
-        "name":"distributor",
-        "timestamp": 1557306075,
-        "metrics": [
-            {
-             "values": {
-                  "vds.distributor.docsstored.average": 3000
-
-              }
+      "name":"qrserver",
+      "timestamp": 1557306075,
+      "metrics": [
+          {
+           "values": {
+                "query_latency.count": 43.0,
+                "query_latency.sum": 3000
             }
-          ]
-      }
+          }
+        ]
+    }
   ]
 }
\ No newline at end of file
diff --git a/configserver/src/test/resources/metrics/content_metrics b/configserver/src/test/resources/metrics/content_metrics
new file mode 100644
index 00000000000..a239aeea5ca
--- /dev/null
+++ b/configserver/src/test/resources/metrics/content_metrics
@@ -0,0 +1,16 @@
+{
+  "services": [
+        {
+        "name":"distributor",
+        "timestamp": 1557306075,
+        "metrics": [
+            {
+             "values": {
+                  "vds.distributor.docsstored.average": 3000
+
+              }
+            }
+          ]
+      }
+  ]
+}
\ No newline at end of file