Parallelize cluster metric aggeregation.

4 files changed, 50 insertions, 39 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index c8b99e2c3a0..de5e77fe849 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -60,6 +60,7 @@ import com.yahoo.vespa.config.server.tenant.Rotations;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.io.File;
@@ -75,11 +76,13 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -87,7 +90,6 @@ import java.util.stream.Collectors;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
-import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
 import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT;
 import static java.nio.file.Files.readAttributes;
 
@@ -649,14 +651,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     // ---------------- Metrics ------------------------------------------------------------------------
 
     public MetricsResponse getMetrics(ApplicationId applicationId) {
-        var metricsRetriever = new MetricsRetriever();
         var clusters = getClustersOfApplication(applicationId);
-        var clusterMetrics = new LinkedHashMap<ClusterInfo, MetricsAggregator>();
-
-        clusters.forEach(cluster -> {
-            var metrics = metricsRetriever.requestMetricsForCluster(cluster);
+        var clusterMetrics = new ConcurrentHashMap<ClusterInfo, MetricsAggregator>();
+        ForkJoinPool pool = new ForkJoinPool(5);
+        pool.submit(() ->
+        clusters.parallelStream().forEach(cluster -> {
+            var metrics = MetricsRetriever.requestMetricsForCluster(cluster);
             clusterMetrics.put(cluster, metrics);
-        });
+        }));
+        pool.shutdown();
+
+        try {
+            pool.awaitTermination(1, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
 
         return new MetricsResponse(200, applicationId, clusterMetrics);
     }
@@ -799,10 +808,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         application.getModel().getHosts().stream()
                 .filter(host -> host.getServices().stream().noneMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver")))
                 .forEach(hostInfo -> {
-                            ServiceInfo metricsService = getServiceInfoByType(hostInfo, METRICS_PROXY_CONTAINER.serviceName);
-                            ServiceInfo clusterServiceInfo =  getServiceInfoByType(hostInfo, "container", "searchnode");
-                            ClusterInfo clusterInfo = createClusterInfo(clusterServiceInfo);
-                            URI host = URI.create("http://" + hostInfo.getHostname() + ":" + servicePort(metricsService) + "/metrics/v1/values?consumer=Vespa");
+                            ClusterInfo clusterInfo = createClusterInfo(hostInfo);
+                            URI host = URI.create("http://" + hostInfo.getHostname() + ":" + MetricsProxyContainer.BASEPORT + "/metrics/v1/values?consumer=Vespa");
                             clusters.computeIfAbsent(clusterInfo.getClusterId(), c -> clusterInfo).addHost(host);
                         }
                 );
@@ -810,15 +817,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
     }
 
-    private ServiceInfo getServiceInfoByType(HostInfo hostInfo, String... types) {
-        List<String> type = List.of(types);
-        return hostInfo.getServices().stream().filter(serviceInfo -> type.contains(serviceInfo.getServiceType())).findFirst().orElseThrow();
-    }
-
-    private ClusterInfo createClusterInfo(ServiceInfo serviceInfo) {
-        String clusterName = serviceInfo.getServiceName();
-        ClusterInfo.ClusterType clusterType = serviceInfo.getServiceType().equals("searchnode") ? ClusterInfo.ClusterType.content : ClusterInfo.ClusterType.container;
-        return new ClusterInfo(clusterName, clusterType);
+    private ClusterInfo createClusterInfo(HostInfo hostInfo) {
+        return hostInfo.getServices().stream()
+                .map(ClusterInfo::fromServiceInfo)
+                .filter(Optional::isPresent)
+                .findFirst().get().orElseThrow();
     }
 
     /** Returns version to use when deploying application in given environment */
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java
index 7507b5c4c2c..bd453dddd74 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterInfo.java
@@ -1,15 +1,22 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.metrics;
 
+import com.yahoo.config.model.api.ServiceInfo;
+
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 
 /**
  * @author olaa
  */
 public class ClusterInfo {
 
+    private static final Set<String> CONTENT_SERVICES = Set.of("storagenode", "searchnode", "distributor", "container-clustercontroller");
+    private static final Set<String> CONTAINER_SERVICES = Set.of("qrserver", "container");
+
     private final String clusterId;
     private final ClusterType clusterType;
     private final List<URI> hostnames;
@@ -40,17 +47,18 @@ public class ClusterInfo {
         hostnames.add(host);
     }
 
+    // Try to determine whether host is content or container based on service
+    public static Optional<ClusterInfo> fromServiceInfo(ServiceInfo serviceInfo) {
+        String serviceType = serviceInfo.getServiceType();
+        ClusterType clusterType;
+        if (CONTENT_SERVICES.contains(serviceType)) clusterType = ClusterType.content;
+        else if (CONTAINER_SERVICES.contains(serviceType)) clusterType = ClusterType.container;
+        else return Optional.empty();
+        return Optional.of(new ClusterInfo(serviceInfo.getServiceName(), clusterType));
+    }
+
     public enum ClusterType {
         content,
-        container;
-
-        public static boolean isValidType(String enumString) {
-            try {
-                valueOf(enumString);
-                return true;
-            } catch (IllegalArgumentException e) {
-                return false;
-            }
-        }
-    };
+        container
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
index 4fbeae11758..8a18843db5f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsRetriever.java
@@ -27,19 +27,18 @@ import java.util.logging.Logger;
  */
 public class MetricsRetriever {
     private static final Logger log = Logger.getLogger(MetricsRetriever.class.getName());
-    private final HttpClient httpClient = HttpClientBuilder.create().build();
 
     /**
      * Call the metrics API on each host in the cluster and aggregate the metrics
      * into a single value.
      */
-    public MetricsAggregator requestMetricsForCluster(ClusterInfo clusterInfo) {
+    public static MetricsAggregator requestMetricsForCluster(ClusterInfo clusterInfo) {
         var aggregator = new MetricsAggregator();
         clusterInfo.getHostnames().forEach(host -> getHostMetrics(host, aggregator));
         return aggregator;
     }
 
-    private void getHostMetrics(URI hostURI, MetricsAggregator metrics) {
+    private static void getHostMetrics(URI hostURI, MetricsAggregator metrics) {
             Slime responseBody = doMetricsRequest(hostURI);
             var parseError = responseBody.get().field("error_message");
 
@@ -53,9 +52,10 @@ public class MetricsRetriever {
             });
     }
 
-    private Slime doMetricsRequest(URI hostURI) {
+    private static Slime doMetricsRequest(URI hostURI) {
         HttpGet get = new HttpGet(hostURI);
         try {
+            HttpClient httpClient = HttpClientBuilder.create().build();
             HttpResponse response = httpClient.execute(get);
             InputStream is = response.getEntity().getContent();
             Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes());
@@ -66,7 +66,7 @@ public class MetricsRetriever {
         }
     }
 
-    private void parseService(Inspector service, MetricsAggregator metrics) {
+    private static void parseService(Inspector service, MetricsAggregator metrics) {
         String serviceName = service.field("name").asString();
         Instant timestamp = Instant.ofEpochSecond(service.field("timestamp").asLong());
         metrics.setTimestamp(timestamp);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
index 1b878a432c9..67e91b90803 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsRetrieverTest.java
@@ -55,7 +55,7 @@ public class MetricsRetrieverTest {
 
         compareAggregators(
                 new MetricsAggregator().addDocumentCount(6000.0),
-                metricsRetriever.requestMetricsForCluster(clusters.get(0))
+                MetricsRetriever.requestMetricsForCluster(clusters.get(0))
         );
 
         compareAggregators(
@@ -64,7 +64,7 @@ public class MetricsRetrieverTest {
                         .addContainerLatency(2000, 0)
                         .addQrLatency(3000, 43)
                         .addFeedLatency(3000, 43),
-                metricsRetriever.requestMetricsForCluster(clusters.get(1))
+                MetricsRetriever.requestMetricsForCluster(clusters.get(1))
         );
 
         wireMock.stop();