diff options
Diffstat (limited to 'configserver/src/main')
11 files changed, 420 insertions, 72 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 935b455c947..c11d7b12a66 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -19,6 +19,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; @@ -44,9 +45,11 @@ import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.TesterClient; -import com.yahoo.vespa.config.server.http.v2.MetricsResponse; +import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; import com.yahoo.vespa.config.server.http.v2.PrepareResult; -import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever; +import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import com.yahoo.vespa.config.server.metrics.DeploymentMetricsRetriever; +import com.yahoo.vespa.config.server.metrics.ProtonMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -745,13 +748,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<ApplicationId> activeApplications(TenantName tenantName) { return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications(); } + // ---------------- Proton Metrics V1 ------------------------------------------------------------------------ - // ---------------- Metrics ------------------------------------------------------------------------ + public ProtonMetricsResponse getProtonMetrics(ApplicationId applicationId) { + Application application = getApplication(applicationId); + ProtonMetricsRetriever protonMetricsRetriever = new ProtonMetricsRetriever(); + return protonMetricsRetriever.getMetrics(application); + } + + + // ---------------- Deployment Metrics V1 ------------------------------------------------------------------------ - public MetricsResponse getMetrics(ApplicationId applicationId) { + public DeploymentMetricsResponse getDeploymentMetrics(ApplicationId applicationId) { Application application = getApplication(applicationId); - ApplicationMetricsRetriever applicationMetricsRetriever = new ApplicationMetricsRetriever(); - return applicationMetricsRetriever.getMetrics(application); + DeploymentMetricsRetriever deploymentMetricsRetriever = new DeploymentMetricsRetriever(); + return deploymentMetricsRetriever.getMetrics(application); } // ---------------- Misc operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 749f57b3104..8426d6b56cf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -55,7 +55,8 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*", @@ -137,8 +138,12 @@ public class ApplicationHandler extends HttpHandler { return applicationRepository.getLogs(applicationId, hostname, apiParams); } - if (isMetricsRequest(request)) { - return applicationRepository.getMetrics(applicationId); + if (isProtonMetricsRequest(request)) { + return applicationRepository.getProtonMetrics(applicationId); + } + + if (isDeploymentMetricsRequest(request)) { + return applicationRepository.getDeploymentMetrics(applicationId); } if (isIsSuspendedRequest(request)) { @@ -231,9 +236,14 @@ public class ApplicationHandler extends HttpHandler { request.getUri().getPath().endsWith("/suspended"); } - private static boolean isMetricsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/metrics"); + private static boolean isProtonMetricsRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 8 && + request.getUri().getPath().endsWith("/metrics/proton"); + } + + private static boolean isDeploymentMetricsRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 8 && + request.getUri().getPath().endsWith("/metrics/deployment"); } private static boolean isLogRequest(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java index 7cf9357217e..c503b60b3a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java @@ -8,7 +8,7 @@ import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.config.server.metrics.ClusterInfo; -import com.yahoo.vespa.config.server.metrics.MetricsAggregator; +import com.yahoo.vespa.config.server.metrics.DeploymentMetricsAggregator; import java.io.IOException; import java.io.OutputStream; @@ -17,11 +17,11 @@ import java.util.Map; /** * @author olaa */ -public class MetricsResponse extends HttpResponse { +public class DeploymentMetricsResponse extends HttpResponse { private final Slime slime = new Slime(); - public MetricsResponse(int status, ApplicationId applicationId, Map<ClusterInfo, MetricsAggregator> aggregatedMetrics) { + public DeploymentMetricsResponse(int status, ApplicationId applicationId, Map<ClusterInfo, DeploymentMetricsAggregator> aggregatedMetrics) { super(status); Cursor application = slime.setObject(); @@ -34,7 +34,7 @@ public class MetricsResponse extends HttpResponse { cluster.setString("clusterId", entry.getKey().getClusterId()); cluster.setString("clusterType", entry.getKey().getClusterType()); - MetricsAggregator aggregator = entry.getValue(); + DeploymentMetricsAggregator aggregator = entry.getValue(); Cursor metrics = cluster.setObject("metrics"); aggregator.aggregateQueryRate().ifPresent(queryRate -> metrics.setDouble("queriesPerSecond", queryRate)); aggregator.aggregateFeedRate().ifPresent(feedRate -> metrics.setDouble("feedPerSecond", feedRate)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java new file mode 100644 index 00000000000..a0ad87a39c9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java @@ -0,0 +1,51 @@ +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.metrics.ProtonMetricsAggregator; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +public class ProtonMetricsResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + /** + * @author akvalsvik + */ + public ProtonMetricsResponse(int status, ApplicationId applicationId, Map<String, ProtonMetricsAggregator> aggregatedProtonMetrics) { + super(status); + + Cursor application = slime.setObject(); + application.setString("applicationId", applicationId.serializedForm()); + + Cursor clusters = application.setArray("clusters"); + + for (var entry : aggregatedProtonMetrics.entrySet()) { + Cursor cluster = clusters.addObject(); + cluster.setString("clusterId", entry.getKey()); + + ProtonMetricsAggregator aggregator = entry.getValue(); + Cursor metrics = cluster.setObject("metrics"); + metrics.setDouble("documentsActiveCount", aggregator.aggregateDocumentActiveCount()); + metrics.setDouble("documentsReadyCount", aggregator.aggregateDocumentReadyCount()); + metrics.setDouble("documentsTotalCount", aggregator.aggregateDocumentTotalCount()); + metrics.setDouble("documentDiskUsage", aggregator.aggregateDocumentDiskUsage()); + metrics.setDouble("resourceDiskUsageAverage", aggregator.aggregateResourceDiskUsageAverage()); + metrics.setDouble("resourceMemoryUsageAverage", aggregator.aggregateResourceMemoryUsageAverage()); + } + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(false).encode(outputStream, slime); + } + + @Override + public String getContentType() { return HttpConfigResponse.JSON_CONTENT_TYPE; } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 42d3ace6bd9..8e7e4eec9b0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -6,16 +6,12 @@ import java.util.logging.Level; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.util.Collection; import java.util.List; @@ -33,9 +29,9 @@ import java.util.logging.Logger; * @author olaa * @author ogronnesby */ -public class ClusterMetricsRetriever { +public class ClusterDeploymentMetricsRetriever { - private static final Logger log = Logger.getLogger(ClusterMetricsRetriever.class.getName()); + private static final Logger log = Logger.getLogger(ClusterDeploymentMetricsRetriever.class.getName()); private static final String VESPA_CONTAINER = "vespa.container"; private static final String VESPA_QRSERVER = "vespa.qrserver"; @@ -57,8 +53,8 @@ public class ClusterMetricsRetriever { * Call the metrics API on each host and aggregate the metrics * into a single value, grouped by cluster. */ - public Map<ClusterInfo, MetricsAggregator> requestMetricsGroupedByCluster(Collection<URI> hosts) { - Map<ClusterInfo, MetricsAggregator> clusterMetricsMap = new ConcurrentHashMap<>(); + public Map<ClusterInfo, DeploymentMetricsAggregator> requestMetricsGroupedByCluster(Collection<URI> hosts) { + Map<ClusterInfo, DeploymentMetricsAggregator> clusterMetricsMap = new ConcurrentHashMap<>(); long startTime = System.currentTimeMillis(); Runnable retrieveMetricsJob = () -> @@ -83,68 +79,59 @@ public class ClusterMetricsRetriever { return clusterMetricsMap; } - private static void getHostMetrics(URI hostURI, Map<ClusterInfo, MetricsAggregator> clusterMetricsMap) { - Slime responseBody = doMetricsRequest(hostURI); - var parseError = responseBody.get().field("error_message"); - - if (parseError.valid()) { - log.info("Failed to retrieve metrics from " + hostURI + ": " + parseError.asString()); - } - - Inspector services = responseBody.get().field("services"); - services.traverse((ArrayTraverser) (i, servicesInspector) -> - parseService(servicesInspector, clusterMetricsMap) - ); - } - - private static Slime doMetricsRequest(URI hostURI) { - HttpGet get = new HttpGet(hostURI); - try (CloseableHttpResponse response = httpClient.execute(get)) { - InputStream is = response.getEntity().getContent(); - Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes()); - is.close(); - return slime; + private static void getHostMetrics(URI hostURI, Map<ClusterInfo, DeploymentMetricsAggregator> clusterMetricsMap) { + Slime responseBody; + try { + responseBody = MetricsSlime.doMetricsRequest(hostURI, httpClient); } catch (IOException e) { // Usually caused by applications being deleted during metric retrieval log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); - return new Slime(); + responseBody = new Slime(); + } + var parseError = responseBody.get().field("error_message"); + + if (parseError.valid()) { + log.info("Failed to retrieve metrics from " + hostURI + ": " + parseError.asString()); } + + Inspector services = responseBody.get().field("services"); + services.traverse((ArrayTraverser) (i, servicesInspector) -> + parseService(servicesInspector, clusterMetricsMap) + ); } - private static void parseService(Inspector service, Map<ClusterInfo, MetricsAggregator> clusterMetricsMap) { + + + private static void parseService(Inspector service, Map<ClusterInfo, DeploymentMetricsAggregator> clusterMetricsMap) { String serviceName = service.field("name").asString(); service.field("metrics").traverse((ArrayTraverser) (i, metric) -> addMetricsToAggeregator(serviceName, metric, clusterMetricsMap) ); } - private static void addMetricsToAggeregator(String serviceName, Inspector metric, Map<ClusterInfo, MetricsAggregator> clusterMetricsMap) { + private static void addMetricsToAggeregator(String serviceName, Inspector metric, Map<ClusterInfo, DeploymentMetricsAggregator> clusterMetricsMap) { if (!WANTED_METRIC_SERVICES.contains(serviceName)) return; Inspector values = metric.field("values"); - ClusterInfo clusterInfo = getClusterInfoFromDimensions(metric.field("dimensions")); - MetricsAggregator metricsAggregator = clusterMetricsMap.computeIfAbsent(clusterInfo, c -> new MetricsAggregator()); + ClusterInfo clusterInfo = MetricsSlime.getClusterInfoFromDimensions(metric.field("dimensions")); + DeploymentMetricsAggregator deploymentMetricsAggregator = clusterMetricsMap.computeIfAbsent(clusterInfo, c -> new DeploymentMetricsAggregator()); switch (serviceName) { case "vespa.container": - metricsAggregator.addContainerLatency( + deploymentMetricsAggregator.addContainerLatency( values.field("query_latency.sum").asDouble(), values.field("query_latency.count").asDouble()); - metricsAggregator.addFeedLatency( + deploymentMetricsAggregator.addFeedLatency( values.field("feed.latency.sum").asDouble(), values.field("feed.latency.count").asDouble()); break; case "vespa.qrserver": - metricsAggregator.addQrLatency( + deploymentMetricsAggregator.addQrLatency( values.field("query_latency.sum").asDouble(), values.field("query_latency.count").asDouble()); break; case "vespa.distributor": - metricsAggregator.addDocumentCount(values.field("vds.distributor.docsstored.average").asDouble()); + deploymentMetricsAggregator.addDocumentCount(values.field("vds.distributor.docsstored.average").asDouble()); break; } } - - private static ClusterInfo getClusterInfoFromDimensions(Inspector dimensions) { - return new ClusterInfo(dimensions.field("clusterid").asString(), dimensions.field("clustertype").asString()); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java new file mode 100644 index 00000000000..e859bdadd28 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java @@ -0,0 +1,105 @@ +package com.yahoo.vespa.config.server.metrics; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.yolean.Exceptions; +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +import static com.yahoo.vespa.config.server.metrics.MetricsSlime.doMetricsRequest; + +public class ClusterProtonMetricsRetriever { + + private static final Logger log = Logger.getLogger(ClusterProtonMetricsRetriever.class.getName()); + + private static final CloseableHttpClient httpClient = VespaHttpClientBuilder + .create(PoolingHttpClientConnectionManager::new) + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout(10 * 1000) + .setSocketTimeout(10 * 1000) + .build()) + .build(); + + + public Map<String, ProtonMetricsAggregator> requestMetricsGroupedByCluster(Collection<URI> hosts) { + Map<String, ProtonMetricsAggregator> clusterMetricsMap = new ConcurrentHashMap<>(); + for (URI uri : hosts) { + addMetricsFromHost(uri, clusterMetricsMap); + } +/* long startTime = System.currentTimeMillis(); + Runnable retrieveMetricsJob = () -> + hosts.parallelStream().forEach(host -> + addMetricsFromHost(host, clusterMetricsMap) + ); + + ForkJoinPool threadPool = new ForkJoinPool(10); + threadPool.submit(retrieveMetricsJob); + threadPool.shutdown(); + + try { + threadPool.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + log.log(Level.FINE, () -> + String.format("Proton metric retrieval for %d nodes took %d milliseconds", hosts.size(), System.currentTimeMillis() - startTime) + );*/ + + return clusterMetricsMap; + } + + private static void addMetricsFromHost(URI hostURI, Map<String, ProtonMetricsAggregator> clusterMetricsMap) { + Slime hostResponseBody; + try { + hostResponseBody = doMetricsRequest(hostURI, httpClient); + } catch (IOException e) { + log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); + hostResponseBody = new Slime(); + } + var parseError = hostResponseBody.get().field("error_message"); + + if (parseError.valid()) { + log.info("Failed to retrieve metrics from " + hostURI + ": " + parseError.asString()); + } + + Inspector nodes = hostResponseBody.get().field("nodes"); + nodes.traverse((ArrayTraverser) (i, nodesInspector) -> + parseNode(nodesInspector, clusterMetricsMap) + ); + } + + private static void parseNode(Inspector node, Map<String, ProtonMetricsAggregator> clusterMetricsMap) { + String nodeRole = node.field("role").asString(); + if(nodeRole.contains("content")) { + ProtonMetricsAggregator aggregator = new ProtonMetricsAggregator(); + clusterMetricsMap.put(nodeRole, aggregator); + node.field("services").traverse((ArrayTraverser) (i, servicesInspector) -> + addServicesToAggregator(servicesInspector, aggregator) + ); + } + } + + private static void addServicesToAggregator(Inspector services, ProtonMetricsAggregator aggregator) { + services.field("metrics").traverse((ArrayTraverser) (i, metricsInspector) -> + addMetricsToAggregator(metricsInspector, aggregator) + ); + } + + private static void addMetricsToAggregator(Inspector metrics, ProtonMetricsAggregator aggregator) { + aggregator.addAll(metrics.field("values")); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java index 9ecec471b07..a4066fc212d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java @@ -7,29 +7,29 @@ import java.util.Optional; * @author olaa * @author ogronnesby */ -public class MetricsAggregator { +public class DeploymentMetricsAggregator { private LatencyMetrics feed; private LatencyMetrics qr; private LatencyMetrics container; private Double documentCount; - public synchronized MetricsAggregator addFeedLatency(double sum, double count) { + public synchronized DeploymentMetricsAggregator addFeedLatency(double sum, double count) { this.feed = combineLatency(this.feed, sum, count); return this; } - public synchronized MetricsAggregator addQrLatency(double sum, double count) { + public synchronized DeploymentMetricsAggregator addQrLatency(double sum, double count) { this.qr = combineLatency(this.qr, sum, count); return this; } - public synchronized MetricsAggregator addContainerLatency(double sum, double count) { + public synchronized DeploymentMetricsAggregator addContainerLatency(double sum, double count) { this.container = combineLatency(this.container, sum, count); return this; } - public synchronized MetricsAggregator addDocumentCount(double count) { + public synchronized DeploymentMetricsAggregator addDocumentCount(double count) { this.documentCount = (this.documentCount == null ? 0.0 : this.documentCount) + count; return this; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java index c6daaaae2f5..43847cd9c3d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.http.v2.MetricsResponse; +import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; import java.net.URI; import java.util.Collection; @@ -17,29 +17,29 @@ import java.util.stream.Collectors; * * @author olaa */ -public class ApplicationMetricsRetriever { +public class DeploymentMetricsRetriever { - private final ClusterMetricsRetriever metricsRetriever; + private final ClusterDeploymentMetricsRetriever metricsRetriever; - public ApplicationMetricsRetriever() { - this(new ClusterMetricsRetriever()); + public DeploymentMetricsRetriever() { + this(new ClusterDeploymentMetricsRetriever()); } - public ApplicationMetricsRetriever(ClusterMetricsRetriever metricsRetriever) { + public DeploymentMetricsRetriever(ClusterDeploymentMetricsRetriever metricsRetriever) { this.metricsRetriever = metricsRetriever; } - public MetricsResponse getMetrics(Application application) { + public DeploymentMetricsResponse getMetrics(Application application) { var hosts = getHostsOfApplication(application); var clusterMetrics = metricsRetriever.requestMetricsGroupedByCluster(hosts); - return new MetricsResponse(200, application.getId(), clusterMetrics); + return new DeploymentMetricsResponse(200, application.getId(), clusterMetrics); } private static Collection<URI> getHostsOfApplication(Application application) { return application.getModel().getHosts().stream() .filter(host -> host.getServices().stream().noneMatch(isLogserver())) .map(HostInfo::getHostname) - .map(ApplicationMetricsRetriever::createMetricsProxyURI) + .map(DeploymentMetricsRetriever::createMetricsProxyURI) .collect(Collectors.toList()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsSlime.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsSlime.java new file mode 100644 index 00000000000..617767bc6a7 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsSlime.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.config.server.metrics; + +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.yolean.Exceptions; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; + +public class MetricsSlime { + + static Slime doMetricsRequest(URI hostURI, CloseableHttpClient httpClient) throws IOException { + HttpGet get = new HttpGet(hostURI); + CloseableHttpResponse response = httpClient.execute(get); + InputStream is = response.getEntity().getContent(); + Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes()); + is.close(); + return slime; + } + + static ClusterInfo getClusterInfoFromDimensions(Inspector dimensions) { + return new ClusterInfo(dimensions.field("clusterid").asString(), dimensions.field("clustertype").asString()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java new file mode 100644 index 00000000000..e1a0c2dc253 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.config.server.metrics; + +import com.yahoo.slime.Inspector; + +public class ProtonMetricsAggregator { + + private static final String DOCUMENT_ACTIVE = "content.proton.documentdb.documents.active.last"; + private static final String DOCUMENT_READY = "content.proton.documentdb.documents.ready.last"; + private static final String DOCUMENT_TOTAL = "content.proton.documentdb.documents.total.last"; + private static final String DOCUMENT_DISK_USAGE = "content.proton.documentdb.disk_usage.last"; + private static final String RESOURCE_DISK_AVERAGE = "content.proton.resource_usage.disk.average"; + private static final String RESOURCE_MEMORY_AVERAGE = "content.proton.resource_usage.memory.average"; + + private Double documentActiveCount = 0.0; + private Double documentReadyCount = 0.0; + private Double documentTotalCount = 0.0; + private Double documentDiskUsage = 0.0; + + private final AverageMetric resourceDiskUsageAverage = new AverageMetric(); + private final AverageMetric resourceMemoryUsageAverage = new AverageMetric(); + + public synchronized ProtonMetricsAggregator addAll(Inspector metric) { + if (metric.field(DOCUMENT_ACTIVE).valid()) addDocumentActiveCount(metric.field(DOCUMENT_ACTIVE).asDouble()); + if (metric.field(DOCUMENT_READY).valid()) addDocumentReadyCount(metric.field(DOCUMENT_READY).asDouble()); + if (metric.field(DOCUMENT_TOTAL).valid()) addDocumentTotalCount(metric.field(DOCUMENT_TOTAL).asDouble()); + if (metric.field(DOCUMENT_DISK_USAGE).valid()) addDocumentDiskUsage(metric.field(DOCUMENT_DISK_USAGE).asDouble()); + if (metric.field(RESOURCE_DISK_AVERAGE).valid()) addResourceDiskUsageAverage(metric.field(RESOURCE_DISK_AVERAGE).asDouble()); + if (metric.field(RESOURCE_MEMORY_AVERAGE).valid()) addResourceMemoryUsageAverage(metric.field(RESOURCE_MEMORY_AVERAGE).asDouble()); + return this; + } + + public ProtonMetricsAggregator addAll(ProtonMetricsAggregator aggregator) { + this.documentActiveCount += aggregator.aggregateDocumentActiveCount(); + this.documentReadyCount += aggregator.aggregateDocumentReadyCount(); + this.documentTotalCount += aggregator.aggregateDocumentTotalCount(); + this.documentDiskUsage += aggregator.aggregateDocumentDiskUsage(); + addResourceDiskUsageAverage(aggregator); + addResourceMemoryUsageAverage(aggregator); + return this; + } + + public ProtonMetricsAggregator addResourceDiskUsageAverage(ProtonMetricsAggregator aggregator) { + this.resourceDiskUsageAverage.averageCount += aggregator.resourceDiskUsageAverage.averageCount; + this.resourceDiskUsageAverage.averageSum += aggregator.resourceDiskUsageAverage.averageSum; + return this; + } + + public ProtonMetricsAggregator addResourceMemoryUsageAverage(ProtonMetricsAggregator aggregator) { + this.resourceMemoryUsageAverage.averageCount += aggregator.resourceMemoryUsageAverage.averageCount; + this.resourceMemoryUsageAverage.averageSum += aggregator.resourceMemoryUsageAverage.averageSum; + return this; + } + + public synchronized ProtonMetricsAggregator addDocumentActiveCount(double documentActiveCount) { + this.documentActiveCount += documentActiveCount; + return this; + } + + public synchronized ProtonMetricsAggregator addDocumentReadyCount(double documentReadyCount) { + this.documentReadyCount += documentReadyCount; + return this; + } + + public synchronized ProtonMetricsAggregator addDocumentTotalCount(double documentTotalCount) { + this.documentTotalCount += documentTotalCount; + return this; + } + + public synchronized ProtonMetricsAggregator addDocumentDiskUsage(double documentDiskUsage) { + this.documentDiskUsage += documentDiskUsage; + return this; + } + + public synchronized ProtonMetricsAggregator addResourceDiskUsageAverage(double resourceDiskUsageAverage) { + this.resourceDiskUsageAverage.averageCount++; + this.resourceDiskUsageAverage.averageSum += resourceDiskUsageAverage; + return this; + } + + public synchronized ProtonMetricsAggregator addResourceMemoryUsageAverage(double resourceMemoryUsageAverage) { + this.resourceMemoryUsageAverage.averageCount++; + this.resourceMemoryUsageAverage.averageSum += resourceMemoryUsageAverage; + return this; + } + + public Double aggregateDocumentActiveCount() { + return this.documentActiveCount; + } + + public Double aggregateDocumentReadyCount() { + return this.documentReadyCount; + } + + public Double aggregateDocumentTotalCount() { + return this.documentTotalCount; + } + + public Double aggregateDocumentDiskUsage() {return this.documentDiskUsage;} + + public Double aggregateResourceDiskUsageAverage() { + return this.resourceDiskUsageAverage.averageSum / this.resourceDiskUsageAverage.averageCount; + } + + public Double aggregateResourceMemoryUsageAverage() { + return this.resourceMemoryUsageAverage.averageSum / this.resourceMemoryUsageAverage.averageCount; + } + + private static class AverageMetric { + double averageSum = 0.0; + double averageCount = 0.0; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java new file mode 100644 index 00000000000..91295ca8bee --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java @@ -0,0 +1,43 @@ +package com.yahoo.vespa.config.server.metrics; + +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import java.net.URI; +import java.util.Collection; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class ProtonMetricsRetriever { + + private final ClusterProtonMetricsRetriever metricsRetriever; + public ProtonMetricsRetriever() { + this( new ClusterProtonMetricsRetriever()); + } + + public ProtonMetricsRetriever(ClusterProtonMetricsRetriever metricsRetriever) { + this.metricsRetriever = metricsRetriever; + } + + public ProtonMetricsResponse getMetrics(Application application) { + var hosts = getHostsOfApplication(application); + var clusterMetrics = metricsRetriever.requestMetricsGroupedByCluster(hosts); + return new ProtonMetricsResponse(200, application.getId(), clusterMetrics); + } + + private static Collection<URI> getHostsOfApplication(Application application) { + return application.getModel().getHosts().stream() + .filter(host -> host.getServices().stream().anyMatch(isSearchNode())) + .map(HostInfo::getHostname) + .map(ProtonMetricsRetriever::createMetricsProxyURI) + .collect(Collectors.toList()); + } + + private static Predicate<ServiceInfo> isSearchNode() { + return serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("searchnode"); + } + private static URI createMetricsProxyURI(String hostname) { + return URI.create("http://" + hostname + ":19092/metrics/v2/values"); + } +} |