From be4a878ffb4005c7796fc52fa8b87157e897b4d1 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 9 May 2019 17:52:05 +0200 Subject: Cfg server retrieves metrics from all applications in zone --- .../vespa/config/server/ApplicationRepository.java | 52 ++++++++++++- .../config/server/http/v2/ApplicationHandler.java | 10 +++ .../config/server/http/v2/MetricsRespone.java | 53 +++++++++++++ .../yahoo/vespa/config/server/metrics/Metrics.java | 63 +++++++++++++++ .../config/server/metrics/MetricsAggregator.java | 82 ++++++++++++++++++++ .../main/resources/configserver-app/services.xml | 1 + .../server/http/v2/ApplicationHandlerTest.java | 2 + .../server/metrics/MetricsAggregatorTest.java | 89 ++++++++++++++++++++++ configserver/src/test/resources/metrics_response | 23 ++++++ 9 files changed, 371 insertions(+), 4 deletions(-) create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java create mode 100644 configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java create mode 100644 configserver/src/test/resources/metrics_response (limited to 'configserver') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b4414562170..b2c2b85d358 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -43,6 +43,8 @@ import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.v2.PrepareResult; +import com.yahoo.vespa.config.server.metrics.Metrics; +import com.yahoo.vespa.config.server.metrics.MetricsAggregator; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.LocalSessionRepo; @@ -67,10 +69,13 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -80,6 +85,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static java.nio.file.Files.readAttributes; /** @@ -579,6 +585,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications(); } + // ---------------- Metrics ------------------------------------------------------------------------ + + public HttpResponse getMetrics() { + MetricsAggregator metricsAggregator = new MetricsAggregator(); + Map>> applicationHosts = getHostsPerApplication(); + return metricsAggregator.aggregateAllMetrics(applicationHosts); + } + // ---------------- Misc operations ---------------------------------------------------------------- public Tenant verifyTenantAndApplication(ApplicationId applicationId) { @@ -717,18 +731,48 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HostInfo for LogServer")); - ServiceInfo containerServiceInfo = logServerHostInfo.getServices().stream() - .filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType())) + ServiceInfo serviceInfo = logServerHostInfo.getServices().stream().filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType())) .findFirst().orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); + int port = servicePort(serviceInfo); + return "http://" + logServerHostInfo.getHostname() + ":" + port + "/logs"; + } - int port = containerServiceInfo.getPorts().stream() + private int servicePort(ServiceInfo serviceInfo) { + int port = serviceInfo.getPorts().stream() .filter(portInfo -> portInfo.getTags().stream().anyMatch(tag -> tag.equalsIgnoreCase("http"))) .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HTTP port")) .getPort(); + return port; + } - return "http://" + logServerHostInfo.getHostname() + ":" + port + "/logs"; + /** Finds all hosts, grouping them by application ID and cluster name */ + private Map>> getHostsPerApplication() { + Map>> applicationHosts = new HashMap<>(); + tenantRepository.getAllTenants().stream() + .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream()) + .forEach(applicationId ->{ + applicationHosts.put(applicationId, getClustersOfApplication(applicationId)); + } + ); + return applicationHosts; } + /** Finds the hosts of an application, grouped by cluster name */ + private Map> getClustersOfApplication(ApplicationId applicationId) { + Application application = getApplication(applicationId); + Map> clusterHosts = new HashMap<>(); + application.getModel().getHosts().stream() + .forEach(hostInfo -> { + ServiceInfo serviceInfo = hostInfo.getServices().stream().filter(service -> METRICS_PROXY_CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Unable to find services " + METRICS_PROXY_CONTAINER.serviceName.toString())); + String clusterName = serviceInfo.getProperty("clusterinfo").orElse(""); + URI host = URI.create("http://" + hostInfo.getHostname() + ":" + servicePort(serviceInfo) + "/metrics"); + clusterHosts.computeIfAbsent(clusterName, l -> new ArrayList()).add(host); + } + ); + return clusterHosts; + + } /** Returns version to use when deploying application in given environment */ static Version decideVersion(ApplicationId application, Environment environment, Version sessionVersion, boolean bootstrap) { if ( environment.isManuallyDeployed() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index e04f83cc648..37f78dafa49 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -102,6 +102,10 @@ public class ApplicationHandler extends HttpHandler { return applicationRepository.getLogs(applicationId, hostname, apiParams); } + if (isMetricsRequest(request) && false) { + return applicationRepository.getMetrics(); + } + if (isIsSuspendedRequest(request)) { return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); } @@ -152,6 +156,7 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", "http://*/application/v2/tenant/*/application/*/logs", + "http://*/application/v2/metrics", "http://*/application/v2/tenant/*/application/*"); } @@ -160,6 +165,11 @@ public class ApplicationHandler extends HttpHandler { request.getUri().getPath().endsWith("/suspended"); } + private static boolean isMetricsRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 1 && + request.getUri().getPath().endsWith("/metrics"); + } + private static boolean isLogRequest(HttpRequest request) { return getBindingMatch(request).groupCount() == 4 && request.getUri().getPath().endsWith("/logs"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java new file mode 100644 index 00000000000..4d6a2f20cd1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.metrics.Metrics; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +/** + * @author olaa + */ +public class MetricsRespone extends HttpResponse { + + private final Slime slime = new Slime(); + + public MetricsRespone(int status, Map> aggregatedMetrics) { + super(status); + + Cursor array = slime.setArray(); + for (Map.Entry> entry : aggregatedMetrics.entrySet()) { + Cursor object = array.addObject(); + object.setString("applicationId", entry.getKey().serializedForm()); + Cursor clusterArray = object.setArray("clusters"); + for (Map.Entry clusterMetrics : entry.getValue().entrySet()) { + Cursor clusterCursor = clusterArray.addObject(); + Metrics metrics = clusterMetrics.getValue(); + clusterCursor.setString("clusterName", clusterMetrics.getKey()); + clusterCursor.setDouble("queriesPerSecond", metrics.getQueriesPerSecond()); + clusterCursor.setDouble("writesPerSecond", metrics.getWritesPerSecond()); + clusterCursor.setDouble("documentCount", metrics.getDocumentCount()); + clusterCursor.setDouble("queryLatencyMillis", metrics.getQueryLatencyMillis()); + clusterCursor.setDouble("writeLatencyMillis", metrics.getWriteLatencyMills()); + } + } + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(false).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java new file mode 100644 index 00000000000..7216523719a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java @@ -0,0 +1,63 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.metrics; + +import java.util.List; + +/** + * @author olaa + */ +public class Metrics { + + private double queriesPerSecond; + private double writesPerSecond; + private double documentCount; + private double queryLatencyMillis; + private double writeLatencyMills; + + public Metrics(double queriesPerSecond, double writesPerSecond, double documentCount, + double queryLatencyMillis, double writeLatencyMills) { + this.queriesPerSecond = queriesPerSecond; + this.writesPerSecond = writesPerSecond; + this.documentCount = documentCount; + this.queryLatencyMillis = queryLatencyMillis; + this.writeLatencyMills = writeLatencyMills; + } + + + public double getQueriesPerSecond() { + return queriesPerSecond; + } + + public double getWritesPerSecond() { + return writesPerSecond; + } + + public double getDocumentCount() { + return documentCount; + } + + public double getQueryLatencyMillis() { + return queryLatencyMillis; + } + + public double getWriteLatencyMills() { + return writeLatencyMills; + } + + public void accumulate(Metrics metrics) { + this.queriesPerSecond += metrics.getQueriesPerSecond(); + this.writesPerSecond += metrics.getWritesPerSecond(); + this.queryLatencyMillis += metrics.getQueryLatencyMillis(); + this.writeLatencyMills += metrics.getWriteLatencyMills(); + } + + public static Metrics averagedMetrics(List metrics) { + return new Metrics( + metrics.stream().mapToDouble(Metrics::getQueriesPerSecond).sum() / metrics.size(), + metrics.stream().mapToDouble(Metrics::getWritesPerSecond).sum() / metrics.size(), + metrics.stream().mapToDouble(Metrics::getDocumentCount).sum() / metrics.size(), + metrics.stream().mapToDouble(Metrics::getQueryLatencyMillis).sum() / metrics.size(), + metrics.stream().mapToDouble(Metrics::getWriteLatencyMills).sum() / metrics.size() + ); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java new file mode 100644 index 00000000000..bf1618b1696 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java @@ -0,0 +1,82 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.metrics; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.config.server.http.v2.MetricsRespone; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + + +/** + * @author olaa + */ +public class MetricsAggregator { + + private static final Logger logger = Logger.getLogger(MetricsAggregator.class.getName()); + HttpClient httpClient = HttpClientBuilder.create().build(); + + public MetricsRespone aggregateAllMetrics(Map>> applicationHosts) { + Map> aggregatedMetrics = applicationHosts.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e-> aggregateMetricsByCluster(e.getValue()))); + return new MetricsRespone(200, aggregatedMetrics); + } + + private Map aggregateMetricsByCluster(Map> clusterHosts) { + logger.log(Level.WARNING, clusterHosts.keySet() +" "); + return clusterHosts.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> aggregateMetrics(e.getValue()) + ) + ); + } + + private Metrics aggregateMetrics(List hosts) { + List metrics= hosts.stream() + .map(host -> getMetrics(host)) + .collect(Collectors.toList()); + Metrics.averagedMetrics(metrics); + logger.log(Level.WARNING, metrics + ""); + return Metrics.averagedMetrics(metrics); + } + + private Metrics getMetrics(URI hostURI) { + HttpGet get = new HttpGet(hostURI); + try { + HttpResponse response = httpClient.execute(get); + + InputStream is = response.getEntity().getContent(); + Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes()); + is.close(); + + Inspector metrics = slime.get().field("metrics"); + Instant timestamp = Instant.ofEpochSecond(slime.get().field("timestamp").asLong()); + double queriesPerSecond = metrics.field("queriesPerSecond").asDouble(); + double writesPerSecond = metrics.field("writesPerSecond").asDouble(); + double documentCount = metrics.field("documentCount").asDouble(); + double queryLatencyMillis = metrics.field("queryLatencyMillis").asDouble(); + double writeLatencyMills = metrics.field("writeLatencyMills").asDouble(); + return new Metrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, writeLatencyMills); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 23864e7d3e8..8ba505f213d 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -129,6 +129,7 @@ http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/* http://*/application/v2/tenant/*/application/* http://*/application/v2/tenant/*/application/*/logs + http://*/application/v2/metrics http://*/config/v2/tenant/*/application/*/* diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index a843212927b..f1891430ccd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -35,6 +35,8 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.time.Clock; +import java.util.logging.Level; +import java.util.logging.Logger; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java new file mode 100644 index 00000000000..52c7cbec43e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java @@ -0,0 +1,89 @@ +package com.yahoo.vespa.config.server.metrics; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.http.v2.MetricsRespone; +import com.yahoo.vespa.config.server.tenant.Tenant; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when;import com.yahoo.config.model.api.Model; + + +/** + * @author olaa + */ +public class MetricsAggregatorTest { + + @Rule + public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); + + @Test + public void testMetricAggregation() throws IOException { + MetricsAggregator metricsAggregator = new MetricsAggregator(); + + ApplicationId applicationId = ApplicationId.from("tenant", "app", "default"); + Map> clusterHosts = Map.of( + "cluster1", List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/3")), + "cluster2", List.of(URI.create("http://localhost:8080/3"), URI.create("http://localhost:8080/3")) + ); + Map>> applications = Map.of(applicationId, clusterHosts); + + stubFor(get(urlEqualTo("/1")) + .willReturn(aResponse() + .withStatus(200) + .withBody(metricsString(10,20,33,40,50)))); + + stubFor(get(urlEqualTo("/2")) + .willReturn(aResponse() + .withStatus(200) + .withBody(metricsString(1,2,3,4,5)))); + + stubFor(get(urlEqualTo("/3")) + .willReturn(aResponse() + .withStatus(200) + .withBody(metricsString(1,2,3,4,5)))); + + MetricsRespone metricsRespone = metricsAggregator.aggregateAllMetrics(applications); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + metricsRespone.render(bos); + assertEquals(Files.readString(Path.of("src/test/resources/metrics_response")), bos.toString()); + wireMock.stop(); + } + + private String metricsString(double queriesPerSecond, double writesPerSecond, double documentCount, double queryLatencyMillis, double writeLatencyMills) { + return "{\"metrics\": " + + "{" + + " \"queriesPerSecond\": " + queriesPerSecond + "," + + " \"writesPerSecond\": " + writesPerSecond + "," + + " \"documentCount\": " + documentCount + "," + + " \"queryLatencyMillis\": " + queryLatencyMillis + "," + + " \"writeLatencyMills\": " + writeLatencyMills + + " }}"; + } +} \ No newline at end of file diff --git a/configserver/src/test/resources/metrics_response b/configserver/src/test/resources/metrics_response new file mode 100644 index 00000000000..b400a0538a6 --- /dev/null +++ b/configserver/src/test/resources/metrics_response @@ -0,0 +1,23 @@ +[ + { + "applicationId": "tenant:app:default", + "clusters": [ + { + "clusterName": "cluster1", + "queriesPerSecond": 5.5, + "writesPerSecond": 11.0, + "documentCount": 18.0, + "queryLatencyMillis": 22.0, + "writeLatencyMillis": 27.5 + }, + { + "clusterName": "cluster2", + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0 + } + ] + } +] -- cgit v1.2.3