From 7fdd0ed5165d5b6dc838960ab3872540bb1b7554 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 20 Mar 2023 15:24:19 +0100 Subject: Remove duplicated method --- .../java/com/yahoo/vespa/config/server/ApplicationRepository.java | 2 +- .../yahoo/vespa/config/server/application/TenantApplications.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 969b6a8201f..9fb9f8ba9f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -992,7 +992,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ApplicationId getApplicationIdForHostname(String hostname) { Optional applicationId = tenantRepository.getAllTenantNames().stream() - .map(tenantName -> tenantRepository.getTenant(tenantName).getApplicationRepo().getApplicationIdForHostName(hostname)) + .map(tenantName -> tenantRepository.getTenant(tenantName).getApplicationRepo().resolveApplicationId(hostname)) .filter(Objects::nonNull) .findFirst(); return applicationId.orElse(null); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 09a687657c6..88e3134ccad 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -401,11 +401,6 @@ public class TenantApplications implements RequestHandler, HostValidator { hostRegistry.verifyHosts(applicationId, newHosts); } - // TODO: Duplicate of resolveApplicationId() above - public ApplicationId getApplicationIdForHostName(String hostname) { - return hostRegistry.getApplicationId(hostname); - } - public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) { -- cgit v1.2.3 From da63d5b730e283c31f535d850c0ca02a2193e259 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 20 Mar 2023 15:43:13 +0100 Subject: More proton metrics -> searchnode metrics --- .../vespa/config/server/ApplicationRepository.java | 2 +- .../config/server/http/v2/ApplicationHandler.java | 8 +- .../v2/response/SearchNodeMetricsResponse.java | 4 +- .../metrics/ClusterProtonMetricsRetriever.java | 111 --------------------- .../metrics/ClusterSearchNodeMetricsRetriever.java | 111 +++++++++++++++++++++ .../server/metrics/SearchNodeMetricsRetriever.java | 6 +- .../ClusterSearchNodeMetricsRetrieverTest.java | 2 +- .../metrics/SearchNodeMetricsRetrieverTest.java | 4 +- 8 files changed, 125 insertions(+), 123 deletions(-) delete mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetriever.java diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9fb9f8ba9f6..f6988a6b566 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -945,7 +945,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- SearchNode Metrics ------------------------------------------------------------------------ - public SearchNodeMetricsResponse getProtonMetrics(ApplicationId applicationId) { + public SearchNodeMetricsResponse getSearchNodeMetrics(ApplicationId applicationId) { Application application = getApplication(applicationId); SearchNodeMetricsRetriever searchNodeMetricsRetriever = new SearchNodeMetricsRetriever(); return searchNodeMetricsRetriever.getMetrics(application); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 9619ad69b3c..86c0c90ca12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,7 +97,9 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path)); + // TODO: Remove when all usage has migrated to .../metrics/searchnode + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return searchNodeMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/{*}")) return serviceStateV1(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); @@ -200,8 +202,8 @@ public class ApplicationHandler extends HttpHandler { return applicationRepository.getLogs(applicationId, hostname, apiParams); } - private HttpResponse protonMetrics(ApplicationId applicationId) { - return applicationRepository.getProtonMetrics(applicationId); + private HttpResponse searchNodeMetrics(ApplicationId applicationId) { + return applicationRepository.getSearchNodeMetrics(applicationId); } private HttpResponse deploymentMetrics(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SearchNodeMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SearchNodeMetricsResponse.java index c9d748a7e4c..311dbccb752 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SearchNodeMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SearchNodeMetricsResponse.java @@ -12,13 +12,13 @@ import java.util.Map; */ public class SearchNodeMetricsResponse extends SlimeJsonResponse { - public SearchNodeMetricsResponse(ApplicationId applicationId, Map aggregatedProtonMetrics) { + public SearchNodeMetricsResponse(ApplicationId applicationId, Map aggregatedMetrics) { Cursor application = slime.setObject(); application.setString("applicationId", applicationId.serializedForm()); Cursor clusters = application.setArray("clusters"); - for (var entry : aggregatedProtonMetrics.entrySet()) { + for (var entry : aggregatedMetrics.entrySet()) { Cursor cluster = clusters.addObject(); cluster.setString("clusterId", entry.getKey()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java deleted file mode 100644 index bbdae8f2318..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.metrics; - -import ai.vespa.util.http.hc5.VespaHttpClientBuilder; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.yolean.Exceptions; -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.core5.http.io.entity.EntityUtils; -import org.apache.hc.core5.util.Timeout; - -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; - -public class ClusterProtonMetricsRetriever { - - private static final Logger log = Logger.getLogger(ClusterProtonMetricsRetriever.class.getName()); - - private static final CloseableHttpClient httpClient = VespaHttpClientBuilder - .custom() - .connectTimeout(Timeout.ofSeconds(10)) - .apacheBuilder() - .setDefaultRequestConfig(RequestConfig.custom().setResponseTimeout(Timeout.ofSeconds(10)).build()) - .build(); - - - public Map requestMetricsGroupedByCluster(Collection hosts) { - Map clusterMetricsMap = new ConcurrentHashMap<>(); - for (URI uri : hosts) { - addMetricsFromHost(uri, clusterMetricsMap); - } -/* long startTime = System.currentTimeMillis(); - Runnable retrieveMetricsJob = () -> - hosts.parallelStream().forEach(host -> - addMetricsFromHost(host, clusterMetricsMap) - ); - - ForkJoinPool threadPool = new ForkJoinPool(10); - threadPool.submit(retrieveMetricsJob); - threadPool.shutdown(); - - try { - threadPool.awaitTermination(1, TimeUnit.MINUTES); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - log.log(Level.FINE, () -> - String.format("Proton metric retrieval for %d nodes took %d milliseconds", hosts.size(), System.currentTimeMillis() - startTime) - );*/ - - return clusterMetricsMap; - } - - private static void addMetricsFromHost(URI hostURI, Map clusterMetricsMap) { - Slime hostResponseBody = doMetricsRequest(hostURI); - Cursor error = hostResponseBody.get().field("error_message"); - - if (error.valid()) { - log.info("Failed to retrieve metrics from " + hostURI + ": " + error.asString()); - } - - Inspector nodes = hostResponseBody.get().field("nodes"); - nodes.traverse((ArrayTraverser) (i, nodesInspector) -> - parseNode(nodesInspector, clusterMetricsMap) - ); - } - - private static void parseNode(Inspector node, Map clusterMetricsMap) { - String nodeRole = node.field("role").asString(); - if(nodeRole.contains("content")) { - SearchNodeMetricsAggregator aggregator = new SearchNodeMetricsAggregator(); - clusterMetricsMap.put(nodeRole, aggregator); - node.field("services").traverse((ArrayTraverser) (i, servicesInspector) -> - addServicesToAggregator(servicesInspector, aggregator) - ); - } - } - - private static void addServicesToAggregator(Inspector services, SearchNodeMetricsAggregator aggregator) { - services.field("metrics").traverse((ArrayTraverser) (i, metricsInspector) -> - addMetricsToAggregator(metricsInspector, aggregator) - ); - } - - private static void addMetricsToAggregator(Inspector metrics, SearchNodeMetricsAggregator aggregator) { - aggregator.addAll(metrics.field("values")); - } - - @SuppressWarnings("deprecation") - private static Slime doMetricsRequest(URI hostURI) { - HttpGet get = new HttpGet(hostURI); - try (CloseableHttpResponse response = httpClient.execute(get)) { - byte[] body = EntityUtils.toByteArray(response.getEntity()); - return SlimeUtils.jsonToSlime(body); - } catch (IOException e) { - log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); - return new Slime(); - } - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetriever.java new file mode 100644 index 00000000000..5b58e79b6d8 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetriever.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.metrics; + +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.yolean.Exceptions; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + +public class ClusterSearchNodeMetricsRetriever { + + private static final Logger log = Logger.getLogger(ClusterSearchNodeMetricsRetriever.class.getName()); + + private static final CloseableHttpClient httpClient = VespaHttpClientBuilder + .custom() + .connectTimeout(Timeout.ofSeconds(10)) + .apacheBuilder() + .setDefaultRequestConfig(RequestConfig.custom().setResponseTimeout(Timeout.ofSeconds(10)).build()) + .build(); + + + public Map requestMetricsGroupedByCluster(Collection hosts) { + Map clusterMetricsMap = new ConcurrentHashMap<>(); + for (URI uri : hosts) { + addMetricsFromHost(uri, clusterMetricsMap); + } +/* long startTime = System.currentTimeMillis(); + Runnable retrieveMetricsJob = () -> + hosts.parallelStream().forEach(host -> + addMetricsFromHost(host, clusterMetricsMap) + ); + + ForkJoinPool threadPool = new ForkJoinPool(10); + threadPool.submit(retrieveMetricsJob); + threadPool.shutdown(); + + try { + threadPool.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + log.log(Level.FINE, () -> + String.format("Proton metric retrieval for %d nodes took %d milliseconds", hosts.size(), System.currentTimeMillis() - startTime) + );*/ + + return clusterMetricsMap; + } + + private static void addMetricsFromHost(URI hostURI, Map clusterMetricsMap) { + Slime hostResponseBody = doMetricsRequest(hostURI); + Cursor error = hostResponseBody.get().field("error_message"); + + if (error.valid()) { + log.info("Failed to retrieve metrics from " + hostURI + ": " + error.asString()); + } + + Inspector nodes = hostResponseBody.get().field("nodes"); + nodes.traverse((ArrayTraverser) (i, nodesInspector) -> + parseNode(nodesInspector, clusterMetricsMap) + ); + } + + private static void parseNode(Inspector node, Map clusterMetricsMap) { + String nodeRole = node.field("role").asString(); + if(nodeRole.contains("content")) { + SearchNodeMetricsAggregator aggregator = new SearchNodeMetricsAggregator(); + clusterMetricsMap.put(nodeRole, aggregator); + node.field("services").traverse((ArrayTraverser) (i, servicesInspector) -> + addServicesToAggregator(servicesInspector, aggregator) + ); + } + } + + private static void addServicesToAggregator(Inspector services, SearchNodeMetricsAggregator aggregator) { + services.field("metrics").traverse((ArrayTraverser) (i, metricsInspector) -> + addMetricsToAggregator(metricsInspector, aggregator) + ); + } + + private static void addMetricsToAggregator(Inspector metrics, SearchNodeMetricsAggregator aggregator) { + aggregator.addAll(metrics.field("values")); + } + + @SuppressWarnings("deprecation") + private static Slime doMetricsRequest(URI hostURI) { + HttpGet get = new HttpGet(hostURI); + try (CloseableHttpResponse response = httpClient.execute(get)) { + byte[] body = EntityUtils.toByteArray(response.getEntity()); + return SlimeUtils.jsonToSlime(body); + } catch (IOException e) { + log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); + return new Slime(); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetriever.java index d417fb8683d..eb2f4ad37a4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetriever.java @@ -11,13 +11,13 @@ import java.util.function.Predicate; public class SearchNodeMetricsRetriever { - private final ClusterProtonMetricsRetriever metricsRetriever; + private final ClusterSearchNodeMetricsRetriever metricsRetriever; public SearchNodeMetricsRetriever() { - this( new ClusterProtonMetricsRetriever()); + this( new ClusterSearchNodeMetricsRetriever()); } - public SearchNodeMetricsRetriever(ClusterProtonMetricsRetriever metricsRetriever) { + public SearchNodeMetricsRetriever(ClusterSearchNodeMetricsRetriever metricsRetriever) { this.metricsRetriever = metricsRetriever; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetrieverTest.java index b9d0be61d65..4690f2eb139 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterSearchNodeMetricsRetrieverTest.java @@ -43,7 +43,7 @@ public class ClusterSearchNodeMetricsRetrieverTest { String expectedClusterNameContent = "content/content/0/0"; String expectedClusterNameMusic = "content/music/0/0"; - Map aggregatorMap = new ClusterProtonMetricsRetriever().requestMetricsGroupedByCluster(hosts); + Map aggregatorMap = new ClusterSearchNodeMetricsRetriever().requestMetricsGroupedByCluster(hosts); compareAggregators( new SearchNodeMetricsAggregator() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetrieverTest.java index f0e6b4171c2..d831b6d78b0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/SearchNodeMetricsRetrieverTest.java @@ -28,7 +28,7 @@ public class SearchNodeMetricsRetrieverTest { @Test public void getMetrics() { SearchNodeMetricsRetrieverTest.MockModel mockModel = new MockModel(mockHosts()); - SearchNodeMetricsRetrieverTest.MockProtonMetricsRetriever mockMetricsRetriever = new MockProtonMetricsRetriever(); + MockSearchNodeMetricsRetriever mockMetricsRetriever = new MockSearchNodeMetricsRetriever(); Application application = new Application(mockModel, null, 0, null, null, ApplicationId.fromSerializedForm("tenant:app:instance")); @@ -53,7 +53,7 @@ public class SearchNodeMetricsRetrieverTest { return List.of(hostInfo1, hostInfo2, hostInfo3); } - static class MockProtonMetricsRetriever extends ClusterProtonMetricsRetriever { + static class MockSearchNodeMetricsRetriever extends ClusterSearchNodeMetricsRetriever { Collection hosts = new ArrayList<>(); -- cgit v1.2.3