diff options
11 files changed, 8 insertions, 372 deletions
diff --git a/application/pom.xml b/application/pom.xml index d173fec19cf..7bcda0ce386 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -119,8 +119,8 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> + <scope>test</scope> </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 1bf9c91384b..efd00528d54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; -import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; @@ -10,8 +9,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; -import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.jdisc.http.ServletPathsConfig; @@ -20,10 +17,8 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; -import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; @@ -50,10 +45,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat RankProfilesConfig.Producer, RankingConstantsConfig.Producer, ServletPathsConfig.Producer, - ContainerMbusConfig.Producer, - MetricsProxyApiConfig.Producer { - - public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); + ContainerMbusConfig.Producer { private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -80,7 +72,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); - addMetricsV2Handler(); addTestrunnerComponentsIfTester(deployState); } @@ -108,14 +99,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } - public void addMetricsV2Handler() { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( - new ComponentModel(METRICS_V2_HANDLER_CLASS, null, null, null)); - handler.addServerBindings("http://*" + MetricsV2Handler.V2_PATH, - "http://*" + MetricsV2Handler.V2_PATH + "/*"); - addComponent(handler); - } - private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); @@ -205,12 +188,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } @Override - public void getConfig(MetricsProxyApiConfig.Builder builder) { - builder.metricsPort(MetricsProxyContainer.BASEPORT) - .metricsApiPath(ApplicationMetricsHandler.V1_PATH); - } - - @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); builder.jvm.verbosegc(true) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 6b1a94e16ae..d3ba2718d71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; @@ -35,8 +34,7 @@ public final class AccessControl { ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, ContainerCluster.STATE_HANDLER_CLASS, - ContainerCluster.LOG_HANDLER_CLASS, - ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS + ContainerCluster.LOG_HANDLER_CLASS ); public static final class Builder { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 75b41ff3667..edbddcd4804 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -25,7 +25,6 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.VipStatusHandler; -import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.servlet.ServletConfigConfig; @@ -226,13 +225,12 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertThat(defaultRootHandler.serverBindings(), contains("http://*/")); JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); - assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus")); + assertThat(applicationStatusHandler.serverBindings(), + contains("http://*/ApplicationStatus")); JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); - assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html")); - - JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName()); - assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*")); + assertThat(fileRequestHandler.serverBindings(), + contains("http://*/status.html")); } @Test diff --git a/container-core/pom.xml b/container-core/pom.xml index e51d99c3b78..f3861c92129 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -16,16 +16,6 @@ <packaging>container-plugin</packaging> <dependencies> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -237,11 +227,6 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>com.github.tomakehurst</groupId> - <artifactId>wiremock-standalone</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java deleted file mode 100644 index 78ea62e1b3a..00000000000 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.handler.metrics; - -import ai.vespa.util.http.VespaHttpClientBuilder; -import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.Path; -import com.yahoo.yolean.Exceptions; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; - -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; - -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.OK; - -/** - * @author gjoranv - */ -public class MetricsV2Handler extends HttpHandlerBase { - - public static final String V2_PATH = "/metrics/v2"; - static final String VALUES_PATH = V2_PATH + "/values"; - - private static final int HTTP_CONNECT_TIMEOUT = 5000; - private static final int HTTP_SOCKET_TIMEOUT = 30000; - - private final String metricsProxyUri; - private final HttpClient httpClient = createHttpClient(); - - @Inject - public MetricsV2Handler(Executor executor, - MetricsProxyApiConfig config) { - super(executor); - metricsProxyUri = "http://localhost:" + config.metricsPort() + config.metricsApiPath(); - } - - @Override - protected Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { - if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); - if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); - return Optional.empty(); - } - - private JsonResponse valuesResponse(String consumer) { - try { - String uri = metricsProxyUri + consumerQuery(consumer); - String metricsJson = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); - return new JsonResponse(OK, metricsJson); - } catch (IOException e) { - log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); - return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); - } - } - - private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create() - .setUserAgent("application-metrics-retriever") - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(HTTP_CONNECT_TIMEOUT) - .setSocketTimeout(HTTP_SOCKET_TIMEOUT) - .build()) - .build(); - } - - static String consumerQuery(String consumer) { - return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer; - } -} diff --git a/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def deleted file mode 100644 index 3e5b973e3f3..00000000000 --- a/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace=container.handler.metrics - -metricsPort int -metricsApiPath string diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java deleted file mode 100644 index b57814e50aa..00000000000 --- a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.handler.metrics; - -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static com.yahoo.container.handler.metrics.MetricsV2Handler.V2_PATH; -import static com.yahoo.container.handler.metrics.MetricsV2Handler.VALUES_PATH; -import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author gjoranv - */ -public class MetricsV2HandlerTest { - - private static final String URI_BASE = "http://localhost"; - - private static final String V2_URI = URI_BASE + V2_PATH; - private static final String VALUES_URI = URI_BASE + VALUES_PATH; - - // Mock applicationmetrics api - private static final String MOCK_METRICS_PATH = "/node0"; - - private static final String TEST_FILE = "application-metrics.json"; - private static final String RESPONSE = getFileContents(TEST_FILE); - private static final String CPU_METRIC = "cpu.util"; - private static final String REPLACED_CPU_METRIC = "replaced_cpu_util"; - private static final String CUSTOM_CONSUMER = "custom-consumer"; - - private static RequestHandlerTestDriver testDriver; - - @Rule - public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort()); - - @Before - public void setup() { - setupWireMock(); - var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), - new MetricsProxyApiConfig.Builder() - .metricsPort(wireMockRule.port()) - .metricsApiPath(MOCK_METRICS_PATH) - .build()); - testDriver = new RequestHandlerTestDriver(handler); - } - - private void setupWireMock() { - wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) - .willReturn(aResponse().withBody(RESPONSE))); - - // Add a slightly different response for a custom consumer. - String myConsumerResponse = RESPONSE.replaceAll(CPU_METRIC, REPLACED_CPU_METRIC); - wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) - .withQueryParam("consumer", equalTo(CUSTOM_CONSUMER)) - .willReturn(aResponse().withBody(myConsumerResponse))); - } - - @Test - public void v2_response_contains_values_uri() throws Exception { - String response = testDriver.sendRequest(V2_URI).readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("resources")); - - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); - - JSONObject valuesUri = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUri.getString("url")); - } - - @Ignore - @Test - public void visually_inspect_values_response() throws Exception { - JSONObject responseJson = getResponseAsJson(null); - System.out.println(responseJson.toString(4)); - } - - @Test - public void invalid_path_yields_error_response() throws Exception { - String response = testDriver.sendRequest(V2_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("error")); - assertTrue(root.getString("error" ).startsWith("No content")); - } - - @Test - public void values_response_is_equal_to_test_file() { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - assertEquals(RESPONSE, response); - } - - @Test - public void consumer_is_propagated_to_metrics_proxy_api() throws JSONException { - JSONObject responseJson = getResponseAsJson(CUSTOM_CONSUMER); - - JSONObject firstNodeMetricsValues = - responseJson.getJSONArray("nodes").getJSONObject(0) - .getJSONObject("node") - .getJSONArray("metrics").getJSONObject(0) - .getJSONObject("values"); - - assertTrue(firstNodeMetricsValues.has(REPLACED_CPU_METRIC)); - } - - private JSONObject getResponseAsJson(String consumer) { - String response = testDriver.sendRequest(VALUES_URI + consumerQuery(consumer)).readAll(); - try { - return new JSONObject(response); - } catch (JSONException e) { - fail("Failed to create json object: " + e.getMessage()); - throw new RuntimeException(e); - } - } - - private static String getFileContents(String filename) { - InputStream in = MetricsV2HandlerTest.class.getClassLoader().getResourceAsStream(filename); - if (in == null) { - throw new RuntimeException("File not found: " + filename); - } - return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); - } - -} diff --git a/container-core/src/test/resources/application-metrics.json b/container-core/src/test/resources/application-metrics.json deleted file mode 100644 index 52cbb721bb1..00000000000 --- a/container-core/src/test/resources/application-metrics.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "nodes": [ - { - "hostname": "node0", - "role": "role0", - "node": { - "timestamp": 1234, - "metrics": [ - { - "values": { - "cpu.util": 16.222 - }, - "dimensions": { - "state": "active" - } - } - ] - }, - "services": [ - { - "name": "searchnode", - "timestamp": 1234, - "status": { - "code": "up" - }, - "metrics": [ - { - "values": { - "queries.count": 4 - }, - "dimensions": { - "documentType": "music" - } - } - ] - } - ] - }, - { - "hostname": "node1", - "role": "role1", - "node": { - "timestamp": 1234, - "metrics": [ - { - "values": { - "cpu.util": 32.444 - }, - "dimensions": { - "state": "active" - } - } - ] - }, - "services": [ - { - "name": "searchnode", - "timestamp": 1234, - "status": { - "code": "up" - }, - "metrics": [ - { - "values": { - "queries.count": 8 - }, - "dimensions": { - "documentType": "music" - } - } - ] - }, - { - "name": "slobrok", - "timestamp": 1234, - "status": { - "code": "unknown", - "description": "Unable to fetch metrics from service 'slobrok'" - }, - "metrics": [ - { - "values": {}, - "dimensions": { - "instance": "slobrok0" - } - } - ] - } - ] - } - ] -} diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 738a4cc8700..a7217d05315 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -106,10 +106,6 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> </exclusions> </dependency> <dependency> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java index 8d525310d6f..465b419be09 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java @@ -22,7 +22,7 @@ public final class PublicDimensions { // Node-specific. public static final String INTERNAL_CLUSTER_TYPE = "clustertype"; public static final String INTERNAL_CLUSTER_ID = "clusterid"; - public static final String CLUSTER_ID = "clusterId"; + public static final String CLUSTER_ID = INTERNAL_CLUSTER_ID; // Internal name (instance) is confusing, so renamed to 'serviceId' for public use. // This is added by the metrics-proxy. |