diff options
author | Ola Aunrønning <olaa@verizonmedia.com> | 2019-09-05 11:27:07 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-09-05 11:27:07 +0200 |
commit | 3b81210bcbb23dd073d2b6903a1179039ae2012e (patch) | |
tree | ba7a62cae3931565c179727f640b5e69a8c077ba | |
parent | 7d8823ef79a852ea21eaf3b092fd989d723bd844 (diff) | |
parent | d84dddd3d6544c5f0f640a908e673cf90ce3667b (diff) |
Merge pull request #10472 from vespa-engine/olaa/metrics-proxy-gather-node-metrics
Yamas handler and system metric gatherer
13 files changed, 369 insertions, 2 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 6cd0a80bee3..94dbdafe980 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -10,6 +10,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.MetricsHandler; +import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; @@ -111,6 +112,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addMetricsProxyComponent(VespaMetrics.class); addHttpHandler(MetricsHandler.class, MetricsHandler.V1_PATH); addHttpHandler(PrometheusHandler.class, PrometheusHandler.V1_PATH); + addHttpHandler(YamasHandler.class, YamasHandler.V1_PATH); } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index db29dbba306..be6f84c2a28 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -6,6 +6,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.http.MetricsHandler; +import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; import com.yahoo.component.ComponentSpecification; @@ -107,6 +108,7 @@ public class MetricsProxyContainerClusterTest { assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(MetricsHandler.class.getName()))); assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(PrometheusHandler.class.getName()))); + assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(YamasHandler.class.getName()))); } @Test diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 14d1203824b..fc81bd1a0ee 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -149,6 +149,10 @@ public class MetricsManager { externalMetrics.setExtraMetrics(packets); } + public Map<DimensionId, String> getExtraDimensions() { + return this.extraDimensions; + } + /** * Returns a space separated list of all distinct service names. */ diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 00f2078ef57..830e2201966 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -17,7 +17,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericJsonModel; /** * Generates metrics values in json format for the metrics/v1 rest api. diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java new file mode 100644 index 00000000000..7a868d0bffa --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -0,0 +1,67 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.yamas; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.node.NodeMetricGatherer; +import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.HttpHandlerBase; +import ai.vespa.metricsproxy.http.JsonResponse; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author olaa + */ + +public class YamasHandler extends HttpHandlerBase { + + public static final String V1_PATH = "/yamas/v1"; + private static final String VALUES_PATH = V1_PATH + "/values"; + + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor, metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List<MetricsPacket> metrics = valuesFetcher.fetch(consumer); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + +}
\ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java new file mode 100644 index 00000000000..91e7fbbc0b5 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java @@ -0,0 +1,46 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Set; +import java.util.logging.Logger; + + +/** + * @author olaa + */ +public class CoredumpMetricGatherer { + + private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); + + private static final Logger logger = Logger.getLogger(CoredumpMetricGatherer.class.getSimpleName()); + + + protected static MetricsPacket.Builder gatherCoredumpMetrics(FileWrapper fileWrapper) { + int coredumps = getNumberOfCoredumps(fileWrapper); + return new MetricsPacket.Builder(ServiceId.toServiceId("system-coredumps-processing")) + .timestamp(Instant.now().getEpochSecond()) + .statusCode(coredumps) + .statusMessage(coredumps == 0 ? "OK" : String.format("Found %d coredumps", coredumps)) + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); + } + + private static int getNumberOfCoredumps(FileWrapper fileWrapper) { + try { + return (int) fileWrapper.walkTree(COREDUMP_PATH) + .filter(fileWrapper::isRegularFile) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java new file mode 100644 index 00000000000..aa01802883e --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class FileWrapper { + + long getFileAgeInSeconds(Path path) throws IOException { + Instant lastModifiedTime = Files.getLastModifiedTime(path).toInstant(); + return Instant.now().getEpochSecond() - lastModifiedTime.getEpochSecond(); + } + + Stream<Path> walkTree(Path path) throws IOException { + return Files.walk(path); + } + + boolean isRegularFile(Path path) { + return Files.isRegularFile(path); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java new file mode 100644 index 00000000000..8fdfc93022c --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java @@ -0,0 +1,44 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.MetricId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Set; + + +/** + * @author olaa + */ +public class HostLifeGatherer { + + private static final Path UPTIME_PATH = Path.of("/proc"); + + protected static MetricsPacket.Builder gatherHostLifeMetrics(FileWrapper fileWrapper) { + long upTime; + int statusCode = 0; + String statusMessage = "OK"; + + try { + upTime = fileWrapper.getFileAgeInSeconds(UPTIME_PATH); + } catch (IOException e) { + upTime = 0; + statusCode = 1; + statusMessage = e.getMessage(); + } + + return new MetricsPacket.Builder(ServiceId.toServiceId("host_life")) + .timestamp(Instant.now().getEpochSecond()) + .statusMessage(statusMessage) + .statusCode(statusCode) + .putMetric(MetricId.toMetricId("uptime"), upTime) + .putMetric(MetricId.toMetricId("alive"), 1) + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java new file mode 100644 index 00000000000..cd71bb781fe --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -0,0 +1,63 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static ai.vespa.metricsproxy.node.CoredumpMetricGatherer.gatherCoredumpMetrics; +import static ai.vespa.metricsproxy.node.HostLifeGatherer.gatherHostLifeMetrics; +import static ai.vespa.metricsproxy.node.ServiceHealthGatherer.gatherServiceHealthMetrics; + +/** + * Fetches miscellaneous system metrics for node, including + * - Current coredump processing + * - Health of Vespa services + * - Host life + * + * @author olaa + */ + + +public class NodeMetricGatherer { + + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final MetricsManager metricsManager; + + @Inject + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this.metricsManager = metricsManager; + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + } + + public List<MetricsPacket> gatherMetrics() { + FileWrapper fileWrapper = new FileWrapper(); + List<MetricsPacket.Builder> metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.add(gatherCoredumpMetrics(fileWrapper)); + metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); + + if (SystemPollerProvider.runningOnLinux()) { + metricPacketBuilders.add(gatherHostLifeMetrics(fileWrapper)); + } + + return metricPacketBuilders.stream() + .map(metricPacketBuilder -> + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()) + .putDimensionsIfAbsent(nodeDimensions.getDimensions()) + .putDimensionsIfAbsent(metricsManager.getExtraDimensions()).build() + ).collect(Collectors.toList()); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java new file mode 100644 index 00000000000..923fb4d646d --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java @@ -0,0 +1,36 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; +import ai.vespa.metricsproxy.service.VespaServices; + +import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * @author olaa + */ +public class ServiceHealthGatherer { + + + protected static List<MetricsPacket.Builder> gatherServiceHealthMetrics(VespaServices vespaServices) { + return vespaServices.getVespaServices() + .stream() + .map(service -> + new MetricsPacket.Builder(ServiceId.toServiceId(service.getMonitoringName())) + .timestamp(Instant.now().getEpochSecond()) + .statusMessage(service.getHealth().getStatus().status) + .statusCode(service.getHealth().getStatus().code) + .putDimension(DimensionId.toDimensionId("instance"), service.getInstanceName()) + .putDimension(DimensionId.toDimensionId("metrictype"), "health") + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))) + ) + .collect(Collectors.toList()); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java index ea47933ddf0..6494054c278 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java @@ -38,7 +38,7 @@ public class SystemPollerProvider implements Provider<SystemPoller> { return poller; } - private static boolean runningOnLinux() { + public static boolean runningOnLinux() { return getOs().contains("nux"); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java new file mode 100644 index 00000000000..e7a905be52e --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class CoredumpGathererTest { + + @Test + public void finds_one_coredump() { + MetricsPacket packet = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()).build(); + + assertEquals("system-coredumps-processing", packet.service.id); + assertEquals(1, packet.statusCode); + } + + static class MockFileWrapper extends FileWrapper { + + + @Override + Stream<Path> walkTree(Path path) { + return Stream.of(Path.of("dummy-path")); + } + + @Override + boolean isRegularFile(Path path) { + return true; + } + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java new file mode 100644 index 00000000000..8317229b73a --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java @@ -0,0 +1,37 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class HostLifeGathererTest { + + @Test + public void host_is_alive() { + MetricsPacket packet = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()).build(); + + assertEquals("host_life", packet.service.id); + assertEquals(0, packet.statusCode); + assertEquals(123l, packet.metrics().get(MetricId.toMetricId("uptime"))); + assertEquals(1, packet.metrics().get(MetricId.toMetricId("alive"))); + + } + + static class MockFileWrapper extends FileWrapper { + @Override + long getFileAgeInSeconds(Path path) { + return 123; + } + } +} |