diff options
Diffstat (limited to 'metrics-proxy')
6 files changed, 408 insertions, 0 deletions
diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 99354ee22e8..ff314147090 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -149,6 +149,11 @@ <artifactId>hamcrest-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 14d1203824b..fc81bd1a0ee 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -149,6 +149,10 @@ public class MetricsManager { externalMetrics.setExtraMetrics(packets); } + public Map<DimensionId, String> getExtraDimensions() { + return this.extraDimensions; + } + /** * Returns a space separated list of all distinct service names. */ diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java new file mode 100644 index 00000000000..c6dfdf8fe9c --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java @@ -0,0 +1,204 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.gatherer; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.vespa.defaults.Defaults; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class NodeMetricGatherer { + + private static final int COREDUMP_AGE_IN_MINUTES = 12600; + private static final JSONObject ROUTING_JSON = createRoutingJSON(); + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final MetricsManager metricsManager; + private final FileWrapper fileWrapper; + + private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); + + @Inject + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); + } + + public NodeMetricGatherer(MetricsManager metricsManager, + VespaServices vespaServices, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions, + FileWrapper fileWrapper) { + this.metricsManager = metricsManager; + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + this.fileWrapper = fileWrapper; + } + + public List<MetricsPacket> gatherMetrics() { + List<MetricsPacket.Builder> metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.addAll(coredumpMetrics()); + metricPacketBuilders.addAll(serviceHealthMetrics()); + metricPacketBuilders.addAll(hostLifeMetrics()); + + List<MetricsPacket> metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); + return metricPacketBuilder.build(); + }).collect(Collectors.toList()); + return metricPackets; + } + + private List<MetricsPacket.Builder> coredumpMetrics() { + + Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); + long coredumps = getCoredumpsFromLastPeriod(crashPath); + + try { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("application", "Vespa.node"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + } + + private List<MetricsPacket.Builder> serviceHealthMetrics() { + return vespaServices.getVespaServices() + .stream() + .map(service -> { + try { + StatusCode healthStatus = service.getHealth().getStatus(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status_code", healthStatus.code); + jsonObject.put("status_message", healthStatus.status); + jsonObject.put("application", service.getMonitoringName()); + JSONObject dimensions = new JSONObject(); + dimensions.put("instance", service.getInstanceName()); + dimensions.put("metrictype", "health"); + jsonObject.put("dimensions", dimensions); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); + } catch (JSONException e) { + throw new RuntimeException(e.getMessage()); + } + }) + .collect(Collectors.toList()); + } + + private List<MetricsPacket.Builder> hostLifeMetrics() { + JSONObject jsonObject = new JSONObject(); + double upTime; + int statusCode = 0; + String statusMessage = "OK"; + try { + upTime = getHostLife(Path.of("/proc/uptime")); // ?? + } catch (IOException e) { + upTime = 0d; + statusCode = 1; + statusMessage = e.getMessage(); + } + + try { + jsonObject.put("application", "host_life"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + + + } + + private long getCoredumpsFromLastPeriod(Path coreDumpPath) { + try { + return fileWrapper.walkTree(coreDumpPath) + .filter(Files::isRegularFile) + .filter(this::isNewFile) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private double getHostLife(Path uptimePath) throws IOException { + return fileWrapper.readAllLines(uptimePath) + .stream() + .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) + .findFirst() + .orElseThrow(); + } + + private boolean isNewFile(Path file) { + try { + return fileWrapper.getLastModifiedTime(file) + .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) + .isBefore(Instant.now()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static JSONObject createRoutingJSON() { + try { + JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); + return jsonObject; + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + static class FileWrapper { + + List<String> readAllLines(Path path) throws IOException { + return Files.readAllLines(path); + } + + Stream<Path> walkTree(Path path) throws IOException { + return Files.walk(path); + } + + Instant getLastModifiedTime(Path path) throws IOException { + return Files.getLastModifiedTime(path).toInstant(); + } + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 00f2078ef57..34fedfcfa7a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -9,6 +9,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaServices; import java.time.Instant; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java new file mode 100644 index 00000000000..e29cf202bd9 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java @@ -0,0 +1,82 @@ +/* +* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +*/ + +package ai.vespa.metricsproxy.http.Yamas; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer; +import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.JsonResponse; +import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.Path; + +import java.util.List; +import java.util.concurrent.Executor; + +import static ai.vespa.metricsproxy.http.RestApiUtil.resourceListResponse; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * @author olaa + */ + + +public class YamasHandler extends ThreadedHttpRequestHandler { + + public static final String V1_PATH = "/yamas/v1"; + private static final String VALUES_PATH = V1_PATH + "/values"; + + private final ValuesFetcher valuesFetcher; + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor); + this.valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); + + Path path = new Path(request.getUri()); + + if (path.matches(V1_PATH)) return resourceListResponse(request.getUri(), List.of(VALUES_PATH)); + if (path.matches(VALUES_PATH)) return valuesResponse(request); + + return new ErrorResponse(NOT_FOUND, "No content at given path"); + } + + private JsonResponse valuesResponse(HttpRequest request) { + try { + List<MetricsPacket> metrics = valuesFetcher.fetch(request.getProperty("consumer")); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + +}
\ No newline at end of file diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java new file mode 100644 index 00000000000..8752032b643 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java @@ -0,0 +1,112 @@ +package ai.vespa.metricsproxy.gatherer; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.HealthMetric; +import ai.vespa.metricsproxy.metric.Metric; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; +import ai.vespa.metricsproxy.service.VespaService; +import ai.vespa.metricsproxy.service.VespaServices; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author olaa + */ +public class NodeMetricGathererTest { + + @Test + public void gatherMetrics() throws Exception { + + MetricsManager metricsManager = mock(MetricsManager.class); + VespaServices vespaServices = mock(VespaServices.class); + ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); + NodeDimensions nodeDimensions = mock(NodeDimensions.class); + NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); + List<VespaService> mockedVespaServices = mockedVespaServices(); + + NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); + when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); + when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); + when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); + when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); + when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + + List<MetricsPacket> packets = nodeMetricGatherer.gatherMetrics(); + assertEquals(5, packets.size()); + Map<DimensionId, String> serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); + List<MetricsPacket> expectedPackets = List.of( + metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), + metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), + metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) + ); + + assertEqualMetricPackets(expectedPackets, packets); + } + + private void assertEqualMetricPackets(List<MetricsPacket> expectedPackets, List<MetricsPacket> actualPackets) { + assertEquals(expectedPackets.size(), actualPackets.size()); + expectedPackets.stream() + .forEach(expectedPacket -> + actualPackets.stream() + .filter(packet -> packet.service.equals(expectedPacket.service)) + .forEach(actualPacket -> { + assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); + assertEquals(expectedPacket.statusCode, actualPacket.statusCode); + assertEquals(expectedPacket.metrics(), actualPacket.metrics()); + assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); + }) + ); + } + + private List<VespaService> mockedVespaServices() { + + HealthMetric healthy = HealthMetric.get("OK", ""); + HealthMetric unhealthy = HealthMetric.get("down", ""); + + VespaService service1 = mock(VespaService.class); + when(service1.getInstanceName()).thenReturn("instance"); + when(service1.getMonitoringName()).thenReturn("service1"); + when(service1.getHealth()).thenReturn(healthy); + + VespaService service2 = mock(VespaService.class); + when(service2.getInstanceName()).thenReturn("instance"); + when(service2.getMonitoringName()).thenReturn("service2"); + when(service2.getHealth()).thenReturn(healthy); + + + VespaService service3 = mock(VespaService.class); + when(service3.getInstanceName()).thenReturn("instance"); + when(service3.getMonitoringName()).thenReturn("service3"); + when(service3.getHealth()).thenReturn(unhealthy); + + return List.of(service1, service2, service3); + + } + + private MetricsPacket metricsPacket(int statusCode, String service, + Collection<Metric> metrics, Map<DimensionId, String> dimensions) { + return new MetricsPacket.Builder(ServiceId.toServiceId(service)) + .putDimensions(dimensions) + .putMetrics(metrics) + .statusCode(statusCode) + .build(); + } +}
\ No newline at end of file |