From 0d70292084a2f50b7c6e95673790f063d2c77767 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 26 Aug 2019 12:21:10 +0200 Subject: Added module to fetch misc node metrics --- .../metricsproxy/MetricsProxyContainerCluster.java | 1 + .../metricsproxy/gatherer/NodeMetricGatherer.java | 174 +++++++++++++++++++++ .../ai/vespa/metricsproxy/http/MetricsHandler.java | 1 + .../ai/vespa/metricsproxy/http/ValuesFetcher.java | 1 + 4 files changed, 177 insertions(+) create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 091c5a3acb4..42c49a7a25c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -13,6 +13,7 @@ import ai.vespa.metricsproxy.http.MetricsHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java new file mode 100644 index 00000000000..895792e8d5f --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java @@ -0,0 +1,174 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.gatherer; + +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; +import ai.vespa.metricsproxy.metric.model.json.YamasArrayJsonModel; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.vespa.defaults.Defaults; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * @author olaa + */ +public class NodeMetricGatherer { + + private static final int COREDUMP_AGE_IN_MINUTES = 12600; + private static final JSONObject ROUTING_JSON = createRoutingJSON(); + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final String coreDumpPath; + private final String hostLifePath; + + private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); + + @Inject + public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this(vespaServices, applicationDimensions, nodeDimensions, "var/crash/processing", "/proc/uptime"); + } + + public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions, String coreDumpPath, String hostLifePath) { + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + this.coreDumpPath = coreDumpPath; + this.hostLifePath = hostLifePath; + } + + public YamasArrayJsonModel gatherMetrics() throws JSONException { + List metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.addAll(coredumpMetrics()); + metricPacketBuilders.addAll(serviceHealthMetrics()); + metricPacketBuilders.addAll(hostLifeMetrics()); + + List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); + return metricPacketBuilder.build(); + }).collect(Collectors.toList()); + return YamasJsonUtil.toYamasArray(metricPackets); + } + + private List coredumpMetrics() throws JSONException { + + Path crashPath = Path.of(Defaults.getDefaults().underVespaHome(coreDumpPath)); + long coredumps = getCoredumpsFromLastPeriod(crashPath); + + JSONObject jsonObject = new JSONObject(); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } + + private List serviceHealthMetrics() { + JSONArray jsonArray = new JSONArray(); + vespaServices.getVespaServices() + .stream() + .forEach(service -> { + try { + StatusCode healthStatus = service.getHealth().getStatus(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status_code", healthStatus.code); + jsonObject.put("status_message", healthStatus.status); + jsonObject.put("application", service.getMonitoringName()); + JSONObject dimensions = new JSONObject(); + dimensions.put("instance", service.getInstanceName()); + dimensions.put("metricsType", "health"); + jsonObject.put("dimensions", dimensions); + jsonObject.put("routing", ROUTING_JSON); + jsonArray.put(jsonObject); + } catch (JSONException e) { + + } + }); + + return YamasJsonUtil.toMetricsPackets(jsonArray.toString()); + } + + private List hostLifeMetrics() throws JSONException { + JSONObject jsonObject = new JSONObject(); + double upTime; + int statusCode = 0; + String statusMessage = "OK"; + try { + upTime = getHostLife(Path.of(hostLifePath)); // ?? + } catch (IOException e) { + upTime = 0d; + statusCode = 1; + statusMessage = e.getMessage(); + } + + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } + + private long getCoredumpsFromLastPeriod(Path coreDumpPath) { + + try { + return Files.walk(coreDumpPath) + .filter(Files::isRegularFile) + .filter(this::isNewFile) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private double getHostLife(Path uptimePath) throws IOException { + return Files.readAllLines(uptimePath) + .stream() + .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) + .findFirst() + .orElseThrow(); + } + + private boolean isNewFile(Path file) { + try { + return Files.getLastModifiedTime(file).toInstant() + .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) + .isBefore(Instant.now()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static final JSONObject createRoutingJSON() { + try { + JSONObject namesspace = new JSONObject(); + namesspace.put("yamas", "Vespa"); + return namesspace; + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java index 8fcab6dfcab..e56c54e39b3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsHandler.java @@ -46,6 +46,7 @@ public class MetricsHandler extends ThreadedHttpRequestHandler { VespaServices vespaServices, MetricsConsumers metricsConsumers) { super(executor); + valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 4686d9c1751..cd433378aee 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -9,6 +9,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaServices; import java.time.Instant; -- cgit v1.2.3 From 6d1f6a5589ae52e1c58fd9c28927562dbda3f468 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 30 Aug 2019 10:58:36 +0200 Subject: Added YamasHandler and test --- metrics-proxy/pom.xml | 5 + .../ai/vespa/metricsproxy/core/MetricsManager.java | 4 + .../metricsproxy/gatherer/NodeMetricGatherer.java | 126 +++++++++++++-------- .../ai/vespa/metricsproxy/http/YamasHandler.java | 110 ++++++++++++++++++ .../gatherer/NodeMetricGathererTest.java | 112 ++++++++++++++++++ 5 files changed, 309 insertions(+), 48 deletions(-) create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 26caa3f66af..29ab5bc3f91 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -141,6 +141,11 @@ hamcrest-core test + + org.mockito + mockito-core + test + diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 14d1203824b..fc81bd1a0ee 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -149,6 +149,10 @@ public class MetricsManager { externalMetrics.setExtraMetrics(packets); } + public Map getExtraDimensions() { + return this.extraDimensions; + } + /** * Returns a space separated list of all distinct service names. */ diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java index 895792e8d5f..c6dfdf8fe9c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java @@ -1,16 +1,15 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.gatherer; +import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.StatusCode; -import ai.vespa.metricsproxy.metric.model.json.YamasArrayJsonModel; import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -21,9 +20,12 @@ import java.nio.file.Path; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author olaa @@ -35,25 +37,29 @@ public class NodeMetricGatherer { private final VespaServices vespaServices; private final ApplicationDimensions applicationDimensions; private final NodeDimensions nodeDimensions; - private final String coreDumpPath; - private final String hostLifePath; + private final MetricsManager metricsManager; + private final FileWrapper fileWrapper; private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); @Inject - public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - this(vespaServices, applicationDimensions, nodeDimensions, "var/crash/processing", "/proc/uptime"); + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); } - public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions, String coreDumpPath, String hostLifePath) { + public NodeMetricGatherer(MetricsManager metricsManager, + VespaServices vespaServices, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions, + FileWrapper fileWrapper) { + this.metricsManager = metricsManager; this.vespaServices = vespaServices; this.applicationDimensions = applicationDimensions; this.nodeDimensions = nodeDimensions; - this.coreDumpPath = coreDumpPath; - this.hostLifePath = hostLifePath; + this.fileWrapper = fileWrapper; } - public YamasArrayJsonModel gatherMetrics() throws JSONException { + public List gatherMetrics() { List metricPacketBuilders = new ArrayList<>(); metricPacketBuilders.addAll(coredumpMetrics()); metricPacketBuilders.addAll(serviceHealthMetrics()); @@ -62,30 +68,36 @@ public class NodeMetricGatherer { List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); return metricPacketBuilder.build(); }).collect(Collectors.toList()); - return YamasJsonUtil.toYamasArray(metricPackets); + return metricPackets; } - private List coredumpMetrics() throws JSONException { + private List coredumpMetrics() { - Path crashPath = Path.of(Defaults.getDefaults().underVespaHome(coreDumpPath)); + Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); long coredumps = getCoredumpsFromLastPeriod(crashPath); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "system-coredumps-processing"); - jsonObject.put("status_code", coredumps); - jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + try { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("application", "Vespa.node"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } } private List serviceHealthMetrics() { - JSONArray jsonArray = new JSONArray(); - vespaServices.getVespaServices() + return vespaServices.getVespaServices() .stream() - .forEach(service -> { + .map(service -> { try { StatusCode healthStatus = service.getHealth().getStatus(); JSONObject jsonObject = new JSONObject(); @@ -94,47 +106,52 @@ public class NodeMetricGatherer { jsonObject.put("application", service.getMonitoringName()); JSONObject dimensions = new JSONObject(); dimensions.put("instance", service.getInstanceName()); - dimensions.put("metricsType", "health"); + dimensions.put("metrictype", "health"); jsonObject.put("dimensions", dimensions); jsonObject.put("routing", ROUTING_JSON); - jsonArray.put(jsonObject); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); } catch (JSONException e) { - + throw new RuntimeException(e.getMessage()); } - }); - - return YamasJsonUtil.toMetricsPackets(jsonArray.toString()); + }) + .collect(Collectors.toList()); } - private List hostLifeMetrics() throws JSONException { + private List hostLifeMetrics() { JSONObject jsonObject = new JSONObject(); double upTime; int statusCode = 0; String statusMessage = "OK"; try { - upTime = getHostLife(Path.of(hostLifePath)); // ?? + upTime = getHostLife(Path.of("/proc/uptime")); // ?? } catch (IOException e) { upTime = 0d; statusCode = 1; statusMessage = e.getMessage(); } - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("status_message", statusMessage); - jsonObject.put("status_code", statusCode); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - jsonObject.put("routing", ROUTING_JSON); + try { + jsonObject.put("application", "host_life"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); } private long getCoredumpsFromLastPeriod(Path coreDumpPath) { - try { - return Files.walk(coreDumpPath) + return fileWrapper.walkTree(coreDumpPath) .filter(Files::isRegularFile) .filter(this::isNewFile) .count(); @@ -144,7 +161,7 @@ public class NodeMetricGatherer { } private double getHostLife(Path uptimePath) throws IOException { - return Files.readAllLines(uptimePath) + return fileWrapper.readAllLines(uptimePath) .stream() .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) .findFirst() @@ -153,7 +170,7 @@ public class NodeMetricGatherer { private boolean isNewFile(Path file) { try { - return Files.getLastModifiedTime(file).toInstant() + return fileWrapper.getLastModifiedTime(file) .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) .isBefore(Instant.now()); } catch (IOException e) { @@ -161,14 +178,27 @@ public class NodeMetricGatherer { } } - private static final JSONObject createRoutingJSON() { + private static JSONObject createRoutingJSON() { try { - JSONObject namesspace = new JSONObject(); - namesspace.put("yamas", "Vespa"); - return namesspace; + JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); + return jsonObject; } catch (JSONException e) { throw new RuntimeException(e); } } + static class FileWrapper { + + List readAllLines(Path path) throws IOException { + return Files.readAllLines(path); + } + + Stream walkTree(Path path) throws IOException { + return Files.walk(path); + } + + Instant getLastModifiedTime(Path path) throws IOException { + return Files.getLastModifiedTime(path).toInstant(); + } + } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java new file mode 100644 index 00000000000..fca1b66342f --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java @@ -0,0 +1,110 @@ +/* +* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +*/ + +package ai.vespa.metricsproxy.http; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.Path; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.URI; +import java.util.List; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static java.util.logging.Level.WARNING; + +/** + * @author olaa + */ + +// TODO: Merge with gjoranv's API util changes +public class YamasHandler extends ThreadedHttpRequestHandler { + + static final String V1_PATH = "/yamas/v1"; + static final String VALUES_PATH = V1_PATH + "/values"; + + private final ValuesFetcher valuesFetcher; + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor); + valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); + + Path path = new Path(request.getUri()); + + if (path.matches(V1_PATH)) return v1Response(request.getUri()); + if (path.matches(VALUES_PATH)) return valuesResponse(request); + + return new ErrorResponse(NOT_FOUND, "No content at given path"); + } + + private JsonResponse v1Response(URI requestUri) { + try { + return new JsonResponse(OK, v1Content(requestUri)); + } catch (JSONException e) { + log.log(WARNING, "Bad JSON construction in " + V1_PATH + " response", e); + return new ErrorResponse(INTERNAL_SERVER_ERROR, "An error occurred, please try path '" + VALUES_PATH + "'"); + } + } + + private JsonResponse valuesResponse(HttpRequest request) { + try { + List metrics = valuesFetcher.fetch(request.getProperty("consumer")); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + // TODO: Use jackson with a "Resources" class instead of JSONObject + private String v1Content(URI requestUri) throws JSONException { + int port = requestUri.getPort(); + String host = requestUri.getHost(); + StringBuilder base = new StringBuilder("http://"); + base.append(host); + if (port >= 0) { + base.append(":").append(port); + } + String uriBase = base.toString(); + JSONArray linkList = new JSONArray(); + for (String api : new String[] {VALUES_PATH}) { + JSONObject resource = new JSONObject(); + resource.put("url", uriBase + api); + linkList.put(resource); + } + return new JSONObject().put("resources", linkList).toString(4); + } +} \ No newline at end of file diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java new file mode 100644 index 00000000000..8752032b643 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java @@ -0,0 +1,112 @@ +package ai.vespa.metricsproxy.gatherer; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.HealthMetric; +import ai.vespa.metricsproxy.metric.Metric; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; +import ai.vespa.metricsproxy.service.VespaService; +import ai.vespa.metricsproxy.service.VespaServices; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author olaa + */ +public class NodeMetricGathererTest { + + @Test + public void gatherMetrics() throws Exception { + + MetricsManager metricsManager = mock(MetricsManager.class); + VespaServices vespaServices = mock(VespaServices.class); + ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); + NodeDimensions nodeDimensions = mock(NodeDimensions.class); + NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); + List mockedVespaServices = mockedVespaServices(); + + NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); + when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); + when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); + when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); + when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); + when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + + List packets = nodeMetricGatherer.gatherMetrics(); + assertEquals(5, packets.size()); + Map serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); + List expectedPackets = List.of( + metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), + metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), + metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) + ); + + assertEqualMetricPackets(expectedPackets, packets); + } + + private void assertEqualMetricPackets(List expectedPackets, List actualPackets) { + assertEquals(expectedPackets.size(), actualPackets.size()); + expectedPackets.stream() + .forEach(expectedPacket -> + actualPackets.stream() + .filter(packet -> packet.service.equals(expectedPacket.service)) + .forEach(actualPacket -> { + assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); + assertEquals(expectedPacket.statusCode, actualPacket.statusCode); + assertEquals(expectedPacket.metrics(), actualPacket.metrics()); + assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); + }) + ); + } + + private List mockedVespaServices() { + + HealthMetric healthy = HealthMetric.get("OK", ""); + HealthMetric unhealthy = HealthMetric.get("down", ""); + + VespaService service1 = mock(VespaService.class); + when(service1.getInstanceName()).thenReturn("instance"); + when(service1.getMonitoringName()).thenReturn("service1"); + when(service1.getHealth()).thenReturn(healthy); + + VespaService service2 = mock(VespaService.class); + when(service2.getInstanceName()).thenReturn("instance"); + when(service2.getMonitoringName()).thenReturn("service2"); + when(service2.getHealth()).thenReturn(healthy); + + + VespaService service3 = mock(VespaService.class); + when(service3.getInstanceName()).thenReturn("instance"); + when(service3.getMonitoringName()).thenReturn("service3"); + when(service3.getHealth()).thenReturn(unhealthy); + + return List.of(service1, service2, service3); + + } + + private MetricsPacket metricsPacket(int statusCode, String service, + Collection metrics, Map dimensions) { + return new MetricsPacket.Builder(ServiceId.toServiceId(service)) + .putDimensions(dimensions) + .putMetrics(metrics) + .statusCode(statusCode) + .build(); + } +} \ No newline at end of file -- cgit v1.2.3 From 94174f2b0d660d183041f22732cadd20b0655720 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 30 Aug 2019 12:10:12 +0200 Subject: Added handler to metrics proxy container cluster --- .../metricsproxy/MetricsProxyContainerCluster.java | 2 + .../metricsproxy/http/Yamas/YamasHandler.java | 82 +++++++++++++++ .../ai/vespa/metricsproxy/http/YamasHandler.java | 110 --------------------- 3 files changed, 84 insertions(+), 110 deletions(-) create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 94ae5c07d8b..4264c40c556 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -10,6 +10,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.MetricsHandler; +import ai.vespa.metricsproxy.http.Yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; @@ -112,6 +113,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster clazz, String bindingPath) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java new file mode 100644 index 00000000000..e29cf202bd9 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java @@ -0,0 +1,82 @@ +/* +* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +*/ + +package ai.vespa.metricsproxy.http.Yamas; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer; +import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.JsonResponse; +import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.Path; + +import java.util.List; +import java.util.concurrent.Executor; + +import static ai.vespa.metricsproxy.http.RestApiUtil.resourceListResponse; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * @author olaa + */ + + +public class YamasHandler extends ThreadedHttpRequestHandler { + + public static final String V1_PATH = "/yamas/v1"; + private static final String VALUES_PATH = V1_PATH + "/values"; + + private final ValuesFetcher valuesFetcher; + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor); + this.valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); + + Path path = new Path(request.getUri()); + + if (path.matches(V1_PATH)) return resourceListResponse(request.getUri(), List.of(VALUES_PATH)); + if (path.matches(VALUES_PATH)) return valuesResponse(request); + + return new ErrorResponse(NOT_FOUND, "No content at given path"); + } + + private JsonResponse valuesResponse(HttpRequest request) { + try { + List metrics = valuesFetcher.fetch(request.getProperty("consumer")); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + +} \ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java deleted file mode 100644 index fca1b66342f..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java +++ /dev/null @@ -1,110 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -*/ - -package ai.vespa.metricsproxy.http; - -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.Path; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.net.URI; -import java.util.List; -import java.util.concurrent.Executor; - -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; -import static com.yahoo.jdisc.Response.Status.OK; -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static java.util.logging.Level.WARNING; - -/** - * @author olaa - */ - -// TODO: Merge with gjoranv's API util changes -public class YamasHandler extends ThreadedHttpRequestHandler { - - static final String V1_PATH = "/yamas/v1"; - static final String VALUES_PATH = V1_PATH + "/values"; - - private final ValuesFetcher valuesFetcher; - private final NodeMetricGatherer nodeMetricGatherer; - - @Inject - public YamasHandler(Executor executor, - MetricsManager metricsManager, - VespaServices vespaServices, - MetricsConsumers metricsConsumers, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions) { - super(executor); - valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); - this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); - - Path path = new Path(request.getUri()); - - if (path.matches(V1_PATH)) return v1Response(request.getUri()); - if (path.matches(VALUES_PATH)) return valuesResponse(request); - - return new ErrorResponse(NOT_FOUND, "No content at given path"); - } - - private JsonResponse v1Response(URI requestUri) { - try { - return new JsonResponse(OK, v1Content(requestUri)); - } catch (JSONException e) { - log.log(WARNING, "Bad JSON construction in " + V1_PATH + " response", e); - return new ErrorResponse(INTERNAL_SERVER_ERROR, "An error occurred, please try path '" + VALUES_PATH + "'"); - } - } - - private JsonResponse valuesResponse(HttpRequest request) { - try { - List metrics = valuesFetcher.fetch(request.getProperty("consumer")); - metrics.addAll(nodeMetricGatherer.gatherMetrics()); - return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); - } catch (JsonRenderingException e) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); - } - } - - // TODO: Use jackson with a "Resources" class instead of JSONObject - private String v1Content(URI requestUri) throws JSONException { - int port = requestUri.getPort(); - String host = requestUri.getHost(); - StringBuilder base = new StringBuilder("http://"); - base.append(host); - if (port >= 0) { - base.append(":").append(port); - } - String uriBase = base.toString(); - JSONArray linkList = new JSONArray(); - for (String api : new String[] {VALUES_PATH}) { - JSONObject resource = new JSONObject(); - resource.put("url", uriBase + api); - linkList.put(resource); - } - return new JSONObject().put("resources", linkList).toString(4); - } -} \ No newline at end of file -- cgit v1.2.3 From db285e6c2f388f76be495b65cb5cbe09ef4725e6 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 30 Aug 2019 12:55:34 +0200 Subject: YamasHandler uses HttpHandlerBase --- .../MetricsProxyContainerClusterTest.java | 2 + .../metricsproxy/gatherer/NodeMetricGatherer.java | 204 -------------------- .../ai/vespa/metricsproxy/http/ValuesFetcher.java | 2 - .../metricsproxy/http/Yamas/YamasHandler.java | 41 ++-- .../metricsproxy/service/NodeMetricGatherer.java | 210 +++++++++++++++++++++ .../gatherer/NodeMetricGathererTest.java | 112 ----------- .../service/NodeMetricGathererTest.java | 111 +++++++++++ 7 files changed, 336 insertions(+), 346 deletions(-) delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index db29dbba306..3c2b20c069b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -6,6 +6,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.http.MetricsHandler; +import ai.vespa.metricsproxy.http.Yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; import com.yahoo.component.ComponentSpecification; @@ -107,6 +108,7 @@ public class MetricsProxyContainerClusterTest { assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(MetricsHandler.class.getName()))); assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(PrometheusHandler.class.getName()))); + assertThat(handlerClasses, hasItem(ComponentSpecification.fromString(YamasHandler.class.getName()))); } @Test diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java deleted file mode 100644 index c6dfdf8fe9c..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.gatherer; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.StatusCode; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; -import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author olaa - */ -public class NodeMetricGatherer { - - private static final int COREDUMP_AGE_IN_MINUTES = 12600; - private static final JSONObject ROUTING_JSON = createRoutingJSON(); - private final VespaServices vespaServices; - private final ApplicationDimensions applicationDimensions; - private final NodeDimensions nodeDimensions; - private final MetricsManager metricsManager; - private final FileWrapper fileWrapper; - - private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); - - @Inject - public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); - } - - public NodeMetricGatherer(MetricsManager metricsManager, - VespaServices vespaServices, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions, - FileWrapper fileWrapper) { - this.metricsManager = metricsManager; - this.vespaServices = vespaServices; - this.applicationDimensions = applicationDimensions; - this.nodeDimensions = nodeDimensions; - this.fileWrapper = fileWrapper; - } - - public List gatherMetrics() { - List metricPacketBuilders = new ArrayList<>(); - metricPacketBuilders.addAll(coredumpMetrics()); - metricPacketBuilders.addAll(serviceHealthMetrics()); - metricPacketBuilders.addAll(hostLifeMetrics()); - - List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { - metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); - return metricPacketBuilder.build(); - }).collect(Collectors.toList()); - return metricPackets; - } - - private List coredumpMetrics() { - - Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); - long coredumps = getCoredumpsFromLastPeriod(crashPath); - - try { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("application", "Vespa.node"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "system-coredumps-processing"); - jsonObject.put("status_code", coredumps); - jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - } - - private List serviceHealthMetrics() { - return vespaServices.getVespaServices() - .stream() - .map(service -> { - try { - StatusCode healthStatus = service.getHealth().getStatus(); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("status_code", healthStatus.code); - jsonObject.put("status_message", healthStatus.status); - jsonObject.put("application", service.getMonitoringName()); - JSONObject dimensions = new JSONObject(); - dimensions.put("instance", service.getInstanceName()); - dimensions.put("metrictype", "health"); - jsonObject.put("dimensions", dimensions); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); - } catch (JSONException e) { - throw new RuntimeException(e.getMessage()); - } - }) - .collect(Collectors.toList()); - } - - private List hostLifeMetrics() { - JSONObject jsonObject = new JSONObject(); - double upTime; - int statusCode = 0; - String statusMessage = "OK"; - try { - upTime = getHostLife(Path.of("/proc/uptime")); // ?? - } catch (IOException e) { - upTime = 0d; - statusCode = 1; - statusMessage = e.getMessage(); - } - - try { - jsonObject.put("application", "host_life"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("status_message", statusMessage); - jsonObject.put("status_code", statusCode); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - - - } - - private long getCoredumpsFromLastPeriod(Path coreDumpPath) { - try { - return fileWrapper.walkTree(coreDumpPath) - .filter(Files::isRegularFile) - .filter(this::isNewFile) - .count(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private double getHostLife(Path uptimePath) throws IOException { - return fileWrapper.readAllLines(uptimePath) - .stream() - .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) - .findFirst() - .orElseThrow(); - } - - private boolean isNewFile(Path file) { - try { - return fileWrapper.getLastModifiedTime(file) - .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) - .isBefore(Instant.now()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static JSONObject createRoutingJSON() { - try { - JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); - return jsonObject; - } catch (JSONException e) { - throw new RuntimeException(e); - } - } - - static class FileWrapper { - - List readAllLines(Path path) throws IOException { - return Files.readAllLines(path); - } - - Stream walkTree(Path path) throws IOException { - return Files.walk(path); - } - - Instant getLastModifiedTime(Path path) throws IOException { - return Files.getLastModifiedTime(path).toInstant(); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 34fedfcfa7a..830e2201966 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -9,7 +9,6 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaServices; import java.time.Instant; @@ -18,7 +17,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericJsonModel; /** * Generates metrics values in json format for the metrics/v1 rest api. diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java index e29cf202bd9..3c22a2e1b3f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java @@ -1,15 +1,12 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -*/ - +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.Yamas; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer; +import ai.vespa.metricsproxy.service.NodeMetricGatherer; import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.HttpHandlerBase; import ai.vespa.metricsproxy.http.JsonResponse; -import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; @@ -17,32 +14,26 @@ import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.Path; +import java.net.URI; import java.util.List; +import java.util.Optional; import java.util.concurrent.Executor; -import static ai.vespa.metricsproxy.http.RestApiUtil.resourceListResponse; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; import static com.yahoo.jdisc.Response.Status.OK; -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; /** * @author olaa */ - -public class YamasHandler extends ThreadedHttpRequestHandler { +public class YamasHandler extends HttpHandlerBase { public static final String V1_PATH = "/yamas/v1"; private static final String VALUES_PATH = V1_PATH + "/values"; - private final ValuesFetcher valuesFetcher; private final NodeMetricGatherer nodeMetricGatherer; @Inject @@ -52,26 +43,20 @@ public class YamasHandler extends ThreadedHttpRequestHandler { MetricsConsumers metricsConsumers, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - super(executor); - this.valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + super(executor, metricsManager, vespaServices, metricsConsumers); this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); } @Override - public HttpResponse handle(HttpRequest request) { - if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); - - Path path = new Path(request.getUri()); - - if (path.matches(V1_PATH)) return resourceListResponse(request.getUri(), List.of(VALUES_PATH)); - if (path.matches(VALUES_PATH)) return valuesResponse(request); - - return new ErrorResponse(NOT_FOUND, "No content at given path"); + public Optional doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); } - private JsonResponse valuesResponse(HttpRequest request) { + private JsonResponse valuesResponse(String consumer) { try { - List metrics = valuesFetcher.fetch(request.getProperty("consumer")); + List metrics = valuesFetcher.fetch(consumer); metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); } catch (JsonRenderingException e) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java new file mode 100644 index 00000000000..959b57c910f --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java @@ -0,0 +1,210 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.service; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import com.google.inject.Inject; +import com.yahoo.vespa.defaults.Defaults; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Fetches miscellaneous system metrics for node, including + * - Current coredump processing + * - Health of Vespa services + * - Host life + * + * @author olaa + */ + + +public class NodeMetricGatherer { + + private static final int COREDUMP_AGE_IN_MINUTES = 12600; + private static final JSONObject ROUTING_JSON = createRoutingJSON(); + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final MetricsManager metricsManager; + private final FileWrapper fileWrapper; + + private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); + + @Inject + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); + } + + public NodeMetricGatherer(MetricsManager metricsManager, + VespaServices vespaServices, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions, + FileWrapper fileWrapper) { + this.metricsManager = metricsManager; + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + this.fileWrapper = fileWrapper; + } + + public List gatherMetrics() { + List metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.addAll(coredumpMetrics()); + metricPacketBuilders.addAll(serviceHealthMetrics()); + metricPacketBuilders.addAll(hostLifeMetrics()); + + List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); + metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); + return metricPacketBuilder.build(); + }).collect(Collectors.toList()); + return metricPackets; + } + + private List coredumpMetrics() { + + Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); + long coredumps = getCoredumpsFromLastPeriod(crashPath); + + try { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("application", "Vespa.node"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + } + + private List serviceHealthMetrics() { + return vespaServices.getVespaServices() + .stream() + .map(service -> { + try { + StatusCode healthStatus = service.getHealth().getStatus(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status_code", healthStatus.code); + jsonObject.put("status_message", healthStatus.status); + jsonObject.put("application", service.getMonitoringName()); + JSONObject dimensions = new JSONObject(); + dimensions.put("instance", service.getInstanceName()); + dimensions.put("metrictype", "health"); + jsonObject.put("dimensions", dimensions); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); + } catch (JSONException e) { + throw new RuntimeException(e.getMessage()); + } + }) + .collect(Collectors.toList()); + } + + private List hostLifeMetrics() { + JSONObject jsonObject = new JSONObject(); + double upTime; + int statusCode = 0; + String statusMessage = "OK"; + try { + upTime = getHostLife(Path.of("/proc/uptime")); + } catch (IOException e) { + upTime = 0d; + statusCode = 1; + statusMessage = e.getMessage(); + } + + try { + jsonObject.put("application", "host_life"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + + + } + + private long getCoredumpsFromLastPeriod(Path coreDumpPath) { + try { + return fileWrapper.walkTree(coreDumpPath) + .filter(Files::isRegularFile) + .filter(this::isNewFile) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private double getHostLife(Path uptimePath) throws IOException { + return fileWrapper.readAllLines(uptimePath) + .stream() + .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) + .findFirst() + .orElseThrow(); + } + + private boolean isNewFile(Path file) { + try { + return fileWrapper.getLastModifiedTime(file) + .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) + .isBefore(Instant.now()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static JSONObject createRoutingJSON() { + try { + JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); + return jsonObject; + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + static class FileWrapper { + + List readAllLines(Path path) throws IOException { + return Files.readAllLines(path); + } + + Stream walkTree(Path path) throws IOException { + return Files.walk(path); + } + + Instant getLastModifiedTime(Path path) throws IOException { + return Files.getLastModifiedTime(path).toInstant(); + } + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java deleted file mode 100644 index 8752032b643..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java +++ /dev/null @@ -1,112 +0,0 @@ -package ai.vespa.metricsproxy.gatherer; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import org.junit.Test; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author olaa - */ -public class NodeMetricGathererTest { - - @Test - public void gatherMetrics() throws Exception { - - MetricsManager metricsManager = mock(MetricsManager.class); - VespaServices vespaServices = mock(VespaServices.class); - ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); - NodeDimensions nodeDimensions = mock(NodeDimensions.class); - NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); - List mockedVespaServices = mockedVespaServices(); - - NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); - when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); - when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); - when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); - when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); - when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - - List packets = nodeMetricGatherer.gatherMetrics(); - assertEquals(5, packets.size()); - Map serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); - List expectedPackets = List.of( - metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), - metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), - metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) - ); - - assertEqualMetricPackets(expectedPackets, packets); - } - - private void assertEqualMetricPackets(List expectedPackets, List actualPackets) { - assertEquals(expectedPackets.size(), actualPackets.size()); - expectedPackets.stream() - .forEach(expectedPacket -> - actualPackets.stream() - .filter(packet -> packet.service.equals(expectedPacket.service)) - .forEach(actualPacket -> { - assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); - assertEquals(expectedPacket.statusCode, actualPacket.statusCode); - assertEquals(expectedPacket.metrics(), actualPacket.metrics()); - assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); - }) - ); - } - - private List mockedVespaServices() { - - HealthMetric healthy = HealthMetric.get("OK", ""); - HealthMetric unhealthy = HealthMetric.get("down", ""); - - VespaService service1 = mock(VespaService.class); - when(service1.getInstanceName()).thenReturn("instance"); - when(service1.getMonitoringName()).thenReturn("service1"); - when(service1.getHealth()).thenReturn(healthy); - - VespaService service2 = mock(VespaService.class); - when(service2.getInstanceName()).thenReturn("instance"); - when(service2.getMonitoringName()).thenReturn("service2"); - when(service2.getHealth()).thenReturn(healthy); - - - VespaService service3 = mock(VespaService.class); - when(service3.getInstanceName()).thenReturn("instance"); - when(service3.getMonitoringName()).thenReturn("service3"); - when(service3.getHealth()).thenReturn(unhealthy); - - return List.of(service1, service2, service3); - - } - - private MetricsPacket metricsPacket(int statusCode, String service, - Collection metrics, Map dimensions) { - return new MetricsPacket.Builder(ServiceId.toServiceId(service)) - .putDimensions(dimensions) - .putMetrics(metrics) - .statusCode(statusCode) - .build(); - } -} \ No newline at end of file diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java new file mode 100644 index 00000000000..59a207bef0d --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java @@ -0,0 +1,111 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.service; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.HealthMetric; +import ai.vespa.metricsproxy.metric.Metric; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author olaa + */ +public class NodeMetricGathererTest { + + @Test + public void gatherMetrics() throws Exception { + + MetricsManager metricsManager = mock(MetricsManager.class); + VespaServices vespaServices = mock(VespaServices.class); + ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); + NodeDimensions nodeDimensions = mock(NodeDimensions.class); + NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); + List mockedVespaServices = mockedVespaServices(); + + NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); + when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); + when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); + when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); + when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); + when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); + + List packets = nodeMetricGatherer.gatherMetrics(); + assertEquals(5, packets.size()); + Map serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); + List expectedPackets = List.of( + metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), + metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), + metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), + metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) + ); + + assertEqualMetricPackets(expectedPackets, packets); + } + + private void assertEqualMetricPackets(List expectedPackets, List actualPackets) { + assertEquals(expectedPackets.size(), actualPackets.size()); + expectedPackets.stream() + .forEach(expectedPacket -> + actualPackets.stream() + .filter(packet -> packet.service.equals(expectedPacket.service)) + .forEach(actualPacket -> { + assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); + assertEquals(expectedPacket.statusCode, actualPacket.statusCode); + assertEquals(expectedPacket.metrics(), actualPacket.metrics()); + assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); + }) + ); + } + + private List mockedVespaServices() { + + HealthMetric healthy = HealthMetric.get("OK", ""); + HealthMetric unhealthy = HealthMetric.get("down", ""); + + VespaService service1 = mock(VespaService.class); + when(service1.getInstanceName()).thenReturn("instance"); + when(service1.getMonitoringName()).thenReturn("service1"); + when(service1.getHealth()).thenReturn(healthy); + + VespaService service2 = mock(VespaService.class); + when(service2.getInstanceName()).thenReturn("instance"); + when(service2.getMonitoringName()).thenReturn("service2"); + when(service2.getHealth()).thenReturn(healthy); + + + VespaService service3 = mock(VespaService.class); + when(service3.getInstanceName()).thenReturn("instance"); + when(service3.getMonitoringName()).thenReturn("service3"); + when(service3.getHealth()).thenReturn(unhealthy); + + return List.of(service1, service2, service3); + + } + + private MetricsPacket metricsPacket(int statusCode, String service, + Collection metrics, Map dimensions) { + return new MetricsPacket.Builder(ServiceId.toServiceId(service)) + .putDimensions(dimensions) + .putMetrics(metrics) + .statusCode(statusCode) + .build(); + } +} \ No newline at end of file -- cgit v1.2.3 From b133086a1851fe15d61fc5d77177b2e4950ad041 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 30 Aug 2019 18:24:51 +0200 Subject: Separated into several classes and moved to own package. Removed use of mocking library. Other misc fixes --- .../metricsproxy/MetricsProxyContainerCluster.java | 3 +- .../MetricsProxyContainerClusterTest.java | 2 +- metrics-proxy/pom.xml | 5 - .../metricsproxy/http/Yamas/YamasHandler.java | 67 ------- .../metricsproxy/http/yamas/YamasHandler.java | 67 +++++++ .../metricsproxy/node/CoredumpMetricGatherer.java | 69 +++++++ .../ai/vespa/metricsproxy/node/FileWrapper.java | 31 +++ .../vespa/metricsproxy/node/HostLifeGatherer.java | 69 +++++++ .../metricsproxy/node/NodeMetricGatherer.java | 71 +++++++ .../metricsproxy/node/ServiceHealthGatherer.java | 44 +++++ .../metricsproxy/service/NodeMetricGatherer.java | 210 --------------------- .../metricsproxy/node/CoredumpGathererTest.java | 54 ++++++ .../metricsproxy/node/HostLifeGathererTest.java | 44 +++++ .../service/NodeMetricGathererTest.java | 111 ----------- 14 files changed, 451 insertions(+), 396 deletions(-) delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 4264c40c556..94dbdafe980 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -10,12 +10,11 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.MetricsHandler; -import ai.vespa.metricsproxy.http.Yamas.YamasHandler; +import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 3c2b20c069b..be6f84c2a28 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -6,7 +6,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.http.MetricsHandler; -import ai.vespa.metricsproxy.http.Yamas.YamasHandler; +import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; import com.yahoo.component.ComponentSpecification; diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index ff314147090..99354ee22e8 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -149,11 +149,6 @@ hamcrest-core test - - org.mockito - mockito-core - test - diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java deleted file mode 100644 index 3c22a2e1b3f..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http.Yamas; - -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.service.NodeMetricGatherer; -import ai.vespa.metricsproxy.http.ErrorResponse; -import ai.vespa.metricsproxy.http.HttpHandlerBase; -import ai.vespa.metricsproxy.http.JsonResponse; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.Path; - -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; - -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.OK; - -/** - * @author olaa - */ - -public class YamasHandler extends HttpHandlerBase { - - public static final String V1_PATH = "/yamas/v1"; - private static final String VALUES_PATH = V1_PATH + "/values"; - - private final NodeMetricGatherer nodeMetricGatherer; - - @Inject - public YamasHandler(Executor executor, - MetricsManager metricsManager, - VespaServices vespaServices, - MetricsConsumers metricsConsumers, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions) { - super(executor, metricsManager, vespaServices, metricsConsumers); - this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); - } - - @Override - public Optional doHandle(URI requestUri, Path apiPath, String consumer) { - if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); - if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); - return Optional.empty(); - } - - private JsonResponse valuesResponse(String consumer) { - try { - List metrics = valuesFetcher.fetch(consumer); - metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers - return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); - } catch (JsonRenderingException e) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); - } - } - -} \ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java new file mode 100644 index 00000000000..7a868d0bffa --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -0,0 +1,67 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.yamas; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.node.NodeMetricGatherer; +import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.HttpHandlerBase; +import ai.vespa.metricsproxy.http.JsonResponse; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author olaa + */ + +public class YamasHandler extends HttpHandlerBase { + + public static final String V1_PATH = "/yamas/v1"; + private static final String VALUES_PATH = V1_PATH + "/values"; + + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor, metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public Optional doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List metrics = valuesFetcher.fetch(consumer); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + +} \ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java new file mode 100644 index 00000000000..4b959137625 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java @@ -0,0 +1,69 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import com.yahoo.vespa.defaults.Defaults; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class CoredumpMetricGatherer { + + private static final int COREDUMP_AGE_IN_MINUTES = 12600; + private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); + + private static final Logger logger = Logger.getLogger(CoredumpMetricGatherer.class.getSimpleName()); + + + protected static List gatherCoredumpMetrics(FileWrapper fileWrapper) { + long coredumps = getCoredumpsFromLastPeriod(fileWrapper); + try { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + } + + private static long getCoredumpsFromLastPeriod(FileWrapper fileWrapper) { + try { + return fileWrapper.walkTree(COREDUMP_PATH) + .filter(file -> fileWrapper.isRegularFile(file)) + .filter(file -> isNewFile(fileWrapper, file)) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static boolean isNewFile(FileWrapper filewrapper, Path file) { + try { + return filewrapper.getLastModifiedTime(file) + .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) + .isAfter(Instant.now()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java new file mode 100644 index 00000000000..d20a44e09bb --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java @@ -0,0 +1,31 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class FileWrapper { + + List readAllLines(Path path) throws IOException { + return Files.readAllLines(path); + } + + Stream walkTree(Path path) throws IOException { + return Files.walk(path); + } + + Instant getLastModifiedTime(Path path) throws IOException { + return Files.getLastModifiedTime(path).toInstant(); + } + + boolean isRegularFile(Path path) { + return Files.isRegularFile(path); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java new file mode 100644 index 00000000000..f5a53ddab50 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java @@ -0,0 +1,69 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class HostLifeGatherer { + + private static final Path UPTIME_PATH = Path.of("/proc/uptime"); + + private static final Logger logger = Logger.getLogger(HostLifeGatherer.class.getSimpleName()); + + protected static List gatherHostLifeMetrics(FileWrapper fileWrapper) { + JSONObject jsonObject = new JSONObject(); + double upTime; + int statusCode = 0; + String statusMessage = "OK"; + try { + upTime = getHostLife(fileWrapper); + } catch (IOException e) { + upTime = 0d; + statusCode = 1; + statusMessage = e.getMessage(); + } + + try { + jsonObject.put("application", "host_life"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + + + } + + + + private static double getHostLife(FileWrapper fileWrapper) throws IOException { + return fileWrapper.readAllLines(UPTIME_PATH) + .stream() + .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) + .findFirst() + .orElseThrow(); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java new file mode 100644 index 00000000000..acddfd13c19 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -0,0 +1,71 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static ai.vespa.metricsproxy.node.CoredumpMetricGatherer.gatherCoredumpMetrics; +import static ai.vespa.metricsproxy.node.HostLifeGatherer.gatherHostLifeMetrics; +import static ai.vespa.metricsproxy.node.ServiceHealthGatherer.gatherServiceHealthMetrics; + +/** + * Fetches miscellaneous system metrics for node, including + * - Current coredump processing + * - Health of Vespa services + * - Host life + * + * @author olaa + */ + + +public class NodeMetricGatherer { + + protected static final JSONObject ROUTING_JSON = createRoutingJSON(); + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final MetricsManager metricsManager; + + @Inject + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this.metricsManager = metricsManager; + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + } + + public List gatherMetrics() { + FileWrapper fileWrapper = new FileWrapper(); + List metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.addAll(gatherCoredumpMetrics(fileWrapper)); + metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); + metricPacketBuilders.addAll(gatherHostLifeMetrics(fileWrapper)); + + return metricPacketBuilders.stream() + .map(metricPacketBuilder -> + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()) + .putDimensionsIfAbsent(nodeDimensions.getDimensions()) + .putDimensionsIfAbsent(metricsManager.getExtraDimensions()).build() + ).collect(Collectors.toList()); + } + + private static JSONObject createRoutingJSON() { + try { + JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); + return jsonObject; + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java new file mode 100644 index 00000000000..b7e441d21f7 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java @@ -0,0 +1,44 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.List; +import java.util.stream.Collectors; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class ServiceHealthGatherer { + + + protected static List gatherServiceHealthMetrics(VespaServices vespaServices) { + return vespaServices.getVespaServices() + .stream() + .map(service -> { + try { + StatusCode healthStatus = service.getHealth().getStatus(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status_code", healthStatus.code); + jsonObject.put("status_message", healthStatus.status); + jsonObject.put("application", service.getMonitoringName()); + JSONObject dimensions = new JSONObject(); + dimensions.put("instance", service.getInstanceName()); + dimensions.put("metrictype", "health"); + jsonObject.put("dimensions", dimensions); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); + } catch (JSONException e) { + throw new RuntimeException(e.getMessage()); + } + }) + .collect(Collectors.toList()); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java deleted file mode 100644 index 959b57c910f..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.StatusCode; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import com.google.inject.Inject; -import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Fetches miscellaneous system metrics for node, including - * - Current coredump processing - * - Health of Vespa services - * - Host life - * - * @author olaa - */ - - -public class NodeMetricGatherer { - - private static final int COREDUMP_AGE_IN_MINUTES = 12600; - private static final JSONObject ROUTING_JSON = createRoutingJSON(); - private final VespaServices vespaServices; - private final ApplicationDimensions applicationDimensions; - private final NodeDimensions nodeDimensions; - private final MetricsManager metricsManager; - private final FileWrapper fileWrapper; - - private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); - - @Inject - public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); - } - - public NodeMetricGatherer(MetricsManager metricsManager, - VespaServices vespaServices, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions, - FileWrapper fileWrapper) { - this.metricsManager = metricsManager; - this.vespaServices = vespaServices; - this.applicationDimensions = applicationDimensions; - this.nodeDimensions = nodeDimensions; - this.fileWrapper = fileWrapper; - } - - public List gatherMetrics() { - List metricPacketBuilders = new ArrayList<>(); - metricPacketBuilders.addAll(coredumpMetrics()); - metricPacketBuilders.addAll(serviceHealthMetrics()); - metricPacketBuilders.addAll(hostLifeMetrics()); - - List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { - metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); - return metricPacketBuilder.build(); - }).collect(Collectors.toList()); - return metricPackets; - } - - private List coredumpMetrics() { - - Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); - long coredumps = getCoredumpsFromLastPeriod(crashPath); - - try { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("application", "Vespa.node"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "system-coredumps-processing"); - jsonObject.put("status_code", coredumps); - jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - } - - private List serviceHealthMetrics() { - return vespaServices.getVespaServices() - .stream() - .map(service -> { - try { - StatusCode healthStatus = service.getHealth().getStatus(); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("status_code", healthStatus.code); - jsonObject.put("status_message", healthStatus.status); - jsonObject.put("application", service.getMonitoringName()); - JSONObject dimensions = new JSONObject(); - dimensions.put("instance", service.getInstanceName()); - dimensions.put("metrictype", "health"); - jsonObject.put("dimensions", dimensions); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); - } catch (JSONException e) { - throw new RuntimeException(e.getMessage()); - } - }) - .collect(Collectors.toList()); - } - - private List hostLifeMetrics() { - JSONObject jsonObject = new JSONObject(); - double upTime; - int statusCode = 0; - String statusMessage = "OK"; - try { - upTime = getHostLife(Path.of("/proc/uptime")); - } catch (IOException e) { - upTime = 0d; - statusCode = 1; - statusMessage = e.getMessage(); - } - - try { - jsonObject.put("application", "host_life"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("status_message", statusMessage); - jsonObject.put("status_code", statusCode); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - - - } - - private long getCoredumpsFromLastPeriod(Path coreDumpPath) { - try { - return fileWrapper.walkTree(coreDumpPath) - .filter(Files::isRegularFile) - .filter(this::isNewFile) - .count(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private double getHostLife(Path uptimePath) throws IOException { - return fileWrapper.readAllLines(uptimePath) - .stream() - .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) - .findFirst() - .orElseThrow(); - } - - private boolean isNewFile(Path file) { - try { - return fileWrapper.getLastModifiedTime(file) - .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) - .isBefore(Instant.now()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static JSONObject createRoutingJSON() { - try { - JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); - return jsonObject; - } catch (JSONException e) { - throw new RuntimeException(e); - } - } - - static class FileWrapper { - - List readAllLines(Path path) throws IOException { - return Files.readAllLines(path); - } - - Stream walkTree(Path path) throws IOException { - return Files.walk(path); - } - - Instant getLastModifiedTime(Path path) throws IOException { - return Files.getLastModifiedTime(path).toInstant(); - } - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java new file mode 100644 index 00000000000..b4f836b78d0 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java @@ -0,0 +1,54 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class CoredumpGathererTest { + + @Test + public void finds_one_coredump() { + List actualPackets = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()) + .stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + + assertEquals(1, actualPackets.size()); + + MetricsPacket packet = actualPackets.get(0); + + assertEquals("system-coredumps-processing", packet.service.id); + assertEquals(1, packet.statusCode); + } + + static class MockFileWrapper extends FileWrapper { + + + @Override + Stream walkTree(Path path) { + return Stream.of(Path.of("dummy-path")); + } + + @Override + Instant getLastModifiedTime(Path path) { + return Instant.now(); + } + + @Override + boolean isRegularFile(Path path) { + return true; + } + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java new file mode 100644 index 00000000000..54568fad74d --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java @@ -0,0 +1,44 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class HostLifeGathererTest { + + @Test + public void host_is_alive() { + List actualPackets = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()) + .stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + + assertEquals(1, actualPackets.size()); + + MetricsPacket packet = actualPackets.get(0); + + Map expectedMetrics = Map.of(MetricId.toMetricId("uptime"), 123d, MetricId.toMetricId("alive"), 1d); + assertEquals("host_life", packet.service.id); + assertEquals(0, packet.statusCode); + assertEquals(expectedMetrics, packet.metrics()); + + } + + static class MockFileWrapper extends FileWrapper { + @Override + List readAllLines(Path path) { + return List.of("123 432"); + } + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java deleted file mode 100644 index 59a207bef0d..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; -import org.junit.Test; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author olaa - */ -public class NodeMetricGathererTest { - - @Test - public void gatherMetrics() throws Exception { - - MetricsManager metricsManager = mock(MetricsManager.class); - VespaServices vespaServices = mock(VespaServices.class); - ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); - NodeDimensions nodeDimensions = mock(NodeDimensions.class); - NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); - List mockedVespaServices = mockedVespaServices(); - - NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); - when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); - when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); - when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); - when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); - when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - - List packets = nodeMetricGatherer.gatherMetrics(); - assertEquals(5, packets.size()); - Map serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); - List expectedPackets = List.of( - metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), - metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), - metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) - ); - - assertEqualMetricPackets(expectedPackets, packets); - } - - private void assertEqualMetricPackets(List expectedPackets, List actualPackets) { - assertEquals(expectedPackets.size(), actualPackets.size()); - expectedPackets.stream() - .forEach(expectedPacket -> - actualPackets.stream() - .filter(packet -> packet.service.equals(expectedPacket.service)) - .forEach(actualPacket -> { - assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); - assertEquals(expectedPacket.statusCode, actualPacket.statusCode); - assertEquals(expectedPacket.metrics(), actualPacket.metrics()); - assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); - }) - ); - } - - private List mockedVespaServices() { - - HealthMetric healthy = HealthMetric.get("OK", ""); - HealthMetric unhealthy = HealthMetric.get("down", ""); - - VespaService service1 = mock(VespaService.class); - when(service1.getInstanceName()).thenReturn("instance"); - when(service1.getMonitoringName()).thenReturn("service1"); - when(service1.getHealth()).thenReturn(healthy); - - VespaService service2 = mock(VespaService.class); - when(service2.getInstanceName()).thenReturn("instance"); - when(service2.getMonitoringName()).thenReturn("service2"); - when(service2.getHealth()).thenReturn(healthy); - - - VespaService service3 = mock(VespaService.class); - when(service3.getInstanceName()).thenReturn("instance"); - when(service3.getMonitoringName()).thenReturn("service3"); - when(service3.getHealth()).thenReturn(unhealthy); - - return List.of(service1, service2, service3); - - } - - private MetricsPacket metricsPacket(int statusCode, String service, - Collection metrics, Map dimensions) { - return new MetricsPacket.Builder(ServiceId.toServiceId(service)) - .putDimensions(dimensions) - .putMetrics(metrics) - .statusCode(statusCode) - .build(); - } -} \ No newline at end of file -- cgit v1.2.3 From f2a04e2e15111ee07c59e803fa198c4f2c3d6805 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Sun, 1 Sep 2019 14:09:53 +0200 Subject: Use metric packet builders --- .../metricsproxy/node/CoredumpMetricGatherer.java | 49 ++++++---------------- .../ai/vespa/metricsproxy/node/FileWrapper.java | 5 --- .../vespa/metricsproxy/node/HostLifeGatherer.java | 41 +++++++----------- .../metricsproxy/node/NodeMetricGatherer.java | 16 +------ .../metricsproxy/node/ServiceHealthGatherer.java | 36 +++++++--------- .../metricsproxy/node/CoredumpGathererTest.java | 17 +------- .../metricsproxy/node/HostLifeGathererTest.java | 12 +----- 7 files changed, 46 insertions(+), 130 deletions(-) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java index 4b959137625..7691f89706b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java @@ -1,69 +1,46 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.node; +import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.metric.model.ServiceId; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Path; import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; +import java.util.Set; import java.util.logging.Logger; -import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; /** * @author olaa */ public class CoredumpMetricGatherer { - private static final int COREDUMP_AGE_IN_MINUTES = 12600; private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); private static final Logger logger = Logger.getLogger(CoredumpMetricGatherer.class.getSimpleName()); - protected static List gatherCoredumpMetrics(FileWrapper fileWrapper) { - long coredumps = getCoredumpsFromLastPeriod(fileWrapper); - try { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "system-coredumps-processing"); - jsonObject.put("status_code", coredumps); - jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } + protected static MetricsPacket.Builder gatherCoredumpMetrics(FileWrapper fileWrapper) { + int coredumps = getCoredumpsFromLastPeriod(fileWrapper); + return new MetricsPacket.Builder(ServiceId.toServiceId("system-coredumps-processing")) + .timestamp(Instant.now().getEpochSecond()) + .statusCode(coredumps) + .statusMessage(coredumps == 0 ? "OK" : String.format("Found %d coredumps", coredumps)) + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); } - private static long getCoredumpsFromLastPeriod(FileWrapper fileWrapper) { + private static int getCoredumpsFromLastPeriod(FileWrapper fileWrapper) { try { - return fileWrapper.walkTree(COREDUMP_PATH) - .filter(file -> fileWrapper.isRegularFile(file)) - .filter(file -> isNewFile(fileWrapper, file)) + return (int) fileWrapper.walkTree(COREDUMP_PATH) + .filter(fileWrapper::isRegularFile) .count(); } catch (IOException e) { throw new UncheckedIOException(e); } } - private static boolean isNewFile(FileWrapper filewrapper, Path file) { - try { - return filewrapper.getLastModifiedTime(file) - .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) - .isAfter(Instant.now()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java index d20a44e09bb..b21f2a004b9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java @@ -4,7 +4,6 @@ package ai.vespa.metricsproxy.node; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; import java.util.List; import java.util.stream.Stream; @@ -21,10 +20,6 @@ public class FileWrapper { return Files.walk(path); } - Instant getLastModifiedTime(Path path) throws IOException { - return Files.getLastModifiedTime(path).toInstant(); - } - boolean isRegularFile(Path path) { return Files.isRegularFile(path); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java index f5a53ddab50..d2b403e0567 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java @@ -1,20 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.node; +import ai.vespa.metricsproxy.metric.Metric; +import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.MetricId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import org.json.JSONException; -import org.json.JSONObject; +import ai.vespa.metricsproxy.metric.model.ServiceId; import java.io.IOException; import java.nio.file.Path; import java.time.Instant; -import java.util.Collections; import java.util.List; -import java.util.logging.Level; +import java.util.Set; import java.util.logging.Logger; -import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; /** * @author olaa @@ -23,13 +22,11 @@ public class HostLifeGatherer { private static final Path UPTIME_PATH = Path.of("/proc/uptime"); - private static final Logger logger = Logger.getLogger(HostLifeGatherer.class.getSimpleName()); - - protected static List gatherHostLifeMetrics(FileWrapper fileWrapper) { - JSONObject jsonObject = new JSONObject(); + protected static MetricsPacket.Builder gatherHostLifeMetrics(FileWrapper fileWrapper) { double upTime; int statusCode = 0; String statusMessage = "OK"; + try { upTime = getHostLife(fileWrapper); } catch (IOException e) { @@ -38,23 +35,13 @@ public class HostLifeGatherer { statusMessage = e.getMessage(); } - try { - jsonObject.put("application", "host_life"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("status_message", statusMessage); - jsonObject.put("status_code", statusCode); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - - + return new MetricsPacket.Builder(ServiceId.toServiceId("host_life")) + .timestamp(Instant.now().getEpochSecond()) + .statusMessage(statusMessage) + .statusCode(statusCode) + .putMetric(MetricId.toMetricId("uptime"), upTime) + .putMetric(MetricId.toMetricId("alive"), 1) + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java index acddfd13c19..b511a23113d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -7,8 +7,6 @@ import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; -import org.json.JSONException; -import org.json.JSONObject; import java.util.ArrayList; import java.util.List; @@ -30,7 +28,6 @@ import static ai.vespa.metricsproxy.node.ServiceHealthGatherer.gatherServiceHeal public class NodeMetricGatherer { - protected static final JSONObject ROUTING_JSON = createRoutingJSON(); private final VespaServices vespaServices; private final ApplicationDimensions applicationDimensions; private final NodeDimensions nodeDimensions; @@ -47,9 +44,9 @@ public class NodeMetricGatherer { public List gatherMetrics() { FileWrapper fileWrapper = new FileWrapper(); List metricPacketBuilders = new ArrayList<>(); - metricPacketBuilders.addAll(gatherCoredumpMetrics(fileWrapper)); + metricPacketBuilders.add(gatherCoredumpMetrics(fileWrapper)); metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); - metricPacketBuilders.addAll(gatherHostLifeMetrics(fileWrapper)); + metricPacketBuilders.add(gatherHostLifeMetrics(fileWrapper)); return metricPacketBuilders.stream() .map(metricPacketBuilder -> @@ -59,13 +56,4 @@ public class NodeMetricGatherer { ).collect(Collectors.toList()); } - private static JSONObject createRoutingJSON() { - try { - JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); - return jsonObject; - } catch (JSONException e) { - throw new RuntimeException(e); - } - } - } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java index b7e441d21f7..923fb4d646d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java @@ -1,17 +1,17 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.node; +import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.StatusCode; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.metric.model.ServiceId; import ai.vespa.metricsproxy.service.VespaServices; -import org.json.JSONException; -import org.json.JSONObject; +import java.time.Instant; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; /** * @author olaa @@ -22,23 +22,15 @@ public class ServiceHealthGatherer { protected static List gatherServiceHealthMetrics(VespaServices vespaServices) { return vespaServices.getVespaServices() .stream() - .map(service -> { - try { - StatusCode healthStatus = service.getHealth().getStatus(); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("status_code", healthStatus.code); - jsonObject.put("status_message", healthStatus.status); - jsonObject.put("application", service.getMonitoringName()); - JSONObject dimensions = new JSONObject(); - dimensions.put("instance", service.getInstanceName()); - dimensions.put("metrictype", "health"); - jsonObject.put("dimensions", dimensions); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); - } catch (JSONException e) { - throw new RuntimeException(e.getMessage()); - } - }) + .map(service -> + new MetricsPacket.Builder(ServiceId.toServiceId(service.getMonitoringName())) + .timestamp(Instant.now().getEpochSecond()) + .statusMessage(service.getHealth().getStatus().status) + .statusCode(service.getHealth().getStatus().code) + .putDimension(DimensionId.toDimensionId("instance"), service.getInstanceName()) + .putDimension(DimensionId.toDimensionId("metrictype"), "health") + .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))) + ) .collect(Collectors.toList()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java index b4f836b78d0..e7a905be52e 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java @@ -5,9 +5,6 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import org.junit.Test; import java.nio.file.Path; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -19,14 +16,7 @@ public class CoredumpGathererTest { @Test public void finds_one_coredump() { - List actualPackets = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()) - .stream() - .map(MetricsPacket.Builder::build) - .collect(Collectors.toList()); - - assertEquals(1, actualPackets.size()); - - MetricsPacket packet = actualPackets.get(0); + MetricsPacket packet = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()).build(); assertEquals("system-coredumps-processing", packet.service.id); assertEquals(1, packet.statusCode); @@ -40,11 +30,6 @@ public class CoredumpGathererTest { return Stream.of(Path.of("dummy-path")); } - @Override - Instant getLastModifiedTime(Path path) { - return Instant.now(); - } - @Override boolean isRegularFile(Path path) { return true; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java index 54568fad74d..783f95f2f27 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java @@ -8,7 +8,6 @@ import org.junit.Test; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -19,16 +18,9 @@ public class HostLifeGathererTest { @Test public void host_is_alive() { - List actualPackets = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()) - .stream() - .map(MetricsPacket.Builder::build) - .collect(Collectors.toList()); + MetricsPacket packet = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()).build(); - assertEquals(1, actualPackets.size()); - - MetricsPacket packet = actualPackets.get(0); - - Map expectedMetrics = Map.of(MetricId.toMetricId("uptime"), 123d, MetricId.toMetricId("alive"), 1d); + Map expectedMetrics = Map.of(MetricId.toMetricId("uptime"), 123d, MetricId.toMetricId("alive"), 1); assertEquals("host_life", packet.service.id); assertEquals(0, packet.statusCode); assertEquals(expectedMetrics, packet.metrics()); -- cgit v1.2.3 From 27d4df4482e1f0bf834590f840a883ab90da9630 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 2 Sep 2019 11:10:59 +0200 Subject: Renamed function --- .../main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java index 7691f89706b..91e7fbbc0b5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java @@ -25,7 +25,7 @@ public class CoredumpMetricGatherer { protected static MetricsPacket.Builder gatherCoredumpMetrics(FileWrapper fileWrapper) { - int coredumps = getCoredumpsFromLastPeriod(fileWrapper); + int coredumps = getNumberOfCoredumps(fileWrapper); return new MetricsPacket.Builder(ServiceId.toServiceId("system-coredumps-processing")) .timestamp(Instant.now().getEpochSecond()) .statusCode(coredumps) @@ -33,7 +33,7 @@ public class CoredumpMetricGatherer { .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); } - private static int getCoredumpsFromLastPeriod(FileWrapper fileWrapper) { + private static int getNumberOfCoredumps(FileWrapper fileWrapper) { try { return (int) fileWrapper.walkTree(COREDUMP_PATH) .filter(fileWrapper::isRegularFile) -- cgit v1.2.3 From e21eed87ff09cd6a5b1a848568786db648f35e08 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 2 Sep 2019 16:02:18 +0200 Subject: Only check host life on linux --- .../main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java | 6 +++++- .../java/ai/vespa/metricsproxy/service/SystemPollerProvider.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java index b511a23113d..cd71bb781fe 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -5,6 +5,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.service.SystemPollerProvider; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; @@ -46,7 +47,10 @@ public class NodeMetricGatherer { List metricPacketBuilders = new ArrayList<>(); metricPacketBuilders.add(gatherCoredumpMetrics(fileWrapper)); metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); - metricPacketBuilders.add(gatherHostLifeMetrics(fileWrapper)); + + if (SystemPollerProvider.runningOnLinux()) { + metricPacketBuilders.add(gatherHostLifeMetrics(fileWrapper)); + } return metricPacketBuilders.stream() .map(metricPacketBuilder -> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java index ea47933ddf0..6494054c278 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java @@ -38,7 +38,7 @@ public class SystemPollerProvider implements Provider { return poller; } - private static boolean runningOnLinux() { + public static boolean runningOnLinux() { return getOs().contains("nux"); } -- cgit v1.2.3 From d84dddd3d6544c5f0f640a908e673cf90ce3667b Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 5 Sep 2019 10:49:23 +0200 Subject: Read /proc for uptime --- .../java/ai/vespa/metricsproxy/node/FileWrapper.java | 6 ++++-- .../ai/vespa/metricsproxy/node/HostLifeGatherer.java | 20 ++++---------------- .../metricsproxy/node/HostLifeGathererTest.java | 9 +++++---- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java index b21f2a004b9..aa01802883e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java @@ -4,6 +4,7 @@ package ai.vespa.metricsproxy.node; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.stream.Stream; @@ -12,8 +13,9 @@ import java.util.stream.Stream; */ public class FileWrapper { - List readAllLines(Path path) throws IOException { - return Files.readAllLines(path); + long getFileAgeInSeconds(Path path) throws IOException { + Instant lastModifiedTime = Files.getLastModifiedTime(path).toInstant(); + return Instant.now().getEpochSecond() - lastModifiedTime.getEpochSecond(); } Stream walkTree(Path path) throws IOException { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java index d2b403e0567..8fdfc93022c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.node; -import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; @@ -10,9 +9,7 @@ import ai.vespa.metricsproxy.metric.model.ServiceId; import java.io.IOException; import java.nio.file.Path; import java.time.Instant; -import java.util.List; import java.util.Set; -import java.util.logging.Logger; /** @@ -20,17 +17,17 @@ import java.util.logging.Logger; */ public class HostLifeGatherer { - private static final Path UPTIME_PATH = Path.of("/proc/uptime"); + private static final Path UPTIME_PATH = Path.of("/proc"); protected static MetricsPacket.Builder gatherHostLifeMetrics(FileWrapper fileWrapper) { - double upTime; + long upTime; int statusCode = 0; String statusMessage = "OK"; try { - upTime = getHostLife(fileWrapper); + upTime = fileWrapper.getFileAgeInSeconds(UPTIME_PATH); } catch (IOException e) { - upTime = 0d; + upTime = 0; statusCode = 1; statusMessage = e.getMessage(); } @@ -44,13 +41,4 @@ public class HostLifeGatherer { .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa"))); } - - - private static double getHostLife(FileWrapper fileWrapper) throws IOException { - return fileWrapper.readAllLines(UPTIME_PATH) - .stream() - .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) - .findFirst() - .orElseThrow(); - } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java index 783f95f2f27..8317229b73a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java @@ -6,6 +6,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import org.junit.Test; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Map; @@ -20,17 +21,17 @@ public class HostLifeGathererTest { public void host_is_alive() { MetricsPacket packet = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()).build(); - Map expectedMetrics = Map.of(MetricId.toMetricId("uptime"), 123d, MetricId.toMetricId("alive"), 1); assertEquals("host_life", packet.service.id); assertEquals(0, packet.statusCode); - assertEquals(expectedMetrics, packet.metrics()); + assertEquals(123l, packet.metrics().get(MetricId.toMetricId("uptime"))); + assertEquals(1, packet.metrics().get(MetricId.toMetricId("alive"))); } static class MockFileWrapper extends FileWrapper { @Override - List readAllLines(Path path) { - return List.of("123 432"); + long getFileAgeInSeconds(Path path) { + return 123; } } } -- cgit v1.2.3