From b133086a1851fe15d61fc5d77177b2e4950ad041 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 30 Aug 2019 18:24:51 +0200 Subject: Separated into several classes and moved to own package. Removed use of mocking library. Other misc fixes --- metrics-proxy/pom.xml | 5 - .../metricsproxy/http/Yamas/YamasHandler.java | 67 ------- .../metricsproxy/http/yamas/YamasHandler.java | 67 +++++++ .../metricsproxy/node/CoredumpMetricGatherer.java | 69 +++++++ .../ai/vespa/metricsproxy/node/FileWrapper.java | 31 +++ .../vespa/metricsproxy/node/HostLifeGatherer.java | 69 +++++++ .../metricsproxy/node/NodeMetricGatherer.java | 71 +++++++ .../metricsproxy/node/ServiceHealthGatherer.java | 44 +++++ .../metricsproxy/service/NodeMetricGatherer.java | 210 --------------------- .../metricsproxy/node/CoredumpGathererTest.java | 54 ++++++ .../metricsproxy/node/HostLifeGathererTest.java | 44 +++++ .../service/NodeMetricGathererTest.java | 111 ----------- 12 files changed, 449 insertions(+), 393 deletions(-) delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java create mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java create mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java (limited to 'metrics-proxy') diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index ff314147090..99354ee22e8 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -149,11 +149,6 @@ hamcrest-core test - - org.mockito - mockito-core - test - diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java deleted file mode 100644 index 3c22a2e1b3f..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/Yamas/YamasHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http.Yamas; - -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.service.NodeMetricGatherer; -import ai.vespa.metricsproxy.http.ErrorResponse; -import ai.vespa.metricsproxy.http.HttpHandlerBase; -import ai.vespa.metricsproxy.http.JsonResponse; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.Path; - -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; - -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.OK; - -/** - * @author olaa - */ - -public class YamasHandler extends HttpHandlerBase { - - public static final String V1_PATH = "/yamas/v1"; - private static final String VALUES_PATH = V1_PATH + "/values"; - - private final NodeMetricGatherer nodeMetricGatherer; - - @Inject - public YamasHandler(Executor executor, - MetricsManager metricsManager, - VespaServices vespaServices, - MetricsConsumers metricsConsumers, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions) { - super(executor, metricsManager, vespaServices, metricsConsumers); - this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); - } - - @Override - public Optional doHandle(URI requestUri, Path apiPath, String consumer) { - if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); - if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); - return Optional.empty(); - } - - private JsonResponse valuesResponse(String consumer) { - try { - List metrics = valuesFetcher.fetch(consumer); - metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers - return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); - } catch (JsonRenderingException e) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); - } - } - -} \ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java new file mode 100644 index 00000000000..7a868d0bffa --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -0,0 +1,67 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.yamas; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.node.NodeMetricGatherer; +import ai.vespa.metricsproxy.http.ErrorResponse; +import ai.vespa.metricsproxy.http.HttpHandlerBase; +import ai.vespa.metricsproxy.http.JsonResponse; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author olaa + */ + +public class YamasHandler extends HttpHandlerBase { + + public static final String V1_PATH = "/yamas/v1"; + private static final String VALUES_PATH = V1_PATH + "/values"; + + private final NodeMetricGatherer nodeMetricGatherer; + + @Inject + public YamasHandler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { + super(executor, metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions); + } + + @Override + public Optional doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List metrics = valuesFetcher.fetch(consumer); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers + return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize()); + } catch (JsonRenderingException e) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + +} \ No newline at end of file diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java new file mode 100644 index 00000000000..4b959137625 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java @@ -0,0 +1,69 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import com.yahoo.vespa.defaults.Defaults; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class CoredumpMetricGatherer { + + private static final int COREDUMP_AGE_IN_MINUTES = 12600; + private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); + + private static final Logger logger = Logger.getLogger(CoredumpMetricGatherer.class.getSimpleName()); + + + protected static List gatherCoredumpMetrics(FileWrapper fileWrapper) { + long coredumps = getCoredumpsFromLastPeriod(fileWrapper); + try { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "system-coredumps-processing"); + jsonObject.put("status_code", coredumps); + jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + } + + private static long getCoredumpsFromLastPeriod(FileWrapper fileWrapper) { + try { + return fileWrapper.walkTree(COREDUMP_PATH) + .filter(file -> fileWrapper.isRegularFile(file)) + .filter(file -> isNewFile(fileWrapper, file)) + .count(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static boolean isNewFile(FileWrapper filewrapper, Path file) { + try { + return filewrapper.getLastModifiedTime(file) + .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) + .isAfter(Instant.now()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java new file mode 100644 index 00000000000..d20a44e09bb --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java @@ -0,0 +1,31 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class FileWrapper { + + List readAllLines(Path path) throws IOException { + return Files.readAllLines(path); + } + + Stream walkTree(Path path) throws IOException { + return Files.walk(path); + } + + Instant getLastModifiedTime(Path path) throws IOException { + return Files.getLastModifiedTime(path).toInstant(); + } + + boolean isRegularFile(Path path) { + return Files.isRegularFile(path); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java new file mode 100644 index 00000000000..f5a53ddab50 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java @@ -0,0 +1,69 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class HostLifeGatherer { + + private static final Path UPTIME_PATH = Path.of("/proc/uptime"); + + private static final Logger logger = Logger.getLogger(HostLifeGatherer.class.getSimpleName()); + + protected static List gatherHostLifeMetrics(FileWrapper fileWrapper) { + JSONObject jsonObject = new JSONObject(); + double upTime; + int statusCode = 0; + String statusMessage = "OK"; + try { + upTime = getHostLife(fileWrapper); + } catch (IOException e) { + upTime = 0d; + statusCode = 1; + statusMessage = e.getMessage(); + } + + try { + jsonObject.put("application", "host_life"); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("status_message", statusMessage); + jsonObject.put("status_code", statusCode); + JSONObject metrics = new JSONObject(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.put("metrics", metrics); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); + } catch (JSONException e) { + logger.log(Level.WARNING, "Error writing JSON", e); + return Collections.emptyList(); + } + + + } + + + + private static double getHostLife(FileWrapper fileWrapper) throws IOException { + return fileWrapper.readAllLines(UPTIME_PATH) + .stream() + .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) + .findFirst() + .orElseThrow(); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java new file mode 100644 index 00000000000..acddfd13c19 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -0,0 +1,71 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static ai.vespa.metricsproxy.node.CoredumpMetricGatherer.gatherCoredumpMetrics; +import static ai.vespa.metricsproxy.node.HostLifeGatherer.gatherHostLifeMetrics; +import static ai.vespa.metricsproxy.node.ServiceHealthGatherer.gatherServiceHealthMetrics; + +/** + * Fetches miscellaneous system metrics for node, including + * - Current coredump processing + * - Health of Vespa services + * - Host life + * + * @author olaa + */ + + +public class NodeMetricGatherer { + + protected static final JSONObject ROUTING_JSON = createRoutingJSON(); + private final VespaServices vespaServices; + private final ApplicationDimensions applicationDimensions; + private final NodeDimensions nodeDimensions; + private final MetricsManager metricsManager; + + @Inject + public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { + this.metricsManager = metricsManager; + this.vespaServices = vespaServices; + this.applicationDimensions = applicationDimensions; + this.nodeDimensions = nodeDimensions; + } + + public List gatherMetrics() { + FileWrapper fileWrapper = new FileWrapper(); + List metricPacketBuilders = new ArrayList<>(); + metricPacketBuilders.addAll(gatherCoredumpMetrics(fileWrapper)); + metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); + metricPacketBuilders.addAll(gatherHostLifeMetrics(fileWrapper)); + + return metricPacketBuilders.stream() + .map(metricPacketBuilder -> + metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()) + .putDimensionsIfAbsent(nodeDimensions.getDimensions()) + .putDimensionsIfAbsent(metricsManager.getExtraDimensions()).build() + ).collect(Collectors.toList()); + } + + private static JSONObject createRoutingJSON() { + try { + JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); + return jsonObject; + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java new file mode 100644 index 00000000000..b7e441d21f7 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java @@ -0,0 +1,44 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.StatusCode; +import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.service.VespaServices; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.List; +import java.util.stream.Collectors; + +import static ai.vespa.metricsproxy.node.NodeMetricGatherer.ROUTING_JSON; + +/** + * @author olaa + */ +public class ServiceHealthGatherer { + + + protected static List gatherServiceHealthMetrics(VespaServices vespaServices) { + return vespaServices.getVespaServices() + .stream() + .map(service -> { + try { + StatusCode healthStatus = service.getHealth().getStatus(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status_code", healthStatus.code); + jsonObject.put("status_message", healthStatus.status); + jsonObject.put("application", service.getMonitoringName()); + JSONObject dimensions = new JSONObject(); + dimensions.put("instance", service.getInstanceName()); + dimensions.put("metrictype", "health"); + jsonObject.put("dimensions", dimensions); + jsonObject.put("routing", ROUTING_JSON); + return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); + } catch (JSONException e) { + throw new RuntimeException(e.getMessage()); + } + }) + .collect(Collectors.toList()); + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java deleted file mode 100644 index 959b57c910f..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/NodeMetricGatherer.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.StatusCode; -import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; -import com.google.inject.Inject; -import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Fetches miscellaneous system metrics for node, including - * - Current coredump processing - * - Health of Vespa services - * - Host life - * - * @author olaa - */ - - -public class NodeMetricGatherer { - - private static final int COREDUMP_AGE_IN_MINUTES = 12600; - private static final JSONObject ROUTING_JSON = createRoutingJSON(); - private final VespaServices vespaServices; - private final ApplicationDimensions applicationDimensions; - private final NodeDimensions nodeDimensions; - private final MetricsManager metricsManager; - private final FileWrapper fileWrapper; - - private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName()); - - @Inject - public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper()); - } - - public NodeMetricGatherer(MetricsManager metricsManager, - VespaServices vespaServices, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions, - FileWrapper fileWrapper) { - this.metricsManager = metricsManager; - this.vespaServices = vespaServices; - this.applicationDimensions = applicationDimensions; - this.nodeDimensions = nodeDimensions; - this.fileWrapper = fileWrapper; - } - - public List gatherMetrics() { - List metricPacketBuilders = new ArrayList<>(); - metricPacketBuilders.addAll(coredumpMetrics()); - metricPacketBuilders.addAll(serviceHealthMetrics()); - metricPacketBuilders.addAll(hostLifeMetrics()); - - List metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> { - metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions()); - metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions()); - return metricPacketBuilder.build(); - }).collect(Collectors.toList()); - return metricPackets; - } - - private List coredumpMetrics() { - - Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); - long coredumps = getCoredumpsFromLastPeriod(crashPath); - - try { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("application", "Vespa.node"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "system-coredumps-processing"); - jsonObject.put("status_code", coredumps); - jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES)); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - } - - private List serviceHealthMetrics() { - return vespaServices.getVespaServices() - .stream() - .map(service -> { - try { - StatusCode healthStatus = service.getHealth().getStatus(); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("status_code", healthStatus.code); - jsonObject.put("status_message", healthStatus.status); - jsonObject.put("application", service.getMonitoringName()); - JSONObject dimensions = new JSONObject(); - dimensions.put("instance", service.getInstanceName()); - dimensions.put("metrictype", "health"); - jsonObject.put("dimensions", dimensions); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0); - } catch (JSONException e) { - throw new RuntimeException(e.getMessage()); - } - }) - .collect(Collectors.toList()); - } - - private List hostLifeMetrics() { - JSONObject jsonObject = new JSONObject(); - double upTime; - int statusCode = 0; - String statusMessage = "OK"; - try { - upTime = getHostLife(Path.of("/proc/uptime")); - } catch (IOException e) { - upTime = 0d; - statusCode = 1; - statusMessage = e.getMessage(); - } - - try { - jsonObject.put("application", "host_life"); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("status_message", statusMessage); - jsonObject.put("status_code", statusCode); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - jsonObject.put("routing", ROUTING_JSON); - return YamasJsonUtil.toMetricsPackets(jsonObject.toString()); - } catch (JSONException e) { - logger.log(Level.WARNING, "Error writing JSON", e); - return Collections.emptyList(); - } - - - } - - private long getCoredumpsFromLastPeriod(Path coreDumpPath) { - try { - return fileWrapper.walkTree(coreDumpPath) - .filter(Files::isRegularFile) - .filter(this::isNewFile) - .count(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private double getHostLife(Path uptimePath) throws IOException { - return fileWrapper.readAllLines(uptimePath) - .stream() - .mapToDouble(line -> Double.valueOf(line.split("\\s")[0])) - .findFirst() - .orElseThrow(); - } - - private boolean isNewFile(Path file) { - try { - return fileWrapper.getLastModifiedTime(file) - .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES) - .isBefore(Instant.now()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static JSONObject createRoutingJSON() { - try { - JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}"); - return jsonObject; - } catch (JSONException e) { - throw new RuntimeException(e); - } - } - - static class FileWrapper { - - List readAllLines(Path path) throws IOException { - return Files.readAllLines(path); - } - - Stream walkTree(Path path) throws IOException { - return Files.walk(path); - } - - Instant getLastModifiedTime(Path path) throws IOException { - return Files.getLastModifiedTime(path).toInstant(); - } - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java new file mode 100644 index 00000000000..b4f836b78d0 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java @@ -0,0 +1,54 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class CoredumpGathererTest { + + @Test + public void finds_one_coredump() { + List actualPackets = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()) + .stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + + assertEquals(1, actualPackets.size()); + + MetricsPacket packet = actualPackets.get(0); + + assertEquals("system-coredumps-processing", packet.service.id); + assertEquals(1, packet.statusCode); + } + + static class MockFileWrapper extends FileWrapper { + + + @Override + Stream walkTree(Path path) { + return Stream.of(Path.of("dummy-path")); + } + + @Override + Instant getLastModifiedTime(Path path) { + return Instant.now(); + } + + @Override + boolean isRegularFile(Path path) { + return true; + } + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java new file mode 100644 index 00000000000..54568fad74d --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java @@ -0,0 +1,44 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.node; + +import ai.vespa.metricsproxy.metric.model.MetricId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class HostLifeGathererTest { + + @Test + public void host_is_alive() { + List actualPackets = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()) + .stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + + assertEquals(1, actualPackets.size()); + + MetricsPacket packet = actualPackets.get(0); + + Map expectedMetrics = Map.of(MetricId.toMetricId("uptime"), 123d, MetricId.toMetricId("alive"), 1d); + assertEquals("host_life", packet.service.id); + assertEquals(0, packet.statusCode); + assertEquals(expectedMetrics, packet.metrics()); + + } + + static class MockFileWrapper extends FileWrapper { + @Override + List readAllLines(Path path) { + return List.of("123 432"); + } + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java deleted file mode 100644 index 59a207bef0d..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/NodeMetricGathererTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; -import org.junit.Test; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author olaa - */ -public class NodeMetricGathererTest { - - @Test - public void gatherMetrics() throws Exception { - - MetricsManager metricsManager = mock(MetricsManager.class); - VespaServices vespaServices = mock(VespaServices.class); - ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class); - NodeDimensions nodeDimensions = mock(NodeDimensions.class); - NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class); - List mockedVespaServices = mockedVespaServices(); - - NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper); - when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream()); - when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0)); - when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456")); - when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices); - when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap()); - - List packets = nodeMetricGatherer.gatherMetrics(); - assertEquals(5, packets.size()); - Map serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health"); - List expectedPackets = List.of( - metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()), - metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()), - metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions), - metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions) - ); - - assertEqualMetricPackets(expectedPackets, packets); - } - - private void assertEqualMetricPackets(List expectedPackets, List actualPackets) { - assertEquals(expectedPackets.size(), actualPackets.size()); - expectedPackets.stream() - .forEach(expectedPacket -> - actualPackets.stream() - .filter(packet -> packet.service.equals(expectedPacket.service)) - .forEach(actualPacket -> { - assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage); - assertEquals(expectedPacket.statusCode, actualPacket.statusCode); - assertEquals(expectedPacket.metrics(), actualPacket.metrics()); - assertEquals(expectedPacket.dimensions(), actualPacket.dimensions()); - }) - ); - } - - private List mockedVespaServices() { - - HealthMetric healthy = HealthMetric.get("OK", ""); - HealthMetric unhealthy = HealthMetric.get("down", ""); - - VespaService service1 = mock(VespaService.class); - when(service1.getInstanceName()).thenReturn("instance"); - when(service1.getMonitoringName()).thenReturn("service1"); - when(service1.getHealth()).thenReturn(healthy); - - VespaService service2 = mock(VespaService.class); - when(service2.getInstanceName()).thenReturn("instance"); - when(service2.getMonitoringName()).thenReturn("service2"); - when(service2.getHealth()).thenReturn(healthy); - - - VespaService service3 = mock(VespaService.class); - when(service3.getInstanceName()).thenReturn("instance"); - when(service3.getMonitoringName()).thenReturn("service3"); - when(service3.getHealth()).thenReturn(unhealthy); - - return List.of(service1, service2, service3); - - } - - private MetricsPacket metricsPacket(int statusCode, String service, - Collection metrics, Map dimensions) { - return new MetricsPacket.Builder(ServiceId.toServiceId(service)) - .putDimensions(dimensions) - .putMetrics(metrics) - .statusCode(statusCode) - .build(); - } -} \ No newline at end of file -- cgit v1.2.3