Merge pull request #10472 from vespa-engine/olaa/metrics-proxy-gather-node-metrics

Yamas handler and system metric gatherer

11 files changed, 365 insertions, 2 deletions

diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
index 14d1203824b..fc81bd1a0ee 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
@@ -149,6 +149,10 @@ public class MetricsManager {
         externalMetrics.setExtraMetrics(packets);
     }
 
+    public Map<DimensionId, String> getExtraDimensions() {
+        return this.extraDimensions;
+    }
+
     /**
      * Returns a space separated list of all distinct service names.
      */
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
index 00f2078ef57..830e2201966 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
@@ -17,7 +17,6 @@ import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
 import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
-import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericJsonModel;
 
 /**
  * Generates metrics values in json format for the metrics/v1 rest api.
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java
new file mode 100644
index 00000000000..7a868d0bffa
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java
@@ -0,0 +1,67 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.yamas;
+
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.node.NodeMetricGatherer;
+import ai.vespa.metricsproxy.http.ErrorResponse;
+import ai.vespa.metricsproxy.http.HttpHandlerBase;
+import ai.vespa.metricsproxy.http.JsonResponse;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException;
+import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.Path;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.OK;
+
+/**
+ * @author olaa
+ */
+
+public class YamasHandler extends HttpHandlerBase {
+
+    public static final String V1_PATH = "/yamas/v1";
+    private static final String VALUES_PATH = V1_PATH + "/values";
+
+    private final NodeMetricGatherer nodeMetricGatherer;
+
+    @Inject
+    public YamasHandler(Executor executor,
+                        MetricsManager metricsManager,
+                        VespaServices vespaServices,
+                        MetricsConsumers metricsConsumers,
+                        ApplicationDimensions applicationDimensions,
+                        NodeDimensions nodeDimensions) {
+        super(executor, metricsManager, vespaServices, metricsConsumers);
+        this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions);
+    }
+
+    @Override
+    public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) {
+        if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH)));
+        if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer));
+        return Optional.empty();
+    }
+
+    private JsonResponse valuesResponse(String consumer) {
+        try {
+            List<MetricsPacket> metrics =  valuesFetcher.fetch(consumer);
+            metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers
+            return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize());
+        } catch (JsonRenderingException e) {
+            return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage());
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java
new file mode 100644
index 00000000000..91e7fbbc0b5
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/CoredumpMetricGatherer.java
@@ -0,0 +1,46 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Set;
+import java.util.logging.Logger;
+
+
+/**
+ * @author olaa
+ */
+public class CoredumpMetricGatherer {
+
+    private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing"));
+
+    private static final Logger logger = Logger.getLogger(CoredumpMetricGatherer.class.getSimpleName());
+
+
+    protected static MetricsPacket.Builder gatherCoredumpMetrics(FileWrapper fileWrapper) {
+        int coredumps = getNumberOfCoredumps(fileWrapper);
+        return new MetricsPacket.Builder(ServiceId.toServiceId("system-coredumps-processing"))
+        .timestamp(Instant.now().getEpochSecond())
+        .statusCode(coredumps)
+        .statusMessage(coredumps == 0 ? "OK" : String.format("Found %d coredumps", coredumps))
+        .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa")));
+    }
+
+    private static int getNumberOfCoredumps(FileWrapper fileWrapper) {
+        try {
+            return (int) fileWrapper.walkTree(COREDUMP_PATH)
+                    .filter(fileWrapper::isRegularFile)
+                    .count();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java
new file mode 100644
index 00000000000..aa01802883e
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/FileWrapper.java
@@ -0,0 +1,28 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * @author olaa
+ */
+public class FileWrapper {
+
+    long getFileAgeInSeconds(Path path) throws IOException {
+        Instant lastModifiedTime = Files.getLastModifiedTime(path).toInstant();
+        return Instant.now().getEpochSecond() - lastModifiedTime.getEpochSecond();
+    }
+
+    Stream<Path> walkTree(Path path) throws IOException {
+        return Files.walk(path);
+    }
+
+    boolean isRegularFile(Path path) {
+        return Files.isRegularFile(path);
+    }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java
new file mode 100644
index 00000000000..8fdfc93022c
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/HostLifeGatherer.java
@@ -0,0 +1,44 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.MetricId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Set;
+
+
+/**
+ * @author olaa
+ */
+public class HostLifeGatherer {
+
+    private static final Path UPTIME_PATH = Path.of("/proc");
+
+    protected static MetricsPacket.Builder gatherHostLifeMetrics(FileWrapper fileWrapper) {
+        long upTime;
+        int statusCode = 0;
+        String statusMessage = "OK";
+
+        try {
+            upTime = fileWrapper.getFileAgeInSeconds(UPTIME_PATH);
+        } catch (IOException e) {
+            upTime = 0;
+            statusCode = 1;
+            statusMessage = e.getMessage();
+        }
+
+        return new MetricsPacket.Builder(ServiceId.toServiceId("host_life"))
+                .timestamp(Instant.now().getEpochSecond())
+                .statusMessage(statusMessage)
+                .statusCode(statusCode)
+                .putMetric(MetricId.toMetricId("uptime"), upTime)
+                .putMetric(MetricId.toMetricId("alive"), 1)
+                .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa")));
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java
new file mode 100644
index 00000000000..cd71bb781fe
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java
@@ -0,0 +1,63 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static ai.vespa.metricsproxy.node.CoredumpMetricGatherer.gatherCoredumpMetrics;
+import static ai.vespa.metricsproxy.node.HostLifeGatherer.gatherHostLifeMetrics;
+import static ai.vespa.metricsproxy.node.ServiceHealthGatherer.gatherServiceHealthMetrics;
+
+/**
+ * Fetches miscellaneous system metrics for node, including
+ *  - Current coredump processing
+ *  - Health of Vespa services
+ *  - Host life
+ *
+ * @author olaa
+ */
+
+
+public class NodeMetricGatherer {
+
+    private final VespaServices vespaServices;
+    private final ApplicationDimensions applicationDimensions;
+    private final NodeDimensions nodeDimensions;
+    private final MetricsManager metricsManager;
+
+    @Inject
+    public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) {
+        this.metricsManager = metricsManager;
+        this.vespaServices = vespaServices;
+        this.applicationDimensions = applicationDimensions;
+        this.nodeDimensions = nodeDimensions;
+    }
+
+    public List<MetricsPacket> gatherMetrics()  {
+        FileWrapper fileWrapper = new FileWrapper();
+        List<MetricsPacket.Builder> metricPacketBuilders = new ArrayList<>();
+        metricPacketBuilders.add(gatherCoredumpMetrics(fileWrapper));
+        metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices));
+
+        if (SystemPollerProvider.runningOnLinux()) {
+            metricPacketBuilders.add(gatherHostLifeMetrics(fileWrapper));
+        }
+
+        return metricPacketBuilders.stream()
+                .map(metricPacketBuilder ->
+                    metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions())
+                    .putDimensionsIfAbsent(nodeDimensions.getDimensions())
+                    .putDimensionsIfAbsent(metricsManager.getExtraDimensions()).build()
+        ).collect(Collectors.toList());
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java
new file mode 100644
index 00000000000..923fb4d646d
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java
@@ -0,0 +1,36 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.metric.model.ConsumerId;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+import ai.vespa.metricsproxy.service.VespaServices;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+/**
+ * @author olaa
+ */
+public class ServiceHealthGatherer {
+
+
+    protected static List<MetricsPacket.Builder> gatherServiceHealthMetrics(VespaServices vespaServices)  {
+        return vespaServices.getVespaServices()
+                .stream()
+                .map(service ->
+                     new MetricsPacket.Builder(ServiceId.toServiceId(service.getMonitoringName()))
+                            .timestamp(Instant.now().getEpochSecond())
+                            .statusMessage(service.getHealth().getStatus().status)
+                            .statusCode(service.getHealth().getStatus().code)
+                            .putDimension(DimensionId.toDimensionId("instance"), service.getInstanceName())
+                            .putDimension(DimensionId.toDimensionId("metrictype"), "health")
+                            .addConsumers(Set.of(ConsumerId.toConsumerId("Vespa")))
+                )
+                .collect(Collectors.toList());
+    }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java
index ea47933ddf0..6494054c278 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java
@@ -38,7 +38,7 @@ public class SystemPollerProvider implements Provider<SystemPoller> {
         return poller;
     }
 
-    private static boolean runningOnLinux() {
+    public static boolean runningOnLinux() {
         return getOs().contains("nux");
     }
 
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java
new file mode 100644
index 00000000000..e7a905be52e
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/CoredumpGathererTest.java
@@ -0,0 +1,39 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author olaa
+ */
+public class CoredumpGathererTest {
+
+    @Test
+    public void finds_one_coredump() {
+    MetricsPacket packet = CoredumpMetricGatherer.gatherCoredumpMetrics(new MockFileWrapper()).build();
+
+    assertEquals("system-coredumps-processing", packet.service.id);
+    assertEquals(1, packet.statusCode);
+    }
+
+    static class MockFileWrapper extends FileWrapper {
+
+
+        @Override
+        Stream<Path> walkTree(Path path)  {
+            return Stream.of(Path.of("dummy-path"));
+        }
+
+        @Override
+        boolean isRegularFile(Path path) {
+            return true;
+        }
+    }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java
new file mode 100644
index 00000000000..8317229b73a
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/HostLifeGathererTest.java
@@ -0,0 +1,37 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.node;
+
+import ai.vespa.metricsproxy.metric.model.MetricId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author olaa
+ */
+public class HostLifeGathererTest {
+
+    @Test
+    public void host_is_alive() {
+        MetricsPacket packet = HostLifeGatherer.gatherHostLifeMetrics(new MockFileWrapper()).build();
+
+        assertEquals("host_life", packet.service.id);
+        assertEquals(0, packet.statusCode);
+        assertEquals(123l, packet.metrics().get(MetricId.toMetricId("uptime")));
+        assertEquals(1, packet.metrics().get(MetricId.toMetricId("alive")));
+
+    }
+
+    static class MockFileWrapper extends FileWrapper {
+        @Override
+        long getFileAgeInSeconds(Path path) {
+            return 123;
+        }
+    }
+}