Added YamasHandler and test

5 files changed, 309 insertions, 48 deletions

diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
index 26caa3f66af..29ab5bc3f91 100644
--- a/metrics-proxy/pom.xml
+++ b/metrics-proxy/pom.xml
@@ -141,6 +141,11 @@
       <artifactId>hamcrest-core</artifactId>
       <scope>test</scope>
     </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-core</artifactId>
+        <scope>test</scope>
+      </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
index 14d1203824b..fc81bd1a0ee 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
@@ -149,6 +149,10 @@ public class MetricsManager {
         externalMetrics.setExtraMetrics(packets);
     }
 
+    public Map<DimensionId, String> getExtraDimensions() {
+        return this.extraDimensions;
+    }
+
     /**
      * Returns a space separated list of all distinct service names.
      */
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java
index 895792e8d5f..c6dfdf8fe9c 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/gatherer/NodeMetricGatherer.java
@@ -1,16 +1,15 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.metricsproxy.gatherer;
 
+import ai.vespa.metricsproxy.core.MetricsManager;
 import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
 import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
 import ai.vespa.metricsproxy.metric.model.MetricsPacket;
 import ai.vespa.metricsproxy.metric.model.StatusCode;
-import ai.vespa.metricsproxy.metric.model.json.YamasArrayJsonModel;
 import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil;
 import ai.vespa.metricsproxy.service.VespaServices;
 import com.google.inject.Inject;
 import com.yahoo.vespa.defaults.Defaults;
-import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -21,9 +20,12 @@ import java.nio.file.Path;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @author olaa
@@ -35,25 +37,29 @@ public class NodeMetricGatherer {
     private final VespaServices vespaServices;
     private final ApplicationDimensions applicationDimensions;
     private final NodeDimensions nodeDimensions;
-    private final String coreDumpPath;
-    private final String hostLifePath;
+    private final MetricsManager metricsManager;
+    private final FileWrapper fileWrapper;
 
     private static final Logger logger = Logger.getLogger(NodeMetricGatherer.class.getSimpleName());
 
     @Inject
-    public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) {
-        this(vespaServices, applicationDimensions, nodeDimensions, "var/crash/processing", "/proc/uptime");
+    public NodeMetricGatherer(MetricsManager metricsManager, VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) {
+        this(metricsManager, vespaServices, applicationDimensions, nodeDimensions, new FileWrapper());
     }
 
-    public NodeMetricGatherer(VespaServices vespaServices, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions, String coreDumpPath, String hostLifePath) {
+    public NodeMetricGatherer(MetricsManager metricsManager,
+                              VespaServices vespaServices,
+                              ApplicationDimensions applicationDimensions,
+                              NodeDimensions nodeDimensions,
+                              FileWrapper fileWrapper) {
+        this.metricsManager = metricsManager;
         this.vespaServices = vespaServices;
         this.applicationDimensions = applicationDimensions;
         this.nodeDimensions = nodeDimensions;
-        this.coreDumpPath = coreDumpPath;
-        this.hostLifePath = hostLifePath;
+        this.fileWrapper = fileWrapper;
     }
 
-    public YamasArrayJsonModel gatherMetrics() throws JSONException {
+    public List<MetricsPacket> gatherMetrics()  {
         List<MetricsPacket.Builder> metricPacketBuilders = new ArrayList<>();
         metricPacketBuilders.addAll(coredumpMetrics());
         metricPacketBuilders.addAll(serviceHealthMetrics());
@@ -62,30 +68,36 @@ public class NodeMetricGatherer {
         List<MetricsPacket> metricPackets = metricPacketBuilders.stream().map(metricPacketBuilder -> {
             metricPacketBuilder.putDimensionsIfAbsent(applicationDimensions.getDimensions());
             metricPacketBuilder.putDimensionsIfAbsent(nodeDimensions.getDimensions());
+            metricPacketBuilder.putDimensionsIfAbsent(metricsManager.getExtraDimensions());
             return metricPacketBuilder.build();
         }).collect(Collectors.toList());
-        return YamasJsonUtil.toYamasArray(metricPackets);
+        return metricPackets;
     }
 
-    private List<MetricsPacket.Builder> coredumpMetrics() throws JSONException {
+    private List<MetricsPacket.Builder> coredumpMetrics() {
 
-        Path crashPath = Path.of(Defaults.getDefaults().underVespaHome(coreDumpPath));
+        Path crashPath = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing"));
         long coredumps = getCoredumpsFromLastPeriod(crashPath);
 
-        JSONObject jsonObject = new JSONObject();
-        jsonObject.put("timestamp", Instant.now().getEpochSecond());
-        jsonObject.put("application", "system-coredumps-processing");
-        jsonObject.put("status_code", coredumps);
-        jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES));
-        jsonObject.put("routing", ROUTING_JSON);
-        return YamasJsonUtil.toMetricsPackets(jsonObject.toString());
+        try {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("application", "Vespa.node");
+            jsonObject.put("timestamp", Instant.now().getEpochSecond());
+            jsonObject.put("application", "system-coredumps-processing");
+            jsonObject.put("status_code", coredumps);
+            jsonObject.put("status_message", coredumps == 0 ? "OK" : String.format("Found %d coredumps in past %d minutes", coredumps, COREDUMP_AGE_IN_MINUTES));
+            jsonObject.put("routing", ROUTING_JSON);
+            return YamasJsonUtil.toMetricsPackets(jsonObject.toString());
+        } catch (JSONException e) {
+            logger.log(Level.WARNING, "Error writing JSON", e);
+            return Collections.emptyList();
+        }
     }
 
     private List<MetricsPacket.Builder> serviceHealthMetrics()  {
-        JSONArray jsonArray = new JSONArray();
-        vespaServices.getVespaServices()
+        return vespaServices.getVespaServices()
                 .stream()
-                .forEach(service -> {
+                .map(service -> {
                     try {
                         StatusCode healthStatus = service.getHealth().getStatus();
                         JSONObject jsonObject = new JSONObject();
@@ -94,47 +106,52 @@ public class NodeMetricGatherer {
                         jsonObject.put("application", service.getMonitoringName());
                         JSONObject dimensions = new JSONObject();
                         dimensions.put("instance", service.getInstanceName());
-                        dimensions.put("metricsType", "health");
+                        dimensions.put("metrictype", "health");
                         jsonObject.put("dimensions", dimensions);
                         jsonObject.put("routing", ROUTING_JSON);
-                        jsonArray.put(jsonObject);
+                        return YamasJsonUtil.toMetricsPackets(jsonObject.toString()).get(0);
                     } catch (JSONException e) {
-
+                        throw new RuntimeException(e.getMessage());
                     }
-                });
-
-        return YamasJsonUtil.toMetricsPackets(jsonArray.toString());
+                })
+                .collect(Collectors.toList());
     }
 
-    private List<MetricsPacket.Builder> hostLifeMetrics() throws JSONException {
+    private List<MetricsPacket.Builder> hostLifeMetrics() {
         JSONObject jsonObject = new JSONObject();
         double upTime;
         int statusCode = 0;
         String statusMessage = "OK";
         try {
-            upTime = getHostLife(Path.of(hostLifePath)); // ??
+            upTime = getHostLife(Path.of("/proc/uptime")); // ??
         } catch (IOException e) {
             upTime = 0d;
             statusCode = 1;
             statusMessage = e.getMessage();
         }
 
-        jsonObject.put("timestamp", Instant.now().getEpochSecond());
-        jsonObject.put("status_message", statusMessage);
-        jsonObject.put("status_code", statusCode);
-        JSONObject metrics = new JSONObject();
-        metrics.put("uptime", upTime);
-        metrics.put("alive", 1);
-        jsonObject.put("metrics", metrics);
-        jsonObject.put("routing", ROUTING_JSON);
+        try {
+            jsonObject.put("application", "host_life");
+            jsonObject.put("timestamp", Instant.now().getEpochSecond());
+            jsonObject.put("status_message", statusMessage);
+            jsonObject.put("status_code", statusCode);
+            JSONObject metrics = new JSONObject();
+            metrics.put("uptime", upTime);
+            metrics.put("alive", 1);
+            jsonObject.put("metrics", metrics);
+            jsonObject.put("routing", ROUTING_JSON);
+            return YamasJsonUtil.toMetricsPackets(jsonObject.toString());
+        } catch (JSONException e) {
+            logger.log(Level.WARNING, "Error writing JSON", e);
+            return Collections.emptyList();
+        }
+
 
-        return YamasJsonUtil.toMetricsPackets(jsonObject.toString());
     }
 
     private long getCoredumpsFromLastPeriod(Path coreDumpPath) {
-
         try {
-            return Files.walk(coreDumpPath)
+            return fileWrapper.walkTree(coreDumpPath)
                     .filter(Files::isRegularFile)
                     .filter(this::isNewFile)
                    .count();
@@ -144,7 +161,7 @@ public class NodeMetricGatherer {
     }
 
     private double getHostLife(Path uptimePath) throws IOException {
-         return Files.readAllLines(uptimePath)
+         return fileWrapper.readAllLines(uptimePath)
                  .stream()
                  .mapToDouble(line -> Double.valueOf(line.split("\\s")[0]))
                  .findFirst()
@@ -153,7 +170,7 @@ public class NodeMetricGatherer {
 
     private boolean isNewFile(Path file) {
         try {
-            return Files.getLastModifiedTime(file).toInstant()
+            return fileWrapper.getLastModifiedTime(file)
                     .plus(COREDUMP_AGE_IN_MINUTES, ChronoUnit.MINUTES)
                     .isBefore(Instant.now());
         } catch (IOException e) {
@@ -161,14 +178,27 @@ public class NodeMetricGatherer {
         }
     }
 
-    private static final JSONObject createRoutingJSON() {
+    private static JSONObject createRoutingJSON() {
         try {
-            JSONObject namesspace = new JSONObject();
-            namesspace.put("yamas", "Vespa");
-            return namesspace;
+            JSONObject jsonObject = new JSONObject("{\"yamas\":{\"namespaces\":[\"Vespa\"]}}");
+            return jsonObject;
         } catch (JSONException e) {
             throw new RuntimeException(e);
         }
     }
 
+    static class FileWrapper {
+
+        List<String> readAllLines(Path path) throws IOException {
+            return Files.readAllLines(path);
+        }
+
+        Stream<Path> walkTree(Path path) throws IOException {
+            return Files.walk(path);
+        }
+
+        Instant getLastModifiedTime(Path path) throws IOException {
+            return Files.getLastModifiedTime(path).toInstant();
+        }
+    }
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java
new file mode 100644
index 00000000000..fca1b66342f
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/YamasHandler.java
@@ -0,0 +1,110 @@
+/*
+* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+*/
+
+package ai.vespa.metricsproxy.http;
+
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.gatherer.NodeMetricGatherer;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException;
+import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+import com.yahoo.restapi.Path;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.net.URI;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED;
+import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+import static com.yahoo.jdisc.Response.Status.OK;
+import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+import static java.util.logging.Level.WARNING;
+
+/**
+ * @author olaa
+ */
+
+// TODO: Merge with gjoranv's API util changes
+public class YamasHandler extends ThreadedHttpRequestHandler {
+
+    static final String V1_PATH = "/yamas/v1";
+    static final String VALUES_PATH = V1_PATH + "/values";
+
+    private final ValuesFetcher valuesFetcher;
+    private final NodeMetricGatherer nodeMetricGatherer;
+
+    @Inject
+    public YamasHandler(Executor executor,
+                        MetricsManager metricsManager,
+                        VespaServices vespaServices,
+                        MetricsConsumers metricsConsumers,
+                        ApplicationDimensions applicationDimensions,
+                        NodeDimensions nodeDimensions) {
+        super(executor);
+        valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers);
+        this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions);
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported");
+
+        Path path = new Path(request.getUri());
+
+        if (path.matches(V1_PATH)) return v1Response(request.getUri());
+        if (path.matches(VALUES_PATH)) return valuesResponse(request);
+
+        return new ErrorResponse(NOT_FOUND, "No content at given path");
+    }
+
+    private JsonResponse v1Response(URI requestUri) {
+        try {
+            return new JsonResponse(OK, v1Content(requestUri));
+        } catch (JSONException e) {
+            log.log(WARNING, "Bad JSON construction in " + V1_PATH +  " response", e);
+            return new ErrorResponse(INTERNAL_SERVER_ERROR, "An error occurred, please try path '" + VALUES_PATH + "'");
+        }
+    }
+
+    private JsonResponse valuesResponse(HttpRequest request) {
+        try {
+            List<MetricsPacket> metrics =  valuesFetcher.fetch(request.getProperty("consumer"));
+            metrics.addAll(nodeMetricGatherer.gatherMetrics());
+            return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics).serialize());
+        } catch (JsonRenderingException e) {
+            return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage());
+        }
+    }
+
+    // TODO: Use jackson with a "Resources" class instead of JSONObject
+    private String v1Content(URI requestUri) throws JSONException {
+        int port = requestUri.getPort();
+        String host = requestUri.getHost();
+        StringBuilder base = new StringBuilder("http://");
+        base.append(host);
+        if (port >= 0) {
+            base.append(":").append(port);
+        }
+        String uriBase = base.toString();
+        JSONArray linkList = new JSONArray();
+        for (String api : new String[] {VALUES_PATH}) {
+            JSONObject resource = new JSONObject();
+            resource.put("url", uriBase + api);
+            linkList.put(resource);
+        }
+        return new JSONObject().put("resources", linkList).toString(4);
+    }
+}
\ No newline at end of file
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java
new file mode 100644
index 00000000000..8752032b643
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/gatherer/NodeMetricGathererTest.java
@@ -0,0 +1,112 @@
+package ai.vespa.metricsproxy.gatherer;
+
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.metric.HealthMetric;
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author olaa
+ */
+public class NodeMetricGathererTest {
+
+    @Test
+    public void gatherMetrics() throws Exception {
+
+        MetricsManager metricsManager = mock(MetricsManager.class);
+        VespaServices vespaServices = mock(VespaServices.class);
+        ApplicationDimensions applicationDimensions = mock(ApplicationDimensions.class);
+        NodeDimensions nodeDimensions = mock(NodeDimensions.class);
+        NodeMetricGatherer.FileWrapper fileWrapper = mock(NodeMetricGatherer.FileWrapper.class);
+        List<VespaService> mockedVespaServices = mockedVespaServices();
+
+        NodeMetricGatherer nodeMetricGatherer = new NodeMetricGatherer(metricsManager, vespaServices, applicationDimensions, nodeDimensions, fileWrapper);
+        when(fileWrapper.walkTree(any())).thenReturn(List.of(Path.of("")).stream());
+        when(fileWrapper.getLastModifiedTime(any())).thenReturn(Instant.ofEpochMilli(0));
+        when(fileWrapper.readAllLines(any())).thenReturn(List.of("123 456"));
+        when(vespaServices.getVespaServices()).thenReturn(mockedVespaServices);
+        when(applicationDimensions.getDimensions()).thenReturn(Collections.emptyMap());
+        when(nodeDimensions.getDimensions()).thenReturn(Collections.emptyMap());
+
+        List<MetricsPacket> packets = nodeMetricGatherer.gatherMetrics();
+        assertEquals(5, packets.size());
+        Map<DimensionId, String> serviceHealthDimensions = Map.of(DimensionId.toDimensionId("instance"), "instance", DimensionId.toDimensionId("metrictype"), "health");
+        List<MetricsPacket> expectedPackets = List.of(
+            metricsPacket(0, "system-coredumps-processing", Collections.emptyList(), Collections.emptyMap()),
+            metricsPacket(0, "host_life", List.of(new Metric("uptime", 123), new Metric("alive", 1)), Collections.emptyMap()),
+            metricsPacket(0, "service1", Collections.emptyList(), serviceHealthDimensions),
+            metricsPacket(0, "service2", Collections.emptyList(), serviceHealthDimensions),
+            metricsPacket(1, "service3", Collections.emptyList(), serviceHealthDimensions)
+        );
+
+        assertEqualMetricPackets(expectedPackets, packets);
+    }
+
+    private void assertEqualMetricPackets(List<MetricsPacket> expectedPackets, List<MetricsPacket> actualPackets) {
+        assertEquals(expectedPackets.size(), actualPackets.size());
+        expectedPackets.stream()
+                .forEach(expectedPacket ->
+                    actualPackets.stream()
+                            .filter(packet -> packet.service.equals(expectedPacket.service))
+                            .forEach(actualPacket -> {
+                                assertEquals(expectedPacket.statusMessage, actualPacket.statusMessage);
+                                assertEquals(expectedPacket.statusCode, actualPacket.statusCode);
+                                assertEquals(expectedPacket.metrics(), actualPacket.metrics());
+                                assertEquals(expectedPacket.dimensions(), actualPacket.dimensions());
+                            })
+                );
+    }
+
+    private List<VespaService> mockedVespaServices() {
+
+        HealthMetric healthy = HealthMetric.get("OK", "");
+        HealthMetric unhealthy = HealthMetric.get("down", "");
+
+        VespaService service1 = mock(VespaService.class);
+        when(service1.getInstanceName()).thenReturn("instance");
+        when(service1.getMonitoringName()).thenReturn("service1");
+        when(service1.getHealth()).thenReturn(healthy);
+
+        VespaService service2 = mock(VespaService.class);
+        when(service2.getInstanceName()).thenReturn("instance");
+        when(service2.getMonitoringName()).thenReturn("service2");
+        when(service2.getHealth()).thenReturn(healthy);
+
+
+        VespaService service3 = mock(VespaService.class);
+        when(service3.getInstanceName()).thenReturn("instance");
+        when(service3.getMonitoringName()).thenReturn("service3");
+        when(service3.getHealth()).thenReturn(unhealthy);
+
+        return List.of(service1, service2, service3);
+
+    }
+
+    private MetricsPacket metricsPacket(int statusCode, String service,
+                                        Collection<Metric> metrics, Map<DimensionId, String> dimensions) {
+        return new MetricsPacket.Builder(ServiceId.toServiceId(service))
+                .putDimensions(dimensions)
+                .putMetrics(metrics)
+                .statusCode(statusCode)
+                .build();
+    }
+}
\ No newline at end of file