Add request handler for generic metrics format.

9 files changed, 289 insertions, 15 deletions

diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
index 66ca25dadf0..3cfe1d2f568 100644
--- a/metrics-proxy/pom.xml
+++ b/metrics-proxy/pom.xml
@@ -60,6 +60,12 @@
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
+      <artifactId>container-core</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
       <artifactId>container-di</artifactId>
       <version>${project.version}</version>
       <scope>provided</scope>
@@ -104,6 +110,12 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>jdisc_http_service</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
@@ -120,6 +132,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-http</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-core</artifactId>
       <scope>test</scope>
@@ -158,6 +175,16 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <!-- Illegal reflective access by guice -->
+          <argLine>
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+          </argLine>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/GenericMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/GenericMetricsHandler.java
new file mode 100644
index 00000000000..488f26b7af7
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/GenericMetricsHandler.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.http;
+
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/**
+ * Handler exposing the generic metrics format via http.
+ *
+ * @author gjoranv
+ */
+public class GenericMetricsHandler extends ThreadedHttpRequestHandler {
+
+    private final MetricsManager metricsManager;
+    private final VespaServices vespaServices;
+
+    @Inject
+    public GenericMetricsHandler(Executor executor, MetricsManager metricsManager, VespaServices vespaServices) {
+        super(executor);
+        this.metricsManager = metricsManager;
+        this.vespaServices = vespaServices;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        try {
+            List<MetricsPacket> metrics = metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now());
+            return new Response(200, GenericJsonUtil.toGenericJsonModel(metrics).serialize());
+        } catch (GenericJsonModel.JsonMetricsRenderingException e) {
+            return new ErrorResponse(500, Exceptions.toMessageString(e));
+        }
+    }
+
+    private static class Response extends HttpResponse {
+        private final byte[] data;
+
+        Response(int code, String data) {
+            super(code);
+            this.data = data.getBytes(Charset.forName(DEFAULT_CHARACTER_ENCODING));
+        }
+
+        @Override
+        public String getContentType() {
+            return "application/json";
+        }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            outputStream.write(data);
+        }
+    }
+
+    private static class ErrorResponse extends Response {
+        ErrorResponse(int code, String data) {
+            super(code, "{\"error\":\"" + data + "\"}");
+        }
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java
index dd8858ff99e..14f085ff388 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java
@@ -38,8 +38,13 @@ public class GenericJsonModel {
             return mapper.writeValueAsString(this);
         } catch (IOException e) {
             log.log(Level.WARNING, "Got exception when rendering metrics:", e);
-            throw new RuntimeException("Could not render metrics. Check the log for details.");
+            throw new JsonMetricsRenderingException("Could not render metrics. Check the log for details.");
         }
     }
 
+    public static class JsonMetricsRenderingException extends RuntimeException {
+        JsonMetricsRenderingException(String message) {
+            super(message);
+        }
+    }
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java
index 5c04f933c4a..d9df3e0c0e9 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JacksonUtil.java
@@ -20,12 +20,12 @@ import java.util.Locale;
  * @author smorgrav
  * @author gjoranv
  */
-class JacksonUtil {
+public class JacksonUtil {
 
     /**
      * Returns an object mapper with a custom floating point serializer to avoid scientific notation
      */
-    static ObjectMapper createObjectMapper() {
+    public static ObjectMapper createObjectMapper() {
         ObjectMapper mapper = new ObjectMapper();
         SimpleModule module = new SimpleModule("DoubleSerializer",
                                                new Version(1, 0, 0, "", null, null));
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
index d3f674176b3..c59daf65cc2 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java
@@ -33,6 +33,7 @@ public class VespaService implements Comparable<VespaService> {
 
     private volatile int pid = -1;
     private volatile String state = "UNKNOWN";
+    private volatile boolean isAlive;
 
     // Used to keep the last polled system metrics for service
     private Metrics systemMetrics;
@@ -42,7 +43,6 @@ public class VespaService implements Comparable<VespaService> {
     private final RemoteHealthMetricFetcher remoteHealthMetricFetcher;
     private final RemoteMetricsFetcher remoteMetricsFetcher;
 
-    private boolean isAlive;
 
     // Used to keep track of log level when health or metrics requests fail
     private AtomicInteger metricsFetchCount = new AtomicInteger(0);
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java
index e18bd38f97a..6f86be3aa30 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java
@@ -4,12 +4,22 @@
 
 package ai.vespa.metricsproxy;
 
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.core.VespaMetrics;
+import ai.vespa.metricsproxy.metric.ExternalMetrics;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.List;
 import java.util.stream.Collectors;
 
 /**
@@ -17,6 +27,15 @@ import java.util.stream.Collectors;
  */
 public class TestUtil {
 
+    public static MetricsManager createMetricsManager(VespaServices vespaServices,
+                                                      MetricsConsumers consumers,
+                                                      ApplicationDimensions applicationDimensions,
+                                                      NodeDimensions nodeDimensions) {
+        VespaMetrics metrics = new VespaMetrics(consumers, vespaServices);
+        return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers),
+                                  applicationDimensions, nodeDimensions);
+    }
+
     public static String getFileContents(String filename) {
         InputStream in = TestUtil.class.getClassLoader().getResourceAsStream(filename);
         if (in == null) {
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java
index 81634323269..1635ccab197 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java
@@ -4,6 +4,7 @@
 
 package ai.vespa.metricsproxy.core;
 
+import ai.vespa.metricsproxy.TestUtil;
 import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer;
 import ai.vespa.metricsproxy.metric.ExternalMetrics;
 import ai.vespa.metricsproxy.metric.Metric;
@@ -56,7 +57,8 @@ public class MetricsManagerTest {
 
     @Before
     public void setupMetricsManager() {
-        metricsManager = getMetricsManager();
+        metricsManager = TestUtil.createMetricsManager(new VespaServices(testServices), getMetricsConsumers(),
+                                                       getApplicationDimensions(), getNodeDimensions());
     }
 
     @Test
@@ -201,15 +203,6 @@ public class MetricsManagerTest {
         return MetricsManager.adjustTimestamp(builder, startTime).getTimestamp();
     }
 
-    private MetricsManager getMetricsManager() {
-        VespaServices vespaServices = new VespaServices(testServices);
-        MetricsConsumers consumers = getMetricsConsumers();
-        VespaMetrics metrics = new VespaMetrics(consumers, vespaServices);
-
-        return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers),
-                                  getApplicationDimensions(),getNodeDimensions());
-    }
-
     private static MetricsConsumers getMetricsConsumers() {
         Consumer.Metric.Dimension.Builder metricDimension = new Consumer.Metric.Dimension.Builder()
                 .key("dim0").value("metric-dim");
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java
new file mode 100644
index 00000000000..744c96d7744
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/GenericMetricsHandlerTest.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.http;
+
+import ai.vespa.metricsproxy.TestUtil;
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.metric.Metric;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
+import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
+import ai.vespa.metricsproxy.metric.model.json.GenericService;
+import ai.vespa.metricsproxy.service.DummyService;
+import ai.vespa.metricsproxy.service.VespaService;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.yahoo.container.jdisc.RequestHandlerTestDriver;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.Executors;
+
+import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID;
+import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
+import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author gjoranv
+ */
+@SuppressWarnings("UnstableApiUsage")
+public class GenericMetricsHandlerTest {
+
+    private static final List<VespaService> testServices = ImmutableList.of(
+            new DummyService(0, "dummy/id/0"),
+            new DummyService(1, "dummy/id/0"));
+
+    private static final String CPU_METRIC = "cpu";
+
+    private static final String URI = "http://localhost/metrics/v1/values";
+
+    private static final VespaServices vespaServices = new VespaServices(testServices);
+
+    private static RequestHandlerTestDriver testDriver;
+
+    @BeforeClass
+    public static void setupMetricsManager() {
+        MetricsManager metricsManager = TestUtil.createMetricsManager(vespaServices, getMetricsConsumers(), getApplicationDimensions(), getNodeDimensions());
+        metricsManager.setExtraMetrics(ImmutableList.of(
+                new MetricsPacket.Builder(toServiceId("foo"))
+                        .timestamp(Instant.now().getEpochSecond())
+                        .putMetrics(ImmutableList.of(new Metric(CPU_METRIC, 12.345)))));
+        GenericMetricsHandler handler = new GenericMetricsHandler(Executors.newSingleThreadExecutor(), metricsManager, vespaServices);
+        testDriver = new RequestHandlerTestDriver(handler);
+    }
+
+    @Ignore
+    @Test
+    public void visually_inspect_response() throws Exception{
+        String response = testDriver.sendRequest(URI).readAll();
+        ObjectMapper mapper = createObjectMapper();
+        var jsonModel = mapper.readValue(response, GenericJsonModel.class);
+        System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
+    }
+
+    @Test
+    public void response_contains_node_metrics() throws Exception {
+        String response = testDriver.sendRequest(URI).readAll();
+        var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class);
+
+        assertNotNull(jsonModel.node);
+        assertEquals(1, jsonModel.node.metrics.size());
+        assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d);
+    }
+
+    @Test
+    public void response_contains_service_metrics() throws Exception {
+        String response = testDriver.sendRequest(URI).readAll();
+        var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class);
+
+        assertEquals(1, jsonModel.services.size());
+        GenericService dummyService = jsonModel.services.get(0);
+        assertEquals(2, dummyService.metrics.size());
+
+        GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService);
+        assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue());
+        assertEquals("metric-dim", dummy0Metrics.dimensions.get("dim0"));
+
+        GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService);
+        assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue());
+        assertEquals("metric-dim", dummy1Metrics.dimensions.get("dim0"));
+    }
+
+    @Test
+    public void all_timestamps_are_equal_and_non_zero() throws Exception {
+        String response = testDriver.sendRequest(URI).readAll();
+        var jsonModel = createObjectMapper().readValue(response, GenericJsonModel.class);
+
+        Long nodeTimestamp = jsonModel.node.timestamp;
+        assertNotEquals(0L, (long) nodeTimestamp);
+        for (var service : jsonModel.services)
+            assertEquals(nodeTimestamp, service.timestamp);
+    }
+
+    private static GenericMetrics getMetricsForInstance(String instance, GenericService service) {
+        for (var metrics : service.metrics) {
+            if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance))
+                return metrics;
+        }
+        throw new RuntimeException("Could not find metrics for service instance " + instance);
+    }
+
+    private static MetricsConsumers getMetricsConsumers() {
+        ConsumersConfig.Consumer.Metric.Dimension.Builder metricDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder()
+                .key("dim0").value("metric-dim");
+
+        return new MetricsConsumers(new ConsumersConfig.Builder()
+                                            .consumer(new ConsumersConfig.Consumer.Builder()
+                                                              .name(VESPA_CONSUMER_ID.id)
+                                                              .metric(new ConsumersConfig.Consumer.Metric.Builder()
+                                                                              .name(CPU_METRIC)
+                                                                              .outputname(CPU_METRIC))
+                                                              .metric(new ConsumersConfig.Consumer.Metric.Builder()
+                                                                              .name(METRIC_1)
+                                                                              .outputname(METRIC_1)
+                                                                              .dimension(metricDimension)))
+                                            .build());
+    }
+
+    private static ApplicationDimensions getApplicationDimensions() {
+        return new ApplicationDimensions(new ApplicationDimensionsConfig.Builder().build());
+    }
+
+    private static NodeDimensions getNodeDimensions() {
+        return new NodeDimensions(new NodeDimensionsConfig.Builder().build());
+    }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
index 44fe383839f..dc3eb12ff2c 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
@@ -64,7 +64,7 @@ public class GenericJsonModelTest {
         assertNotNull(jsonModel.node);
         assertEquals(1, jsonModel.node.metrics.size());
         GenericMetrics nodeMetrics = jsonModel.node.metrics.get(0);
-        assertEquals(1.234, jsonModel.node.metrics.get(0).values.get("node-metric"), 0.001d);
+        assertEquals(1.234, nodeMetrics.values.get("node-metric"), 0.001d);
         assertEquals("node-dim-value", nodeMetrics.dimensions.get("node-dim"));
 
         assertEquals(1, jsonModel.services.size());