Add conversion from metrics packets to the generic json model.

6 files changed, 132 insertions, 1 deletions

diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java
index 62465909798..26ae177d767 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java
@@ -39,7 +39,7 @@ public class ExternalMetrics {
     public static final DimensionId STATE_DIMENSION = toDimensionId("state");
     public static final DimensionId ORCHESTRATOR_STATE_DIMENSION = toDimensionId("orchestratorState");
 
-    static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node");
+    public static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node");
 
     private volatile List<MetricsPacket.Builder> metrics = new ArrayList<>();
     private final MetricsConsumers consumers;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java
new file mode 100644
index 00000000000..495e3ec1f7d
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ */
+
+package ai.vespa.metricsproxy.metric.model.json;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.ServiceId;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Utilities for converting between metrics packets and the generic json format.
+ *
+ * @author gjoranv
+ */
+public class GenericJsonUtil {
+
+    private GenericJsonUtil() { }
+
+    public static GenericJsonModel toGenericJsonModel(List<MetricsPacket> metricsPackets) {
+        Map<ServiceId, List<MetricsPacket>> packetsByService = metricsPackets.stream()
+                .collect(Collectors.groupingBy(packet -> packet.service));
+
+        var jsonModel = new GenericJsonModel();
+        var genericServices = new ArrayList<GenericService>();
+        packetsByService.forEach((serviceId, packets) -> {
+            var genericMetricsList = packets.stream()
+                    .map(packet -> new GenericMetrics(packet.metrics(), packet.dimensions()))
+                    .collect(toList());
+            var genericService = new GenericService(serviceId.id,
+                                                    packets.get(0).timestamp,
+                                                    genericMetricsList);
+            if (VESPA_NODE_SERVICE_ID.equals(serviceId)) {
+                jsonModel.node = new GenericNode(genericService.timestamp, genericService.metrics);
+            } else {
+                genericServices.add(genericService);
+
+            }
+        });
+
+        jsonModel.services = genericServices;
+        return jsonModel;
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java
index 0756269e2ff..46f515bfb47 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericMetrics.java
@@ -4,14 +4,20 @@
 
 package ai.vespa.metricsproxy.metric.model.json;
 
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricId;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.google.common.base.Functions;
+import com.yahoo.stream.CustomCollectors;
 
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
+import static com.yahoo.stream.CustomCollectors.toLinkedMap;
 
 /**
  * @author gjoranv
@@ -27,4 +33,11 @@ public class GenericMetrics {
     @JsonProperty("dimensions")
     public Map<String, String> dimensions;
 
+    public GenericMetrics() { }
+
+    GenericMetrics(Map<MetricId, Number> values, Map<DimensionId, String> dimensions) {
+        this.values = values.entrySet().stream().collect(toLinkedMap(entry -> entry.getKey().id, entry -> entry.getValue().doubleValue()));
+        this.dimensions = dimensions.entrySet().stream().collect(toLinkedMap(entry -> entry.getKey().id, Map.Entry::getValue));
+    }
+
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java
index 5f5f5a8f802..6b6f8d4d9cb 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericNode.java
@@ -27,4 +27,10 @@ public class GenericNode {
     @JsonProperty("metrics")
     public List<GenericMetrics> metrics;
 
+    public GenericNode() { }
+
+    GenericNode(Long timestamp, List<GenericMetrics> metrics) {
+        this.timestamp = timestamp;
+        this.metrics = metrics;
+    }
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java
index 89bba514ebb..bd3dbf935ed 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericService.java
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
@@ -34,9 +35,25 @@ public class GenericService {
     @JsonProperty("metrics")
     public List<GenericMetrics> metrics;
 
+    public GenericService() { }
+
+    GenericService(String name, Long timestamp, List<GenericMetrics> metrics) {
+        this.name = name;
+        this.timestamp = timestamp;
+        status = new Status("up");
+        this.metrics = metrics;
+    }
+
+
     @JsonInclude(NON_EMPTY)
     @JsonPropertyOrder({ "code", "description" })
     public static class Status {
+        public Status() { }
+
+        Status(String code) {
+            this.code = code;
+        }
+
         @JsonProperty("code")
         public String code;
 
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
index eff8d31f47c..4bd9d4758c4 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModelTest.java
@@ -4,14 +4,21 @@
 
 package ai.vespa.metricsproxy.metric.model.json;
 
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.List;
 
 import static ai.vespa.metricsproxy.TestUtil.getContents;
+import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
 import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 /**
  * @author gjoranv
@@ -36,6 +43,42 @@ public class GenericJsonModelTest {
         assertEquals(expected, trimmed);
     }
 
+    @Test
+    public void metrics_packets_can_be_converted_to_generic_json_model() throws Exception {
+        var nodePacket = new MetricsPacket.Builder(VESPA_NODE_SERVICE_ID)
+                .timestamp(123456L)
+                .putMetric(toMetricId("node-metric"), 1.234)
+                .putDimension(toDimensionId("node-dim"), "node-dim-value")
+                .build();
+
+        var servicePacket = new MetricsPacket.Builder(toServiceId("my-service"))
+                .timestamp(123456L)
+                .putMetric(toMetricId("service-metric"), 1234)
+                .putDimension(toDimensionId("service-dim"), "service-dim-value")
+                .build();
+
+        var metricsPackets = List.of(servicePacket, nodePacket);
+
+        GenericJsonModel jsonModel = GenericJsonUtil.toGenericJsonModel(metricsPackets);
+
+        assertNotNull(jsonModel.node);
+        assertEquals(1, jsonModel.node.metrics.size());
+        GenericMetrics nodeMetrics = jsonModel.node.metrics.get(0);
+        assertEquals(1.234, jsonModel.node.metrics.get(0).values.get("node-metric"), 0.001d);
+        assertEquals("node-dim-value", nodeMetrics.dimensions.get("node-dim"));
+
+        assertEquals(1, jsonModel.services.size());
+        GenericService service = jsonModel.services.get(0);
+        assertEquals(1, service.metrics.size());
+        GenericMetrics serviceMetrics = service.metrics.get(0);
+        assertEquals(1234L, serviceMetrics.values.get("service-metric").longValue());
+        assertEquals("service-dim-value", serviceMetrics.dimensions.get("service-dim"));
+
+        // Visual inspection
+        System.out.println(createObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
+    }
+
+
     private GenericJsonModel genericJsonModelFromTestFile() throws IOException {
         ObjectMapper mapper = createObjectMapper();
         return mapper.readValue(getContents(TEST_FILE), GenericJsonModel.class);