Using the jackson ObjectMapper directly at the rootlevel causes the whole object structure to be build in memory.

A compromise here is to drive the parsing with the stream api and use an object mapper per entry in the array.

1 files changed, 110 insertions, 57 deletions

diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java
index 464f215edc4..54976f70e55 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java
@@ -4,9 +4,10 @@ package ai.vespa.metricsproxy.service;
 import ai.vespa.metricsproxy.metric.Metric;
 import ai.vespa.metricsproxy.metric.Metrics;
 import ai.vespa.metricsproxy.metric.model.DimensionId;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -59,76 +60,128 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher {
         return remoteMetrics;
     }
 
-    private Metrics parse(String data) throws IOException {
-        JsonNode o = jsonMapper.readTree(data);
-        if (!(o.has("metrics"))) {
-            return new Metrics(); //empty
+    private Metrics parseSnapshot(JsonParser parser) throws IOException {
+        if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
+            throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken());
         }
+        Metrics metrics = new Metrics();
+        for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) {
+            String fieldName = parser.getCurrentName();
+            JsonToken token = parser.nextToken();
+            if (fieldName.equals("to")) {
+                long timestamp = parser.getLongValue();
+                long now = System.currentTimeMillis() / 1000;
+                timestamp = Metric.adjustTime(timestamp, now);
+                metrics = new Metrics(timestamp);
+            } else {
+                if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
+                    parser.skipChildren();
+                }
+            }
+        }
+        return metrics;
+    }
 
-        JsonNode metrics = o.get("metrics");
-        ArrayNode values;
-        long timestamp;
-
-        try {
-            JsonNode snapshot = metrics.get("snapshot");
-            timestamp = snapshot.get("to").asLong();
-            values = (ArrayNode) metrics.get("values");
-        } catch (Exception e) {
-            // snapshot might not have been produced. Do not throw exception into log
-            return new Metrics();
+    private void parseValues(JsonParser parser, Metrics metrics) throws IOException {
+        if (parser.getCurrentToken() != JsonToken.START_ARRAY) {
+            throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken());
         }
-        long now = System.currentTimeMillis() / 1000;
-        timestamp = Metric.adjustTime(timestamp, now);
-        Metrics m = new Metrics(timestamp);
 
-        Map<DimensionId, String> noDims = Collections.emptyMap();
         Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>();
-        for (int i = 0; i < values.size(); i++) {
-            JsonNode metric = values.get(i);
-            String name = metric.get("name").textValue();
-            String description = "";
+        while (parser.nextToken() == JsonToken.START_OBJECT) {
+            // read everything from this START_OBJECT to the matching END_OBJECT
+            // and return it as a tree model ObjectNode
+            JsonNode value = jsonMapper.readTree(parser);
+            handleValue(value, metrics.getTimeStamp(), metrics, uniqueDimensions);
 
-            if (metric.has("description")) {
-                description = metric.get("description").textValue();
-            }
+            // do whatever you need to do with this object
+        }
+    }
 
-            Map<DimensionId, String> dim = noDims;
-            if (metric.has("dimensions")) {
-                JsonNode dimensions = metric.get("dimensions");
-                StringBuilder sb = new StringBuilder();
-                for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) {
-                    String k = (String) it.next();
-                    String v = dimensions.get(k).asText();
-                    sb.append(toDimensionId(k)).append(v);
-                }
-                if ( ! uniqueDimensions.containsKey(sb.toString())) {
-                    dim = new HashMap<>();
-                    for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) {
-                        String k = (String) it.next();
-                        String v = dimensions.get(k).textValue();
-                        dim.put(toDimensionId(k), v);
-                    }
-                    uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim));
+    private Metrics parseMetrics(JsonParser parser) throws IOException {
+        if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
+            throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken());
+        }
+        Metrics metrics = new Metrics();
+        for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) {
+            String fieldName = parser.getCurrentName();
+            JsonToken token = parser.nextToken();
+            if (fieldName.equals("snapshot")) {
+                metrics = parseSnapshot(parser);
+            } else if (fieldName.equals("values")) {
+                parseValues(parser, metrics);
+            } else {
+                if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
+                    parser.skipChildren();
                 }
-                dim = uniqueDimensions.get(sb.toString());
             }
+        }
+        return metrics;
+    }
+    private Metrics parse(String data) throws IOException {
+        JsonParser parser = jsonMapper.createParser(data);
+
+        if (parser.nextToken() != JsonToken.START_OBJECT) {
+            throw new IOException("Expected start of object, got " + parser.currentToken());
+        }
 
-            JsonNode aggregates = metric.get("values");
-            for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) {
-                String aggregator = (String) it.next();
-                JsonNode aggregatorValue = aggregates.get(aggregator);
-                if (aggregatorValue == null) {
-                    throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing");
+        Metrics metrics  = new Metrics();
+        for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) {
+            String fieldName = parser.getCurrentName();
+            JsonToken token = parser.nextToken();
+            if (fieldName.equals("metrics")) {
+                metrics = parseMetrics(parser);
+            } else {
+                if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
+                    parser.skipChildren();
                 }
-                Number value = aggregatorValue.numberValue();
-                if (value == null) {
-                    throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number");
+            }
+        }
+        return metrics;
+    }
+
+    private void handleValue(JsonNode metric, long timestamp, Metrics metrics, Map<String, Map<DimensionId, String>> uniqueDimensions) {
+        String name = metric.get("name").textValue();
+        String description = "";
+
+        if (metric.has("description")) {
+            description = metric.get("description").textValue();
+        }
+
+        Map<DimensionId, String> dim = Collections.emptyMap();
+        if (metric.has("dimensions")) {
+            JsonNode dimensions = metric.get("dimensions");
+            StringBuilder sb = new StringBuilder();
+            for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) {
+                String k = (String) it.next();
+                String v = dimensions.get(k).asText();
+                sb.append(toDimensionId(k)).append(v);
+            }
+            if ( ! uniqueDimensions.containsKey(sb.toString())) {
+                dim = new HashMap<>();
+                for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) {
+                    String k = (String) it.next();
+                    String v = dimensions.get(k).textValue();
+                    dim.put(toDimensionId(k), v);
                 }
-                StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator);
-                m.add(new Metric(metricName.toString(), value, timestamp, dim, description));
+                uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim));
             }
+            dim = uniqueDimensions.get(sb.toString());
         }
 
-        return m;
+        JsonNode aggregates = metric.get("values");
+        for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) {
+            String aggregator = (String) it.next();
+            JsonNode aggregatorValue = aggregates.get(aggregator);
+            if (aggregatorValue == null) {
+                throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing");
+            }
+            Number value = aggregatorValue.numberValue();
+            if (value == null) {
+                throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number");
+            }
+            StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator);
+            metrics.add(new Metric(metricName.toString(), value, timestamp, dim, description));
+        }
     }
 }