diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-03-08 22:37:49 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-03-08 22:37:49 +0100 |
commit | b04ec57706b4d30711f76ebb8cb7fc8c8469b0a9 (patch) | |
tree | 8b4f64573e88623b1f9d19d2e0e7835613e8eb06 /metrics-proxy/src | |
parent | 396733d1eebdc920e1c312927064116fd7f727f3 (diff) |
Using the jackson ObjectMapper directly at the rootlevel causes the whole object structure to be build in memory.
A compromise here is to drive the parsing with the stream api and use an object mapper per entry in the array.
Diffstat (limited to 'metrics-proxy/src')
-rw-r--r-- | metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java | 167 |
1 files changed, 110 insertions, 57 deletions
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 464f215edc4..54976f70e55 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -4,9 +4,10 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.DimensionId; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; import java.util.Collections; @@ -59,76 +60,128 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { return remoteMetrics; } - private Metrics parse(String data) throws IOException { - JsonNode o = jsonMapper.readTree(data); - if (!(o.has("metrics"))) { - return new Metrics(); //empty + private Metrics parseSnapshot(JsonParser parser) throws IOException { + if (parser.getCurrentToken() != JsonToken.START_OBJECT) { + throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken()); } + Metrics metrics = new Metrics(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (fieldName.equals("to")) { + long timestamp = parser.getLongValue(); + long now = System.currentTimeMillis() / 1000; + timestamp = Metric.adjustTime(timestamp, now); + metrics = new Metrics(timestamp); + } else { + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + parser.skipChildren(); + } + } + } + return metrics; + } - JsonNode metrics = o.get("metrics"); - ArrayNode values; - long timestamp; - - try { - JsonNode snapshot = metrics.get("snapshot"); - timestamp = snapshot.get("to").asLong(); - values = (ArrayNode) metrics.get("values"); - } catch (Exception e) { - // snapshot might not have been produced. Do not throw exception into log - return new Metrics(); + private void parseValues(JsonParser parser, Metrics metrics) throws IOException { + if (parser.getCurrentToken() != JsonToken.START_ARRAY) { + throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken()); } - long now = System.currentTimeMillis() / 1000; - timestamp = Metric.adjustTime(timestamp, now); - Metrics m = new Metrics(timestamp); - Map<DimensionId, String> noDims = Collections.emptyMap(); Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); - for (int i = 0; i < values.size(); i++) { - JsonNode metric = values.get(i); - String name = metric.get("name").textValue(); - String description = ""; + while (parser.nextToken() == JsonToken.START_OBJECT) { + // read everything from this START_OBJECT to the matching END_OBJECT + // and return it as a tree model ObjectNode + JsonNode value = jsonMapper.readTree(parser); + handleValue(value, metrics.getTimeStamp(), metrics, uniqueDimensions); - if (metric.has("description")) { - description = metric.get("description").textValue(); - } + // do whatever you need to do with this object + } + } - Map<DimensionId, String> dim = noDims; - if (metric.has("dimensions")) { - JsonNode dimensions = metric.get("dimensions"); - StringBuilder sb = new StringBuilder(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).asText(); - sb.append(toDimensionId(k)).append(v); - } - if ( ! uniqueDimensions.containsKey(sb.toString())) { - dim = new HashMap<>(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).textValue(); - dim.put(toDimensionId(k), v); - } - uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); + private Metrics parseMetrics(JsonParser parser) throws IOException { + if (parser.getCurrentToken() != JsonToken.START_OBJECT) { + throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken()); + } + Metrics metrics = new Metrics(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (fieldName.equals("snapshot")) { + metrics = parseSnapshot(parser); + } else if (fieldName.equals("values")) { + parseValues(parser, metrics); + } else { + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + parser.skipChildren(); } - dim = uniqueDimensions.get(sb.toString()); } + } + return metrics; + } + private Metrics parse(String data) throws IOException { + JsonParser parser = jsonMapper.createParser(data); + + if (parser.nextToken() != JsonToken.START_OBJECT) { + throw new IOException("Expected start of object, got " + parser.currentToken()); + } - JsonNode aggregates = metric.get("values"); - for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) { - String aggregator = (String) it.next(); - JsonNode aggregatorValue = aggregates.get(aggregator); - if (aggregatorValue == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing"); + Metrics metrics = new Metrics(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (fieldName.equals("metrics")) { + metrics = parseMetrics(parser); + } else { + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + parser.skipChildren(); } - Number value = aggregatorValue.numberValue(); - if (value == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number"); + } + } + return metrics; + } + + private void handleValue(JsonNode metric, long timestamp, Metrics metrics, Map<String, Map<DimensionId, String>> uniqueDimensions) { + String name = metric.get("name").textValue(); + String description = ""; + + if (metric.has("description")) { + description = metric.get("description").textValue(); + } + + Map<DimensionId, String> dim = Collections.emptyMap(); + if (metric.has("dimensions")) { + JsonNode dimensions = metric.get("dimensions"); + StringBuilder sb = new StringBuilder(); + for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { + String k = (String) it.next(); + String v = dimensions.get(k).asText(); + sb.append(toDimensionId(k)).append(v); + } + if ( ! uniqueDimensions.containsKey(sb.toString())) { + dim = new HashMap<>(); + for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { + String k = (String) it.next(); + String v = dimensions.get(k).textValue(); + dim.put(toDimensionId(k), v); } - StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator); - m.add(new Metric(metricName.toString(), value, timestamp, dim, description)); + uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); } + dim = uniqueDimensions.get(sb.toString()); } - return m; + JsonNode aggregates = metric.get("values"); + for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) { + String aggregator = (String) it.next(); + JsonNode aggregatorValue = aggregates.get(aggregator); + if (aggregatorValue == null) { + throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing"); + } + Number value = aggregatorValue.numberValue(); + if (value == null) { + throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number"); + } + StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator); + metrics.add(new Metric(metricName.toString(), value, timestamp, dim, description)); + } } } |