Parse metrics/v2 response and store in batch

11 files changed, 130 insertions, 40 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
new file mode 100644
index 00000000000..f3ce82d3bf9
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -0,0 +1,69 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values
+ *
+ * @author bratseth
+ */
+public class MetricsResponse {
+
+    private final List<NodeMetrics.MetricValue> metricValues = new ArrayList<>();
+
+    public MetricsResponse(byte[] response) {
+        this(SlimeUtils.jsonToSlime(response));
+    }
+
+    public MetricsResponse(String response) {
+        this(SlimeUtils.jsonToSlime(response));
+    }
+
+    public List<NodeMetrics.MetricValue> metrics() { return metricValues; }
+
+    private MetricsResponse(Slime response) {
+        Inspector root = response.get();
+        Inspector nodes = root.field("nodes");
+        nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node));
+    }
+
+    private void consumeNode(Inspector node) {
+        String hostname = node.field("hostname").asString();
+        consumeNodeMetrics(hostname, node.field("node"));
+        consumeServiceMetrics(hostname, node.field("services"));
+    }
+
+    private void consumeNodeMetrics(String hostname, Inspector node) {
+        long timestamp = node.field("timestamp").asLong();
+        Map<String, Double> values = consumeMetrics(node.field("metrics"));
+        addMetricIfPresent(hostname, "cpu.util", timestamp, values);
+    }
+
+    private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) {
+        if (values.containsKey(metricName))
+            metricValues.add(new NodeMetrics.MetricValue(hostname, "cpu.util", timestamp, values.get("cpu.util").floatValue()));
+    }
+
+    private void consumeServiceMetrics(String hostname, Inspector node) {
+        String name = node.field("name").asString();
+        long timestamp = node.field("timestamp").asLong();
+        Map<String, Double> values = consumeMetrics(node.field("metrics"));
+    }
+
+    private Map<String, Double> consumeMetrics(Inspector metrics) {
+        Map<String, Double> values = new HashMap<>();
+        metrics.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble()));
+        return values;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
index 09a3ff789cc..38a2194319a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
@@ -1,6 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.autoscale;
 
+import com.yahoo.config.provision.ApplicationId;
+
 import java.util.Collection;
 
 /**
@@ -11,24 +13,29 @@ import java.util.Collection;
 public interface NodeMetrics {
 
     /**
-     * Fetches node metrics for a node. This call may be expensive.
+     * Fetches metrics for an application. This call may be expensive.
      *
-     * @param hostname the hostname of the node to fetch metrics from
+     * @param application the application to fetch metrics from
      */
-    Collection<Metric> fetchMetrics(String hostname);
+    Collection<MetricValue> fetchMetrics(ApplicationId application);
 
-    final class Metric {
+    final class MetricValue {
 
-        private String name;
-        private float value;
+        private final String hostname;
+        private final String name;
+        private long timestamp;
+        private final float value;
 
-        public Metric(String name, float value) {
+        public MetricValue(String hostname, String name, long timestamp, float value) {
+            this.hostname = hostname;
             this.name = name;
+            this.timestamp = timestamp;
             this.value = value;
         }
 
+        public String hostname() { return hostname; }
         public String name() { return name; }
-
+        public long timestamp() { return timestamp; }
         public float value() { return value; }
 
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
index d101bd2ce91..ebea8c3c9ef 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
@@ -1,12 +1,11 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.autoscale;
 
-import com.yahoo.vespa.hosted.provision.Node;
-
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -31,10 +30,14 @@ public class NodeMetricsDb {
     private final Object lock = new Object();
 
     /** Add a measurement to this */
-    public void add(String hostname, Resource resource, Instant timestamp, float value) {
+    public void add(Collection<NodeMetrics.MetricValue> metricValues) {
         synchronized (lock) {
-            List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(hostname, resource), (__) -> new ArrayList<>());
-            measurements.add(new Measurement(timestamp.toEpochMilli(), value));
+            for (var value : metricValues) {
+                List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(),
+                                                                                       Resource.fromMetric(value.name())),
+                                                                    (__) -> new ArrayList<>());
+                measurements.add(new Measurement(value.timestamp(), value.value()));
+            }
         }
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsHttpFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsHttpFetcher.java
index 0993cd73b72..8c9a4da806f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsHttpFetcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsHttpFetcher.java
@@ -1,7 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.autoscale;
 
-import java.util.ArrayList;
+import com.yahoo.config.provision.ApplicationId;
+
 import java.util.Collection;
 
 /**
@@ -11,9 +12,12 @@ import java.util.Collection;
  */
 public class NodeMetricsHttpFetcher implements NodeMetrics {
 
+    private static final String apiPath = "/metrics/v2";
+
     @Override
-    public Collection<Metric> fetchMetrics(String hostname) {
-        return new ArrayList<>();
+    public Collection<MetricValue> fetchMetrics(ApplicationId application) {
+        String response = ""; // TODO
+        return new MetricsResponse(response).metrics();
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
index 842f2b1f1b4..53f5493c321 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
@@ -11,7 +11,7 @@ import com.yahoo.config.provision.NodeResources;
 public enum Resource {
 
     cpu {
-        String metric() { return "cpu"; } // TODO: Full metric name
+        String metric() { return "cpu.util"; }
         double idealAverageLoad() { return 0.2; }
         double valueFrom(NodeResources resources) { return resources.vcpu(); }
     },
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 1fef2717281..68b6858c477 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Deployer;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
@@ -42,7 +41,7 @@ public class AutoscalingMaintainer extends Maintainer {
     protected void maintain() {
         if ( ! nodeRepository().zone().environment().isProduction()) return;
 
-        nodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes));
+        activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes));
     }
 
     private void autoscale(ApplicationId application, List<Node> applicationNodes) {
@@ -56,11 +55,6 @@ public class AutoscalingMaintainer extends Maintainer {
         });
     }
 
-    private Map<ApplicationId, List<Node>> nodesByApplication() {
-        return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
-                               .stream().collect(Collectors.groupingBy(n -> n.allocation().get().owner()));
-    }
-
     private Map<ClusterSpec, List<Node>> nodesByCluster(List<Node> applicationNodes) {
         return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster()));
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index 0d5a8587902..27fba9e8f8e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -2,17 +2,22 @@
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
@@ -75,6 +80,12 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     
     private String name() { return this.getClass().getSimpleName(); }
 
+    /** A utility to group active tenant applications by application */
+    protected Map<ApplicationId, List<Node>> activeNodesByApplication() {
+        return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
+                               .stream().collect(Collectors.groupingBy(n -> n.allocation().get().owner()));
+    }
+
     static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) {
         if ( ! cluster.contains(host))
             return interval.toMillis();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
index 86b4347dea7..9b5aae8fcbc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
@@ -1,16 +1,12 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
-import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
-import com.yahoo.vespa.hosted.provision.autoscale.Resource;
 
 import java.time.Duration;
-import java.time.Instant;
-import java.util.Collection;
 import java.util.logging.Level;
 
 /**
@@ -36,15 +32,13 @@ public class NodeMetricsDbMaintainer extends Maintainer {
     @Override
     protected void maintain() {
         int warnings = 0;
-        for (Node node : nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()) {
+        for (ApplicationId application : activeNodesByApplication().keySet()) {
             try {
-                Collection<NodeMetrics.Metric> metrics = nodeMetrics.fetchMetrics(node.hostname());
-                Instant timestamp = nodeRepository().clock().instant();
-                metrics.forEach(metric -> nodeMetricsDb.add(node.hostname(), Resource.fromMetric(metric.name()), timestamp, metric.value()));
+                nodeMetricsDb.add(nodeMetrics.fetchMetrics(application));
             }
             catch (Exception e) {
                 if (warnings++ < maxWarningsPerInvocation)
-                    log.log(Level.WARNING, "Could not update metrics from " + node, e); // TODO: Exclude allowed to be down nodes
+                    log.log(Level.WARNING, "Could not update metrics for " + application, e);
             }
         }
         nodeMetricsDb.gc(nodeRepository().clock());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
index a8f7cd1971a..6515ba6a3e4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
@@ -1,6 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.testutils;
 
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
 
 import java.util.ArrayList;
@@ -12,7 +13,7 @@ import java.util.Collection;
 public class MockNodeMetrics implements NodeMetrics {
 
     @Override
-    public Collection<Metric> fetchMetrics(String hostname) {
+    public Collection<MetricValue> fetchMetrics(ApplicationId application) {
         return new ArrayList<>();
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index 95d64d167df..fdcba7ba565 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -127,13 +127,17 @@ class AutoscalingTester {
                                 int count, ApplicationId applicationId) {
         List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active);
         float oneExtraNodeFactor = (float)(nodes.size() - 1.0) / (nodes.size());
-        System.out.println("Naive value " + value + ", adjusted " + value * oneExtraNodeFactor);
         for (int i = 0; i < count; i++) {
             clock().advance(Duration.ofMinutes(1));
             for (Node node : nodes) {
-                for (Resource r : Resource.values())
-                    db.add(node.hostname(), r, clock().instant(),
-                           (r == resource ? value : (float)r.idealAverageLoad() * otherResourcesLoad) * oneExtraNodeFactor);
+                for (Resource r : Resource.values()) {
+                    float effectiveValue = (r == resource ? value : (float) r.idealAverageLoad() * otherResourcesLoad)
+                                           * oneExtraNodeFactor;
+                    db.add(List.of(new NodeMetrics.MetricValue(node.hostname(),
+                                                               r.metric(),
+                                                               clock().instant().toEpochMilli(),
+                                                               effectiveValue)));
+                }
             }
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
index 348e1b295f6..93bc4e2baa8 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
@@ -5,6 +5,7 @@ import com.yahoo.test.ManualClock;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -15,10 +16,12 @@ public class NodeMetricsDbTest {
     public void testNodeMetricsDb() {
         ManualClock clock = new ManualClock();
         NodeMetricsDb db = new NodeMetricsDb();
+        List<NodeMetrics.MetricValue> values = new ArrayList<>();
         for (int i = 0; i < 40; i++) {
-            db.add("host0", Resource.cpu, clock.instant(), 0.9f);
+            values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().toEpochMilli(), 0.9f));
             clock.advance(Duration.ofHours(1));
         }
+        db.add(values);
 
         assertEquals(32, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu,    List.of("host0")).measurementCount());
         assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount());