Query by duration into the past instead of startTime

11 files changed, 61 insertions, 32 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index 4ab6f259374..9c87b13c244 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -68,7 +68,7 @@ public class Autoscaler {
             return Advice.dontScale("Won't autoscale now: Less than " + scalingWindow + " since last rescaling");
 
         ClusterTimeseries clusterTimeseries =
-                new ClusterTimeseries(nodeRepository.clock().instant().minus(scalingWindow), cluster, clusterNodes, metricsDb);
+                new ClusterTimeseries(scalingWindow, cluster, clusterNodes, metricsDb);
         AllocatableClusterResources currentAllocation =
                 new AllocatableClusterResources(clusterNodes.asList(), nodeRepository, cluster.exclusive());
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
index e1a3ceca033..147335753eb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
@@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.applications.Cluster;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
 import java.util.function.Predicate;
@@ -22,9 +23,9 @@ public class ClusterTimeseries {
     /** The measurements for all nodes in this snapshot */
     private final List<NodeTimeseries> allTimeseries;
 
-    public ClusterTimeseries(Instant startTime, Cluster cluster, NodeList clusterNodes, MetricsDb db) {
+    public ClusterTimeseries(Duration period, Cluster cluster, NodeList clusterNodes, MetricsDb db) {
         this.clusterNodes = clusterNodes;
-        var timeseries = db.getNodeTimeseries(startTime, clusterNodes);
+        var timeseries = db.getNodeTimeseries(period, clusterNodes);
 
         if (cluster.lastScalingEvent().isPresent())
             timeseries = filter(timeseries, snapshot -> snapshot.generation() < 0 || // Content nodes do not yet send generation
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
index 45173650d60..1b1e5933604 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
@@ -5,6 +5,7 @@ import com.yahoo.collections.Pair;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,7 +46,8 @@ public class MemoryMetricsDb implements MetricsDb {
     }
 
     @Override
-    public List<NodeTimeseries> getNodeTimeseries(Instant startTime, Set<String> hostnames) {
+    public List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames) {
+        Instant startTime = nodeRepository.clock().instant().minus(period);
         synchronized (lock) {
             return hostnames.stream()
                             .map(hostname -> db.getOrDefault(hostname, new NodeTimeseries(hostname, List.of())).justAfter(startTime))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
index 68acdcc88f7..6fdc87f2448 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Clock;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
@@ -26,11 +27,13 @@ public interface MetricsDb {
     /**
      * Returns a list with one entry for each hostname containing
      * the snapshots recorded after the given time (or an empty snapshot if none).
+     *
+     * @param period the duration into the past to return data for
      */
-    List<NodeTimeseries> getNodeTimeseries(Instant startTime, Set<String> hostnames);
+    List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames);
 
-    default List<NodeTimeseries> getNodeTimeseries(Instant startTime, NodeList nodes) {
-        return getNodeTimeseries(startTime, nodes.stream().map(Node::hostname).collect(Collectors.toSet()));
+    default List<NodeTimeseries> getNodeTimeseries(Duration period, NodeList nodes) {
+        return getNodeTimeseries(period, nodes.stream().map(Node::hostname).collect(Collectors.toSet()));
     }
 
     /** Must be called intermittently (as long as add is called) to gc old data */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 8e90294b4a5..81401f9f743 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -116,10 +116,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     }
 
     @Override
-    public List<NodeTimeseries> getNodeTimeseries(Instant startTime, Set<String> hostnames) {
+    public List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames) {
         try (SqlCompiler compiler = new SqlCompiler(engine)) {
             SqlExecutionContext context = newContext();
-            var snapshots = getSnapshots(startTime, hostnames, compiler, context);
+            var snapshots = getSnapshots(clock.instant().minus(period), hostnames, compiler, context);
             return snapshots.entrySet().stream()
                             .map(entry -> new NodeTimeseries(entry.getKey(), entry.getValue()))
                             .collect(Collectors.toList());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 1d2938742aa..171a42f3cf1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -108,7 +108,8 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
                         .anyMatch(node -> node.history().hasEventAt(History.Event.Type.retired, event.at())))
             return cluster;
         // - 2. all nodes have switched to the right config generation
-        for (NodeTimeseries nodeTimeseries : metricsDb.getNodeTimeseries(event.at(), clusterNodes)) {
+        for (NodeTimeseries nodeTimeseries : metricsDb.getNodeTimeseries(Duration.between(event.at(), clock().instant()),
+                                                                         clusterNodes)) {
             Optional<MetricSnapshot> firstOnNewGeneration =
                     nodeTimeseries.asList().stream()
                                            .filter(snapshot -> snapshot.generation() >= event.generation()).findFirst();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
index f2be6308fd3..9206f80080b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.provision.applications.Application;
 import com.yahoo.vespa.hosted.provision.applications.Cluster;
 import com.yahoo.vespa.hosted.provision.applications.ScalingEvent;
 import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources;
+import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb;
 
 import java.net.URI;
 import java.util.Collection;
@@ -20,26 +21,33 @@ import java.util.List;
  */
 public class ApplicationSerializer {
 
-    public static Slime toSlime(Application application, List<Node> applicationNodes, URI applicationUri) {
+    public static Slime toSlime(Application application, List<Node> applicationNodes, MetricsDb metricsDb, URI applicationUri) {
         Slime slime = new Slime();
-        toSlime(application, applicationNodes, slime.setObject(), applicationUri);
+        toSlime(application, applicationNodes, metricsDb, slime.setObject(), applicationUri);
         return slime;
     }
 
     private static void toSlime(Application application,
                                 List<Node> applicationNodes,
+                                MetricsDb metricsDb,
                                 Cursor object,
                                 URI applicationUri) {
         object.setString("url", applicationUri.toString());
         object.setString("id", application.id().toFullString());
-        clustersToSlime(application.clusters().values(), applicationNodes, object.setObject("clusters"));
+        clustersToSlime(application.clusters().values(), applicationNodes, metricsDb, object.setObject("clusters"));
     }
 
-    private static void clustersToSlime(Collection<Cluster> clusters, List<Node> applicationNodes, Cursor clustersObject) {
-        clusters.forEach(cluster -> toSlime(cluster, applicationNodes, clustersObject));
+    private static void clustersToSlime(Collection<Cluster> clusters,
+                                        List<Node> applicationNodes,
+                                        MetricsDb metricsDb,
+                                        Cursor clustersObject) {
+        clusters.forEach(cluster -> toSlime(cluster, applicationNodes, metricsDb, clustersObject));
     }
 
-    private static void toSlime(Cluster cluster, List<Node> applicationNodes, Cursor clustersObject) {
+    private static void toSlime(Cluster cluster,
+                                List<Node> applicationNodes,
+                                MetricsDb metricsDb,
+                                Cursor clustersObject) {
         NodeList nodes = NodeList.copyOf(applicationNodes).not().retired().cluster(cluster.id());
         if (nodes.isEmpty()) return;
         ClusterResources currentResources = nodes.toResources();
@@ -51,6 +59,7 @@ public class ApplicationSerializer {
         if (cluster.shouldSuggestResources(currentResources))
             cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested")));
         cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target")));
+        //toSlime(metricsDb.getNodeTimeseries(NodeList.copyOf(nodes),))
         scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents"));
         clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus());
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
index c4c11222702..102018763ed 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb;
 import com.yahoo.vespa.hosted.provision.node.Address;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
@@ -71,14 +72,16 @@ public class NodesV2ApiHandler extends LoggingRequestHandler {
 
     private final Orchestrator orchestrator;
     private final NodeRepository nodeRepository;
+    private final MetricsDb metricsDb;
     private final NodeFlavors nodeFlavors;
 
     @Inject
     public NodesV2ApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator,
-                             NodeRepository nodeRepository, NodeFlavors flavors) {
+                             NodeRepository nodeRepository, MetricsDb metricsDb, NodeFlavors flavors) {
         super(parentCtx);
         this.orchestrator = orchestrator;
         this.nodeRepository = nodeRepository;
+        this.metricsDb = metricsDb;
         this.nodeFlavors = flavors;
     }
 
@@ -436,6 +439,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler {
             return ErrorResponse.notFoundError("No application '" + id + "'");
         Slime slime = ApplicationSerializer.toSlime(application.get(),
                                                     nodeRepository.nodes().list(Node.State.active).owner(id).asList(),
+                                                    metricsDb,
                                                     withPath("/nodes/v2/applications/" + id, uri));
         return new SlimeJsonResponse(slime);
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
index 48c90125ac8..02d258313db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
@@ -48,9 +48,9 @@ public class NodeMetricsDbTest {
         // Avoid off-by-one bug when the below windows starts exactly on one of the above getEpochSecond() timestamps.
         clock.advance(Duration.ofMinutes(1));
 
-        assertEquals(35, measurementCount(db.getNodeTimeseries(clock.instant().minus(Duration.ofHours(72)), Set.of(node0))));
+        assertEquals(35, measurementCount(db.getNodeTimeseries(Duration.ofHours(72), Set.of(node0))));
         db.gc();
-        assertEquals(23, measurementCount(db.getNodeTimeseries(clock.instant().minus(Duration.ofHours(72)), Set.of(node0))));
+        assertEquals(23, measurementCount(db.getNodeTimeseries(Duration.ofHours(72), Set.of(node0))));
     }
 
     private int measurementCount(List<NodeTimeseries> measurements) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index 1f61faed735..6090d2d5f34 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -43,7 +43,8 @@ public class QuestMetricsDbTest {
         clock.advance(Duration.ofSeconds(1));
 
         // Read all of one host
-        List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                                    Set.of("host1"));
         assertEquals(1, nodeTimeSeries1.size());
         assertEquals("host1", nodeTimeSeries1.get(0).hostname());
         assertEquals(1000, nodeTimeSeries1.get(0).size());
@@ -55,14 +56,15 @@ public class QuestMetricsDbTest {
         assertEquals(1, snapshot.generation(), delta);
 
         // Read all from 2 hosts
-        List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host2", "host3"));
+        List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                                    Set.of("host2", "host3"));
         assertEquals(2, nodeTimeSeries2.size());
         assertEquals(Set.of("host2", "host3"), nodeTimeSeries2.stream().map(ts -> ts.hostname()).collect(Collectors.toSet()));
         assertEquals(1000, nodeTimeSeries2.get(0).size());
         assertEquals(1000, nodeTimeSeries2.get(1).size());
 
         // Read a short interval from 3 hosts
-        List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(clock.instant().minus(Duration.ofSeconds(3)),
+        List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(Duration.ofSeconds(3),
                                                                     Set.of("host1", "host2", "host3"));
         assertEquals(3, nodeTimeSeries3.size());
         assertEquals(Set.of("host1", "host2", "host3"), nodeTimeSeries3.stream().map(ts -> ts.hostname()).collect(Collectors.toSet()));
@@ -83,15 +85,18 @@ public class QuestMetricsDbTest {
         db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3"));
         clock.advance(Duration.ofSeconds(1));
 
-        List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                                    Set.of("host1"));
         assertEquals(10, nodeTimeSeries1.get(0).size());
 
         db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3"));
-        List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                                    Set.of("host1"));
         assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size());
 
         db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3"));
-        List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                                    Set.of("host1"));
         assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size());
     }
 
@@ -107,11 +112,14 @@ public class QuestMetricsDbTest {
         clock.advance(Duration.ofHours(dayOffset));
         db.add(timeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3"));
 
-        assertEquals(24 * 10, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size());
+        assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                   Set.of("host1")).get(0).size());
         db.gc();
-        assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size());
+        assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                              Set.of("host1")).get(0).size());
         db.gc(); // no-op
-        assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(startTime, Set.of("host1")).get(0).size());
+        assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                                              Set.of("host1")).get(0).size());
     }
 
     /** To manually test that we can read existing data */
@@ -128,7 +136,7 @@ public class QuestMetricsDbTest {
         clock.advance(Duration.ofSeconds(9)); // Adjust to last data written
         QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
 
-        List<NodeTimeseries> timeseries = db.getNodeTimeseries(clock.instant().minus(Duration.ofSeconds(9)), Set.of("host1"));
+        List<NodeTimeseries> timeseries = db.getNodeTimeseries(Duration.ofSeconds(9), Set.of("host1"));
         assertFalse("Could read existing data", timeseries.isEmpty());
         assertEquals(10, timeseries.get(0).size());
 
@@ -139,7 +147,7 @@ public class QuestMetricsDbTest {
         clock.advance(Duration.ofSeconds(1));
         db.add(timeseries(2, Duration.ofSeconds(1), clock, "host1"));
         System.out.println("New data written and read:");
-        timeseries = db.getNodeTimeseries(clock.instant().minus(Duration.ofSeconds(2)), Set.of("host1"));
+        timeseries = db.getNodeTimeseries(Duration.ofSeconds(2), Set.of("host1"));
         for (var snapshot : timeseries.get(0).asList())
             System.out.println("  " + snapshot);
     }
@@ -156,7 +164,8 @@ public class QuestMetricsDbTest {
         Instant startTime = clock.instant();
         db.add(timeseries(10, Duration.ofSeconds(1), clock, "host1"));
 
-        int added = db.getNodeTimeseries(startTime, Set.of("host1")).get(0).asList().size();
+        int added = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+                                         Set.of("host1")).get(0).asList().size();
         System.out.println("Added " + added + " rows of data");
         db.close();
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
index 0a12a255f30..e99f7740c29 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
@@ -46,7 +46,7 @@ public class NodeMetricsDbMaintainerTest {
                                                                          Duration.ofHours(1),
                                                                          new TestMetric());
         assertTrue(maintainer.maintain());
-        List<NodeTimeseries> timeseriesList = db.getNodeTimeseries(Instant.ofEpochMilli(0),
+        List<NodeTimeseries> timeseriesList = db.getNodeTimeseries(Duration.ofDays(1),
                                                                    Set.of("host-1.yahoo.com", "host-2.yahoo.com"));
         assertEquals(2, timeseriesList.size());
         List<MetricSnapshot> allSnapshots = timeseriesList.stream()