Consider growth rate vs. scaling time, part 2

11 files changed, 195 insertions, 50 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index 801532562d8..2c0e0a2bdb0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -81,7 +81,7 @@ public class Autoscaler {
                                "Have measurements from " + nodesMeasured + " but require from " + clusterNodes.size());
 
 
-        var clusterTimeseries = metricsDb.getClusterTimeseries(cluster.id());
+        var clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id());
         var target = ResourceTarget.idealLoad(clusterTimeseries, clusterNodesTimeseries, currentAllocation, application);
 
         Optional<AllocatableClusterResources> bestAllocation =
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
index 6cfb0cdbd88..a09ebd31f6f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.applications.Cluster;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -47,12 +48,48 @@ public class ClusterTimeseries {
 
     /** The max query growth rate we can predict from this time-series as a fraction of the current traffic per minute */
     public double maxQueryGrowthRate() {
-        return 0.1; // default
+        if (snapshots.isEmpty()) return 0.1;
+
+        // Find the period having the highest growth rate, where total growth exceeds 30% increase
+        double maxGrowthRate = 0; // In query rate per minute
+        for (int start = 0; start < snapshots.size(); start++) {
+            for (int end = start + 1; end < snapshots.size(); end++) {
+                if (queryRateAt(end) >= queryRateAt(start) * 1.3) {
+                    Duration duration = durationBetween(start, end);
+                    if (duration.isZero()) continue;
+                    double growthRate = (queryRateAt(end) - queryRateAt(start)) / duration.toMinutes();
+                    if (growthRate > maxGrowthRate)
+                        maxGrowthRate = growthRate;
+                }
+            }
+        }
+        if (maxGrowthRate == 0) { // No periods of significant growth
+            if (durationBetween(0, snapshots.size() - 1).toHours() < 24)
+                return 0.1; //       ... because not much data
+            else
+                return 0.0; //       ... because load is stable
+        }
+        if (queryRateNow() == 0) return 0.1; // Growth not expressible as a fraction of the current rate
+        return maxGrowthRate / queryRateNow();
     }
 
     /** The current query rate as a fraction of the peak rate in this timeseries */
     public double currentQueryFractionOfMax() {
-        return 0.5; // default
+        if (snapshots.isEmpty()) return 0.5;
+        var max = snapshots.stream().mapToDouble(ClusterMetricSnapshot::queryRate).max().getAsDouble();
+        return snapshots.get(snapshots.size() - 1).queryRate() / max;
+    }
+
+    private double queryRateAt(int index) {
+        return snapshots.get(index).queryRate();
+    }
+
+    private double queryRateNow() {
+        return queryRateAt(snapshots.size() - 1);
+    }
+
+    private Duration durationBetween(int startIndex, int endIndex) {
+        return Duration.between(snapshots.get(startIndex).at(), snapshots.get(endIndex).at());
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
index c20b5c64d64..bf8d354665a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -31,7 +32,7 @@ public class MemoryMetricsDb implements MetricsDb {
     /** Metric time series by node (hostname). Each list of metric snapshots is sorted by increasing timestamp */
     private final Map<String, NodeTimeseries> nodeTimeseries = new HashMap<>();
 
-    private final Map<ClusterSpec.Id, ClusterTimeseries> clusterTimeseries = new HashMap<>();
+    private final Map<Pair<ApplicationId, ClusterSpec.Id>, ClusterTimeseries> clusterTimeseries = new HashMap<>();
 
     /** Lock all access for now since we modify lists inside a map */
     private final Object lock = new Object();
@@ -53,14 +54,20 @@ public class MemoryMetricsDb implements MetricsDb {
     }
 
     @Override
-    public void addClusterMetrics(Map<ClusterSpec.Id, ClusterMetricSnapshot> clusterMetrics) {
+    public void addClusterMetrics(ApplicationId application, Map<ClusterSpec.Id, ClusterMetricSnapshot> clusterMetrics) {
         synchronized (lock) {
             for (var value : clusterMetrics.entrySet()) {
-                add(value.getKey(), value.getValue());
+                add(application, value.getKey(), value.getValue());
             }
         }
     }
 
+    public void clearClusterMetrics(ApplicationId application, ClusterSpec.Id cluster) {
+        synchronized (lock) {
+            clusterTimeseries.remove(new Pair<>(application, cluster));
+        }
+    }
+
     @Override
     public List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames) {
         Instant startTime = nodeRepository.clock().instant().minus(period);
@@ -72,8 +79,9 @@ public class MemoryMetricsDb implements MetricsDb {
     }
 
     @Override
-    public ClusterTimeseries getClusterTimeseries(ClusterSpec.Id cluster) {
-        return clusterTimeseries.computeIfAbsent(cluster, __ -> new ClusterTimeseries(cluster, new ArrayList<>()));
+    public ClusterTimeseries getClusterTimeseries(ApplicationId application, ClusterSpec.Id cluster) {
+        return clusterTimeseries.computeIfAbsent(new Pair<>(application, cluster),
+                                                 __ -> new ClusterTimeseries(cluster, new ArrayList<>()));
     }
 
     @Override
@@ -107,9 +115,10 @@ public class MemoryMetricsDb implements MetricsDb {
         nodeTimeseries.put(hostname, timeseries.add(snapshot));
     }
 
-    private void add(ClusterSpec.Id cluster, ClusterMetricSnapshot snapshot) {
-        var existing = clusterTimeseries.computeIfAbsent(cluster, __ -> new ClusterTimeseries(cluster, new ArrayList<>()));
-        clusterTimeseries.put(cluster, existing.add(snapshot));
+    private void add(ApplicationId application, ClusterSpec.Id cluster, ClusterMetricSnapshot snapshot) {
+        var key = new Pair<>(application, cluster);
+        var existing = clusterTimeseries.computeIfAbsent(key, __ -> new ClusterTimeseries(cluster, new ArrayList<>()));
+        clusterTimeseries.put(key, existing.add(snapshot));
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
index 4becf5ca88b..568c5f88661 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -27,7 +28,7 @@ public interface MetricsDb {
     /** Adds node snapshots to this. */
     void addNodeMetrics(Collection<Pair<String, NodeMetricSnapshot>> nodeMetrics);
 
-    void addClusterMetrics(Map<ClusterSpec.Id,  ClusterMetricSnapshot> clusterMetrics);
+    void addClusterMetrics(ApplicationId application, Map<ClusterSpec.Id,  ClusterMetricSnapshot> clusterMetrics);
 
     /**
      * Returns a list with one entry for each hostname containing
@@ -42,14 +43,14 @@ public interface MetricsDb {
     }
 
     /** Returns all cluster level metric snapshots for a given cluster */
-    ClusterTimeseries getClusterTimeseries(ClusterSpec.Id clusterId);
+    ClusterTimeseries getClusterTimeseries(ApplicationId applicationId, ClusterSpec.Id clusterId);
 
     /** Must be called intermittently (as long as add is called) to gc old data */
     void gc();
 
     void close();
 
-    static MetricsDb createTestInstance(NodeRepository nodeRepository) {
+    static MemoryMetricsDb createTestInstance(NodeRepository nodeRepository) {
         return new MemoryMetricsDb(nodeRepository);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
index 4a8eeba2134..a628d610ba1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -74,7 +74,7 @@ public class MetricsResponse {
                                                                     Metric.memory.from(values),
                                                                     Metric.disk.from(values),
                                                                     (long)Metric.generation.from(values),
-                                                                Metric.inService.from(values) > 0,
+                                                                    Metric.inService.from(values) > 0,
                                                                     clusterIsStable(node.get(), applicationNodes, nodeRepository),
                                                                     Metric.queryRate.from(values))));
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 189fe815387..efa1de6bb97 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -5,6 +5,7 @@ import com.google.inject.Inject;
 import com.yahoo.collections.ListMap;
 import com.yahoo.collections.Pair;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.io.IOUtils;
 import com.yahoo.vespa.defaults.Defaults;
@@ -125,30 +126,31 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     }
 
     @Override
-    public void addClusterMetrics(Map<ClusterSpec.Id, ClusterMetricSnapshot> snapshots) {
+    public void addClusterMetrics(ApplicationId application, Map<ClusterSpec.Id, ClusterMetricSnapshot> snapshots) {
         try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), clusterTable)) {
-            addClusterMetrics(snapshots, writer);
+            addClusterMetrics(application, snapshots, writer);
         }
         catch (CairoException e) {
             if (e.getMessage().contains("Cannot read offset")) {
                 // This error seems non-recoverable
                 repair(e);
                 try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), clusterTable)) {
-                    addClusterMetrics(snapshots, writer);
+                    addClusterMetrics(application, snapshots, writer);
                 }
             }
         }
     }
 
-    private void addClusterMetrics(Map<ClusterSpec.Id, ClusterMetricSnapshot> snapshots, TableWriter writer) {
+    private void addClusterMetrics(ApplicationId applicationId, Map<ClusterSpec.Id, ClusterMetricSnapshot> snapshots, TableWriter writer) {
         for (var snapshot : snapshots.entrySet()) {
             long atMillis = adjustIfRecent(snapshot.getValue().at().toEpochMilli(), highestTimestampAdded);
             if (atMillis < highestTimestampAdded) continue; // Ignore old data
             highestTimestampAdded = atMillis;
             TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds
-            row.putStr(0, snapshot.getKey().value());
-            // (1 is timestamp)
-            row.putFloat(2, (float)snapshot.getValue().queryRate());
+            row.putStr(0, applicationId.serializedForm());
+            row.putStr(1, snapshot.getKey().value());
+            // (2 is timestamp)
+            row.putFloat(3, (float)snapshot.getValue().queryRate());
             row.append();
         }
         writer.commit();
@@ -169,10 +171,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     }
 
     @Override
-    public ClusterTimeseries getClusterTimeseries(ClusterSpec.Id clusterId) {
+    public ClusterTimeseries getClusterTimeseries(ApplicationId applicationId, ClusterSpec.Id clusterId) {
         try (SqlCompiler compiler = new SqlCompiler(engine)) {
             SqlExecutionContext context = newContext();
-            return getClusterSnapshots(clusterId, compiler, context);
+            return getClusterSnapshots(applicationId, clusterId, compiler, context);
         }
         catch (SqlException e) {
             throw new IllegalStateException("Could not read cluster timeseries data in Quest stored in " + dataDir, e);
@@ -271,7 +273,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
     private void createClusterTable(SqlExecutionContext context) {
         try (SqlCompiler compiler = new SqlCompiler(engine)) {
             compiler.compile("create table " + clusterTable +
-                             " (cluster string, at timestamp, queries_rate float)" +
+                             " (application string, cluster string, at timestamp, queries_rate float)" +
                              " timestamp(at)" +
                              "PARTITION BY DAY;",
                              context);
@@ -362,7 +364,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
         }
     }
 
-    private ClusterTimeseries getClusterSnapshots(ClusterSpec.Id cluster,
+    private ClusterTimeseries getClusterSnapshots(ApplicationId application,
+                                                  ClusterSpec.Id cluster,
                                                   SqlCompiler compiler,
                                                   SqlExecutionContext context) throws SqlException {
         String sql = "select * from " + clusterTable;
@@ -371,10 +374,12 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
             try (RecordCursor cursor = factory.getCursor(context)) {
                 Record record = cursor.getRecord();
                 while (cursor.hasNext()) {
-                    String clusterId = record.getStr(0).toString();
+                    String applicationIdString = record.getStr(0).toString();
+                    if ( ! application.serializedForm().equals(applicationIdString)) continue;
+                    String clusterId = record.getStr(1).toString();
                     if (cluster.value().equals(clusterId)) {
-                        snapshots.add(new ClusterMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(1) / 1000),
-                                                                record.getFloat(2)));
+                        snapshots.add(new ClusterMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(2) / 1000),
+                                                                record.getFloat(3)));
                     }
                 }
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
index 681ef54995f..409abeaba02 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java
@@ -49,7 +49,7 @@ public class ResourceTarget {
     }
 
     /** Create a target of achieving ideal load given a current load */
-    public static ResourceTarget idealLoad(ClusterTimeseries clusterTimeseries,
+    public static ResourceTarget  idealLoad(ClusterTimeseries clusterTimeseries,
                                            ClusterNodesTimeseries clusterNodesTimeseries,
                                            AllocatableClusterResources current,
                                            Application application) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
index 4aae14b0d2d..01ab73c20b2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
@@ -74,7 +74,7 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer {
         }
         else if (response != null) {
             metricsDb.addNodeMetrics(response.nodeMetrics());
-            metricsDb.addClusterMetrics(response.clusterMetrics());
+            metricsDb.addClusterMetrics(application, response.clusterMetrics());
         }
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
index be72b4de18f..3537294c045 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -491,6 +491,44 @@ public class AutoscalingTest {
 
     }
 
+    @Test
+    public void test_autoscaling_considers_growth_rate() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 1, 1, resources);
+        ClusterResources max = new ClusterResources(10, 1, resources);
+        AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2));
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        tester.deploy(application1, cluster1, 5, 1, resources);
+        tester.addCpuMeasurements(0.25f, 1f, 120, application1);
+
+        // (no query rate data)
+        tester.assertResources("Advice to scale up sine we assume we need 2x cpu for growth when no data",
+                               7, 1, 3,  100, 100,
+                               tester.autoscale(application1, cluster1.id(), min, max).target());
+
+        tester.setScalingDuration(application1, cluster1.id(), Duration.ofMinutes(5));
+        tester.addQueryRateMeasurements(application1, cluster1.id(),
+                                        100,
+                                        t -> 10.0 + (t < 50 ? t : 100 - t));
+        tester.assertResources("Advice to scale down since observed growth is much slower than scaling time",
+                               4, 1, 3,  100, 100,
+                               tester.autoscale(application1, cluster1.id(), min, max).target());
+
+        tester.clearQueryRateMeasurements(application1, cluster1.id());
+
+        System.out.println("The fast growth one");
+        tester.setScalingDuration(application1, cluster1.id(), Duration.ofMinutes(60));
+        tester.addQueryRateMeasurements(application1, cluster1.id(),
+                                        100,
+                                        t -> 10.0 + (t < 50 ? t * t * t : 125000 - (t - 49) * (t - 49) * (t - 49)));
+        tester.assertResources("Advice to scale up since observed growth is much faster than scaling time",
+                               10, 1, 3,  100, 100,
+                               tester.autoscale(application1, cluster1.id(), min, max).target());
+    }
+
     /**
      * This calculator subtracts the memory tax when forecasting overhead, but not when actually
      * returning information about nodes. This is allowed because the forecast is a *worst case*.
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index 6688c0f9ce8..ce3293aa518 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -20,15 +20,21 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.Nodelike;
 import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
+import com.yahoo.vespa.hosted.provision.applications.ScalingEvent;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.IntFunction;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -40,7 +46,7 @@ class AutoscalingTester {
 
     private final ProvisioningTester provisioningTester;
     private final Autoscaler autoscaler;
-    private final MetricsDb db;
+    private final MemoryMetricsDb db;
     private final MockHostResourcesCalculator hostResourcesCalculator;
 
     /** Creates an autoscaling tester with a single host type ready */
@@ -210,6 +216,41 @@ class AutoscalingTester {
         nodeRepository().applications().put(application, nodeRepository().nodes().lock(applicationId));
     }
 
+    /** Creates a single redeployment event with bogus data except for the given duration */
+    public void setScalingDuration(ApplicationId applicationId, ClusterSpec.Id clusterId, Duration duration) {
+        Application application = nodeRepository().applications().require(applicationId);
+        Cluster cluster = application.cluster(clusterId).get();
+        cluster = new Cluster(clusterId,
+                              cluster.exclusive(),
+                              cluster.minResources(),
+                              cluster.maxResources(),
+                              cluster.suggestedResources(),
+                              cluster.targetResources(),
+                              List.of(), // Remove scaling events
+                              cluster.autoscalingStatus());
+        cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(),
+                                                   0,
+                                                   clock().instant().minus(Duration.ofDays(1).minus(duration))).withCompletion(clock().instant().minus(Duration.ofDays(1))));
+        application = application.with(cluster);
+        nodeRepository().applications().put(application, nodeRepository().nodes().lock(applicationId));
+    }
+
+    /** Creates the given number of measurements, spaced 5 minutes between, using the given function */
+    public void addQueryRateMeasurements(ApplicationId application,
+                                         ClusterSpec.Id cluster,
+                                         int measurements,
+                                         IntFunction<Double> queryRate) {
+        Instant time = clock().instant();
+        for (int i = 0; i < measurements; i++) {
+            db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i))));
+            time = time.plus(Duration.ofMinutes(5));
+        }
+    }
+
+    public void clearQueryRateMeasurements(ApplicationId application, ClusterSpec.Id cluster) {
+        db.clearClusterMetrics(application, cluster);
+    }
+
     public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId,
                                                            ClusterResources min, ClusterResources max) {
         Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId))
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index 8fed8e02873..18b92fa6b0f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.io.IOUtils;
 import com.yahoo.test.ManualClock;
@@ -86,32 +87,45 @@ public class QuestMetricsDbTest {
         QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
         Instant startTime = clock.instant();
 
+        var application1 = ApplicationId.from("t1", "a1", "i1");
+        var application2 = ApplicationId.from("t1", "a2", "i1");
         var cluster1 = new ClusterSpec.Id("cluster1");
         var cluster2 = new ClusterSpec.Id("cluster2");
-        db.addClusterMetrics(Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0)));
-        db.addClusterMetrics(Map.of(cluster2, new ClusterMetricSnapshot(clock.instant(), 60.0)));
+        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0)));
+        db.addClusterMetrics(application1, Map.of(cluster2, new ClusterMetricSnapshot(clock.instant(), 60.0)));
         clock.advance(Duration.ofMinutes(1));
-        db.addClusterMetrics(Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 45.0)));
+        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 45.0)));
+        clock.advance(Duration.ofMinutes(1));
+        db.addClusterMetrics(application2, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 90.0)));
+
+        ClusterTimeseries clusterTimeseries11 = db.getClusterTimeseries(application1, cluster1);
+        assertEquals(cluster1, clusterTimeseries11.cluster());
+        assertEquals(2, clusterTimeseries11.asList().size());
+
+        ClusterMetricSnapshot snapshot111 = clusterTimeseries11.get(0);
+        assertEquals(startTime, snapshot111.at());
+        assertEquals(30, snapshot111.queryRate(), delta);
+        ClusterMetricSnapshot snapshot112 = clusterTimeseries11.get(1);
+        assertEquals(startTime.plus(Duration.ofMinutes(1)), snapshot112.at());
+        assertEquals(45, snapshot112.queryRate(), delta);
+
 
-        ClusterTimeseries clusterTimeseries1 = db.getClusterTimeseries(cluster1);
-        assertEquals(cluster1, clusterTimeseries1.cluster());
-        assertEquals(2, clusterTimeseries1.asList().size());
+        ClusterTimeseries clusterTimeseries12 = db.getClusterTimeseries(application1, cluster2);
+        assertEquals(cluster2, clusterTimeseries12.cluster());
+        assertEquals(1, clusterTimeseries12.asList().size());
 
-        ClusterMetricSnapshot snapshot11 = clusterTimeseries1.get(0);
-        assertEquals(startTime, snapshot11.at());
-        assertEquals(30, snapshot11.queryRate(), delta);
+        ClusterMetricSnapshot snapshot121 = clusterTimeseries12.get(0);
+        assertEquals(startTime, snapshot121.at());
+        assertEquals(60, snapshot121.queryRate(), delta);
 
-        ClusterMetricSnapshot snapshot12 = clusterTimeseries1.get(1);
-        assertEquals(startTime.plus(Duration.ofMinutes(1)), snapshot12.at());
-        assertEquals(45, snapshot12.queryRate(), delta);
 
-        ClusterTimeseries clusterTimeseries2 = db.getClusterTimeseries(cluster2);
-        assertEquals(cluster2, clusterTimeseries2.cluster());
-        assertEquals(1, clusterTimeseries2.asList().size());
+        ClusterTimeseries clusterTimeseries21 = db.getClusterTimeseries(application2, cluster1);
+        assertEquals(cluster1, clusterTimeseries21.cluster());
+        assertEquals(1, clusterTimeseries21.asList().size());
 
-        ClusterMetricSnapshot snapshot21 = clusterTimeseries2.get(0);
-        assertEquals(startTime, snapshot21.at());
-        assertEquals(60, snapshot21.queryRate(), delta);
+        ClusterMetricSnapshot snapshot211 = clusterTimeseries21.get(0);
+        assertEquals(startTime.plus(Duration.ofMinutes(2)), snapshot211.at());
+        assertEquals(90, snapshot211.queryRate(), delta);
     }
 
     @Test