Record query rates

12 files changed, 121 insertions, 34 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
index ddd519382a6..1be1e313945 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
@@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.application;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.OptionalDouble;
 import java.util.function.Function;
 
 /**
@@ -14,14 +16,21 @@ import java.util.function.Function;
  */
 public class ApplicationActivity {
 
-    public static final ApplicationActivity none = new ApplicationActivity(Optional.empty(), Optional.empty());
+    public static final ApplicationActivity none = new ApplicationActivity(Optional.empty(), Optional.empty(),
+                                                                           OptionalDouble.empty(),
+                                                                           OptionalDouble.empty());
 
     private final Optional<Instant> lastQueried;
     private final Optional<Instant> lastWritten;
+    private final OptionalDouble lastQueriesPerSecond;
+    private final OptionalDouble lastWritesPerSecond;
 
-    private ApplicationActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten) {
-        this.lastQueried = lastQueried;
-        this.lastWritten = lastWritten;
+    private ApplicationActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten,
+                                OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) {
+        this.lastQueried = Objects.requireNonNull(lastQueried, "lastQueried must be non-null");
+        this.lastWritten = Objects.requireNonNull(lastWritten, "lastWritten must be non-null");
+        this.lastQueriesPerSecond = Objects.requireNonNull(lastQueriesPerSecond, "lastQueriesPerSecond must be non-null");
+        this.lastWritesPerSecond = Objects.requireNonNull(lastWritesPerSecond, "lastWritesPerSecond must be non-null");
     }
 
     /** The last time any deployment in this was queried */
@@ -34,23 +43,36 @@ public class ApplicationActivity {
         return lastWritten;
     }
 
+    /** Query rate the last time this was queried */
+    public OptionalDouble lastQueriesPerSecond() {
+        return lastQueriesPerSecond;
+    }
+
+    /** Write rate the last time this was written */
+    public OptionalDouble lastWritesPerSecond() {
+        return lastWritesPerSecond;
+    }
+
     public static ApplicationActivity from(Collection<Deployment> deployments) {
-        Optional<Instant> lastQueried = lastActivity(deployments, DeploymentActivity::lastQueried);
-        Optional<Instant> lastWritten = lastActivity(deployments, DeploymentActivity::lastWritten);
-        if (!lastQueried.isPresent() && !lastWritten.isPresent()) {
+        Optional<DeploymentActivity> lastActivityByQuery = lastActivityBy(DeploymentActivity::lastQueried, deployments);
+        Optional<DeploymentActivity> lastActivityByWrite = lastActivityBy(DeploymentActivity::lastWritten, deployments);
+        if (!lastActivityByQuery.isPresent() && !lastActivityByWrite.isPresent()) {
             return none;
         }
-        return new ApplicationActivity(lastQueried, lastWritten);
+        return new ApplicationActivity(lastActivityByQuery.flatMap(DeploymentActivity::lastQueried),
+                                       lastActivityByWrite.flatMap(DeploymentActivity::lastWritten),
+                                       lastActivityByQuery.map(DeploymentActivity::lastQueriesPerSecond)
+                                                          .orElseGet(OptionalDouble::empty),
+                                       lastActivityByWrite.map(DeploymentActivity::lastWritesPerSecond)
+                                                          .orElseGet(OptionalDouble::empty));
     }
 
-    private static Optional<Instant> lastActivity(Collection<Deployment> deployments,
-                                                  Function<DeploymentActivity, Optional<Instant>> activityField) {
+    private static Optional<DeploymentActivity> lastActivityBy(Function<DeploymentActivity, Optional<Instant>> field,
+                                                               Collection<Deployment> deployments) {
         return deployments.stream()
                           .map(Deployment::activity)
-                          .map(activityField)
-                          .filter(Optional::isPresent)
-                          .map(Optional::get)
-                          .max(Comparator.naturalOrder());
+                          .filter(activity -> field.apply(activity).isPresent())
+                          .max(Comparator.comparing(activity -> field.apply(activity).get()));
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java
index d4635212e80..881de040e28 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
 import java.time.Instant;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.OptionalDouble;
 
 /**
  * Recent activity in a deployment.
@@ -15,14 +16,21 @@ public class DeploymentActivity {
     /** Query rates at or below this threshold indicate inactivity */
     private static final double inactivityThreshold = 0;
 
-    public static final DeploymentActivity none = new DeploymentActivity(Optional.empty(), Optional.empty());
+    public static final DeploymentActivity none = new DeploymentActivity(Optional.empty(), Optional.empty(),
+                                                                         OptionalDouble.empty(),
+                                                                         OptionalDouble.empty());
 
     private final Optional<Instant> lastQueried;
     private final Optional<Instant> lastWritten;
+    private final OptionalDouble lastQueriesPerSecond;
+    private final OptionalDouble lastWritesPerSecond;
 
-    private DeploymentActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten) {
+    private DeploymentActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten,
+                               OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) {
         this.lastQueried = Objects.requireNonNull(lastQueried, "lastQueried must be non-null");
         this.lastWritten = Objects.requireNonNull(lastWritten, "lastWritten must be non-null");
+        this.lastQueriesPerSecond = Objects.requireNonNull(lastQueriesPerSecond, "lastQueriesPerSecond must be non-null");
+        this.lastWritesPerSecond = Objects.requireNonNull(lastWritesPerSecond, "lastWritesPerSecond must be non-null");
     }
 
     /** The last time this deployment received queries (search) */
@@ -35,17 +43,38 @@ public class DeploymentActivity {
         return lastWritten;
     }
 
+    /** Query rate the last time this deployment received queries (search) */
+    public OptionalDouble lastQueriesPerSecond() {
+        return lastQueriesPerSecond;
+    }
+
+    /** Write rate the last time this deployment received writes (feed) */
+    public OptionalDouble lastWritesPerSecond() {
+        return lastWritesPerSecond;
+    }
+
     /** Record activity using given metrics */
     public DeploymentActivity recordAt(Instant instant, DeploymentMetrics metrics) {
         return new DeploymentActivity(activityAt(instant, lastQueried, metrics.queriesPerSecond()),
-                                      activityAt(instant, lastWritten, metrics.writesPerSecond()));
+                                      activityAt(instant, lastWritten, metrics.writesPerSecond()),
+                                      activeRate(metrics.queriesPerSecond(), lastQueriesPerSecond),
+                                      activeRate(metrics.writesPerSecond(), lastWritesPerSecond));
     }
 
-    public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt) {
+    public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt,
+                                            OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) {
         if (!queriedAt.isPresent() && !writtenAt.isPresent()) {
             return none;
         }
-        return new DeploymentActivity(queriedAt, writtenAt);
+        return new DeploymentActivity(queriedAt, writtenAt, lastQueriesPerSecond, lastWritesPerSecond);
+    }
+
+    public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt) {
+        return create(queriedAt, writtenAt, OptionalDouble.empty(), OptionalDouble.empty());
+    }
+
+    private static OptionalDouble activeRate(double newRate, OptionalDouble oldRate) {
+        return newRate > inactivityThreshold ? OptionalDouble.of(newRate) : oldRate;
     }
 
     private static Optional<Instant> activityAt(Instant newInstant, Optional<Instant> oldInstant, double rate) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 6ad2452b2a2..96d252d3d1c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -35,6 +35,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalDouble;
 import java.util.OptionalLong;
 
 /**
@@ -71,6 +72,8 @@ public class ApplicationSerializer {
     private final String commitField = "commitField";
     private final String lastQueriedField = "lastQueried";
     private final String lastWrittenField = "lastWritten";
+    private final String lastQueriesPerSecondField = "lastQueriesPerSecond";
+    private final String lastWritesPerSecondField = "lastWritesPerSecond";
 
     // DeploymentJobs fields
     private final String projectIdField = "projectId";
@@ -154,6 +157,8 @@ public class ApplicationSerializer {
         deploymentMetricsToSlime(deployment.metrics(), object);
         deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli()));
         deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli()));
+        deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value));
+        deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value));
     }
 
     private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) {
@@ -296,7 +301,9 @@ public class ApplicationSerializer {
                               clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
                               deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)),
                               DeploymentActivity.create(optionalInstant(deploymentObject.field(lastQueriedField)),
-                                                        optionalInstant(deploymentObject.field(lastWrittenField))));
+                                                        optionalInstant(deploymentObject.field(lastWrittenField)),
+                                                        optionalDouble(deploymentObject.field(lastQueriesPerSecondField)),
+                                                        optionalDouble(deploymentObject.field(lastWritesPerSecondField))));
     }
 
     private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) {
@@ -425,6 +432,10 @@ public class ApplicationSerializer {
         return field.valid() ? OptionalLong.of(field.asLong()) : OptionalLong.empty();
     }
 
+    private OptionalDouble optionalDouble(Inspector field) {
+        return field.valid() ? OptionalDouble.of(field.asDouble()) : OptionalDouble.empty();
+    }
+
     private Optional<String> optionalString(Inspector field) {
         return SlimeUtils.optionalString(field);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 3eced6d943e..edf68054481 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -427,8 +427,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         // Activity
         Cursor activity = object.setObject("activity");
-        application.activity().lastQueried().ifPresent(lastQueried -> activity.setLong("queriedAt", lastQueried.toEpochMilli()));
-        application.activity().lastWritten().ifPresent(lastQueried -> activity.setLong("writtenAt", lastQueried.toEpochMilli()));
+        application.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", instant.toEpochMilli()));
+        application.activity().lastWritten().ifPresent(instant -> activity.setLong("lastWritten", instant.toEpochMilli()));
+        application.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value));
+        application.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value));
 
         application.ownershipIssueId().ifPresent(issueId -> object.setString("ownershipIssueId", issueId.value()));
         application.deploymentJobs().issueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value()));
@@ -478,6 +480,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                                                   instant.toEpochMilli()));
         deployment.activity().lastWritten().ifPresent(instant -> activity.setLong("lastWritten",
                                                                                   instant.toEpochMilli()));
+        deployment.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value));
+        deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value));
 
         // Cost
         DeploymentCost appCost = deployment.calculateCost();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index a651210767d..768164c3002 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -60,21 +60,28 @@ public class DeploymentMetricsMaintainerTest {
         maintainer.maintain();
         assertEquals(t2, deployment.get().activity().lastQueried().get());
         assertEquals(t2, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
 
-        // Query traffic disappears. Query activity time is no longer updated
+        // Query traffic disappears. Query activity stops updating
         tester.clock().advance(Duration.ofHours(1));
         Instant t3 = tester.clock().instant();
         tester.metricsService().setMetric("queriesPerSecond", 0D);
+        tester.metricsService().setMetric("writesPerSecond", 5D);
         maintainer.maintain();
         assertEquals(t2, deployment.get().activity().lastQueried().get());
         assertEquals(t3, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
 
-        // Feed traffic disappears. Feed activity time is no longer updated
+        // Feed traffic disappears. Feed activity stops updating
         tester.clock().advance(Duration.ofHours(1));
         tester.metricsService().setMetric("writesPerSecond", 0D);
         maintainer.maintain();
         assertEquals(t2, deployment.get().activity().lastQueried().get());
         assertEquals(t3, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 5c5827fa167..656c377b84b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -36,6 +36,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalDouble;
 import java.util.OptionalLong;
 
 import static com.yahoo.config.provision.SystemName.main;
@@ -71,7 +72,8 @@ public class ApplicationSerializerTest {
         deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
                                        createClusterUtils(3, 0.2), createClusterInfo(3, 4),
                                        new DeploymentMetrics(2,3,4,5,6),
-                                       DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt))));
+                                       DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt),
+                                                                 OptionalDouble.of(200), OptionalDouble.of(10))));
 
         OptionalLong projectId = OptionalLong.of(123L);
         List<JobStatus> statusList = new ArrayList<>();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
index f8c4c26d6a8..ee54e2741ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
@@ -251,7 +251,9 @@
     "writeServiceQuality": 0.7
   },
   "activity": {
-    "queriedAt": 1527848130000,
-    "writtenAt": 1527848130000
+    "lastQueried": 1527848130000,
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
index fc0f83c2cdc..c93ff6a0dd2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
@@ -236,7 +236,9 @@
     "writeServiceQuality": 0.7
   },
   "activity": {
-    "queriedAt": 1527848130000,
-    "writtenAt": 1527848130000
+    "lastQueried": 1527848130000,
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
index 8bb1ee83282..a1bd96e46d2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
@@ -223,8 +223,10 @@
     "writeServiceQuality": 0.7
   },
   "activity": {
-    "queriedAt": 1527848130000,
-    "writtenAt": 1527848130000
+    "lastQueried": 1527848130000,
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   },
   "ownershipIssueId": "321",
   "deploymentIssueId": "123"
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
index 79e86b5f7f4..2ce9d69472a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
@@ -18,7 +18,9 @@
   "gitCommit": "commit1",
   "activity": {
     "lastQueried": 1527848130000,
-    "lastWritten": 1527848130000
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   },
   "cost": {
     "tco": 74,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json
index 8fccd738554..ab86b891fae 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json
@@ -17,7 +17,9 @@
   "screwdriverId": "123",
   "activity": {
     "lastQueried": 1527848130000,
-    "lastWritten": 1527848130000
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   },
   "cost": {
     "tco": 74,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json
index 066e840fe16..902bda472cf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json
@@ -24,7 +24,9 @@
   "gitCommit": "commit1",
   "activity": {
     "lastQueried": 1527848130000,
-    "lastWritten": 1527848130000
+    "lastWritten": 1527848130000,
+    "lastQueriesPerSecond": 1.0,
+    "lastWritesPerSecond": 2.0
   },
   "cost": {
     "tco": 74,