Record deployment activity

10 files changed, 229 insertions, 82 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 913adf06f22..795b12f8af9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -79,7 +79,8 @@ public class LockedApplication extends Application {
         Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant,
                                                   previousDeployment.clusterUtils(),
                                                   previousDeployment.clusterInfo(),
-                                                  previousDeployment.metrics());
+                                                  previousDeployment.metrics(),
+                                                  previousDeployment.activity());
         return with(newDeployment);
     }
 
@@ -96,6 +97,12 @@ public class LockedApplication extends Application {
 
     }
 
+    public LockedApplication recordActivityAt(Instant instant, ZoneId zone) {
+        Deployment deployment = deployments().get(zone);
+        if (deployment == null) return this;
+        return with(deployment.recordActivityAt(instant));
+    }
+
     public LockedApplication with(ZoneId zone, DeploymentMetrics deploymentMetrics) {
         Deployment deployment = deployments().get(zone);
         if (deployment == null) return this;    // No longer deployed in this zone.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index 8fa0c6da49c..0a062427a8a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec.Id;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 
 import java.time.Instant;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -25,27 +26,25 @@ public class Deployment {
     private final Map<Id, ClusterUtilization> clusterUtils;
     private final Map<Id, ClusterInfo> clusterInfo;
     private final DeploymentMetrics metrics;
+    private final DeploymentActivity activity;
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) {
-        this(zone, applicationVersion, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics());
+        this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), Collections.emptyMap(),
+             new DeploymentMetrics(), DeploymentActivity.none);
     }
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime,
-                      Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics) {
-        Objects.requireNonNull(zone, "zone cannot be null");
-        Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null");
-        Objects.requireNonNull(version, "version cannot be null");
-        Objects.requireNonNull(deployTime, "deployTime cannot be null");
-        Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null");
-        Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null");
-        Objects.requireNonNull(metrics, "deployment metrics cannot be null");
-        this.zone = zone;
-        this.applicationVersion = applicationVersion;
-        this.version = version;
-        this.deployTime = deployTime;
-        this.clusterUtils = clusterUtils;
-        this.clusterInfo = clusterInfo;
-        this.metrics = metrics;
+                      Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo,
+                      DeploymentMetrics metrics,
+                      DeploymentActivity activity) {
+        this.zone = Objects.requireNonNull(zone, "zone cannot be null");
+        this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null");
+        this.version = Objects.requireNonNull(version, "version cannot be null");
+        this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null");
+        this.clusterUtils = Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null");
+        this.clusterInfo = Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null");
+        this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null");
+        this.activity = Objects.requireNonNull(activity, "activity cannot be null");
     }
 
     /** Returns the zone this was deployed to */
@@ -60,29 +59,42 @@ public class Deployment {
     /** Returns the time this was deployed */
     public Instant at() { return deployTime; }
 
+    /** Returns metrics for this */
+    public DeploymentMetrics metrics() {
+        return metrics;
+    }
+
+    /** Returns activity for this */
+    public DeploymentActivity activity() { return activity; }
+
+    /** Returns information about the clusters allocated to this */
     public  Map<Id, ClusterInfo> clusterInfo() {
         return clusterInfo;
     }
 
+    /** Returns utilization of the clusters allocated to this */
     public  Map<Id, ClusterUtilization> clusterUtils() {
         return clusterUtils;
     }
 
+    public Deployment recordActivityAt(Instant instant) {
+        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics,
+                              activity.recordAt(instant, metrics));
+    }
+
     public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics);
+        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics,
+                              activity);
     }
 
     public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, newClusterInfo, metrics);
+        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, newClusterInfo, metrics,
+                              activity);
     }
 
     public Deployment withMetrics(DeploymentMetrics metrics) {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics);
-    }
-
-    /** @return Key metrics for the deployment (application level) like QPS and document count */
-    public DeploymentMetrics metrics() {
-        return metrics;
+        return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics,
+                              activity);
     }
 
     /**
@@ -109,4 +121,5 @@ public class Deployment {
     public String toString() {
         return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime;
     }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java
new file mode 100644
index 00000000000..d4635212e80
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java
@@ -0,0 +1,55 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Recent activity in a deployment.
+ *
+ * @author mpolden
+ */
+public class DeploymentActivity {
+
+    /** Query rates at or below this threshold indicate inactivity */
+    private static final double inactivityThreshold = 0;
+
+    public static final DeploymentActivity none = new DeploymentActivity(Optional.empty(), Optional.empty());
+
+    private final Optional<Instant> lastQueried;
+    private final Optional<Instant> lastWritten;
+
+    private DeploymentActivity(Optional<Instant> lastQueried, Optional<Instant> lastWritten) {
+        this.lastQueried = Objects.requireNonNull(lastQueried, "lastQueried must be non-null");
+        this.lastWritten = Objects.requireNonNull(lastWritten, "lastWritten must be non-null");
+    }
+
+    /** The last time this deployment received queries (search) */
+    public Optional<Instant> lastQueried() {
+        return lastQueried;
+    }
+
+    /** The last time this deployment received writes (feed) */
+    public Optional<Instant> lastWritten() {
+        return lastWritten;
+    }
+
+    /** Record activity using given metrics */
+    public DeploymentActivity recordAt(Instant instant, DeploymentMetrics metrics) {
+        return new DeploymentActivity(activityAt(instant, lastQueried, metrics.queriesPerSecond()),
+                                      activityAt(instant, lastWritten, metrics.writesPerSecond()));
+    }
+
+    public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt) {
+        if (!queriedAt.isPresent() && !writtenAt.isPresent()) {
+            return none;
+        }
+        return new DeploymentActivity(queriedAt, writtenAt);
+    }
+
+    private static Optional<Instant> activityAt(Instant newInstant, Optional<Instant> oldInstant, double rate) {
+        return rate > inactivityThreshold ? Optional.of(newInstant) : oldInstant;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 821efba013d..4dacb2e32d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -15,8 +15,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
- * Retrieve deployment metrics like qps and document count from the metric service and
- * update the applications with this info.
+ * Retrieve deployment metrics such as QPS and document count from the metric service and
+ * update applications with this info.
  *
  * @author smorgrav
  */
@@ -39,19 +39,19 @@ public class DeploymentMetricsMaintainer extends Maintainer {
                 for (Deployment deployment : application.deployments().values()) {
                     MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
                             .getDeploymentMetrics(application.id(), deployment.zone());
-                    DeploymentMetrics appMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(),
+                    DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(),
                                                                          deploymentMetrics.writesPerSecond(),
                                                                          deploymentMetrics.documentCount(),
                                                                          deploymentMetrics.queryLatencyMillis(),
                                                                          deploymentMetrics.writeLatencyMillis());
 
                     controller().applications().lockIfPresent(application.id(), lockedApplication ->
-                            controller().applications().store(lockedApplication.with(deployment.zone(), appMetrics)));
+                            controller().applications().store(lockedApplication.with(deployment.zone(), newMetrics)
+                                                                               .recordActivityAt(controller().clock().instant(), deployment.zone())));
                 }
-            }
-            catch (UncheckedIOException e) {
+            } catch (UncheckedIOException e) {
                 if (!hasWarned) // produce only one warning per maintenance interval
-                    log.log(Level.WARNING, "Failed talking to YAMAS: " + Exceptions.toMessageString(e) +
+                    log.log(Level.WARNING, "Failed to query metrics service: " + Exceptions.toMessageString(e) +
                                            ". Retrying in " + maintenanceInterval());
                 hasWarned = true;
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 21eea21ba68..6ad2452b2a2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
@@ -68,6 +69,8 @@ public class ApplicationSerializer {
     private final String repositoryField = "repositoryField";
     private final String branchField = "branchField";
     private final String commitField = "commitField";
+    private final String lastQueriedField = "lastQueried";
+    private final String lastWrittenField = "lastWritten";
 
     // DeploymentJobs fields
     private final String projectIdField = "projectId";
@@ -148,10 +151,12 @@ public class ApplicationSerializer {
         toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField));
         clusterInfoToSlime(deployment.clusterInfo(), object);
         clusterUtilsToSlime(deployment.clusterUtils(), object);
-        metricsToSlime(deployment.metrics(), object);
+        deploymentMetricsToSlime(deployment.metrics(), object);
+        deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli()));
+        deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli()));
     }
 
-    private void metricsToSlime(DeploymentMetrics metrics, Cursor object) {
+    private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) {
         Cursor root = object.setObject(deploymentMetricsField);
         root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond());
         root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond());
@@ -289,19 +294,17 @@ public class ApplicationSerializer {
                               Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
                               clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)),
                               clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
-                              deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)));
+                              deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)),
+                              DeploymentActivity.create(optionalInstant(deploymentObject.field(lastQueriedField)),
+                                                        optionalInstant(deploymentObject.field(lastWrittenField))));
     }
 
     private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) {
-
-        double queriesPerSecond = object.field(deploymentMetricsQPSField).asDouble();
-        double writesPerSecond = object.field(deploymentMetricsWPSField).asDouble();
-        double documentCount = object.field(deploymentMetricsDocsField).asDouble();
-        double queryLatencyMillis = object.field(deploymentMetricsQueryLatencyField).asDouble();
-        double writeLatencyMills = object.field(deploymentMetricsWriteLatencyField).asDouble();
-
-        return new DeploymentMetrics(queriesPerSecond, writesPerSecond,
-                documentCount, queryLatencyMillis, writeLatencyMills);
+        return new DeploymentMetrics(object.field(deploymentMetricsQPSField).asDouble(),
+                                     object.field(deploymentMetricsWPSField).asDouble(),
+                                     object.field(deploymentMetricsDocsField).asDouble(),
+                                     object.field(deploymentMetricsQueryLatencyField).asDouble(),
+                                     object.field(deploymentMetricsWriteLatencyField).asDouble());
     }
 
     private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime(Inspector object) {
@@ -426,4 +429,9 @@ public class ApplicationSerializer {
         return SlimeUtils.optionalString(field);
     }
 
+    private Optional<Instant> optionalInstant(Inspector field) {
+        OptionalLong value = optionalLong(field);
+        return value.isPresent() ? Optional.of(Instant.ofEpochMilli(value.getAsLong())) : Optional.empty();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 98189613bd0..cf2fa182d0a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
-import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
 import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
@@ -41,6 +40,7 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 
 import java.util.Optional;
 import java.util.OptionalLong;
+import java.util.function.Supplier;
 import java.util.logging.Logger;
 
 import static org.junit.Assert.assertNotNull;
@@ -64,25 +64,28 @@ public final class ControllerTester {
     private final ArtifactRepositoryMock artifactRepository;
     private final EntityService entityService;
     private final MockBuildService buildService;
+    private final MockMetricsService metricsService;
 
     private Controller controller;
 
-    public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb) {
+    public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb,
+                            MockMetricsService metricsService) {
         this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()),
              new ZoneRegistryMock(), new GitHubMock(), curatorDb, rotationsConfig,
-             new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService(), new MockBuildService());
+             new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService(), new MockBuildService(),
+             metricsService);
     }
 
     public ControllerTester(ManualClock clock) {
-        this(clock, defaultRotationsConfig(), new MockCuratorDb());
+        this(clock, defaultRotationsConfig(), new MockCuratorDb(), new MockMetricsService());
     }
 
     public ControllerTester(RotationsConfig rotationsConfig) {
-        this(new ManualClock(), rotationsConfig, new MockCuratorDb());
+        this(new ManualClock(), rotationsConfig, new MockCuratorDb(), new MockMetricsService());
     }
 
     public ControllerTester(MockCuratorDb curatorDb) {
-        this(new ManualClock(), defaultRotationsConfig(), curatorDb);
+        this(new ManualClock(), defaultRotationsConfig(), curatorDb, new MockMetricsService());
     }
 
     public ControllerTester() {
@@ -93,7 +96,8 @@ public final class ControllerTester {
                              ConfigServerMock configServer, ZoneRegistryMock zoneRegistry,
                              GitHubMock gitHub, CuratorDb curator, RotationsConfig rotationsConfig,
                              MemoryNameService nameService, ArtifactRepositoryMock artifactRepository,
-                             EntityService entityService, MockBuildService buildService) {
+                             EntityService entityService, MockBuildService buildService,
+                             MockMetricsService metricsService) {
         this.athenzDb = athenzDb;
         this.clock = clock;
         this.configServer = configServer;
@@ -105,8 +109,10 @@ public final class ControllerTester {
         this.artifactRepository = artifactRepository;
         this.entityService = entityService;
         this.buildService = buildService;
+        this.metricsService = metricsService;
         this.controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry,
-                                           athenzDb, nameService, artifactRepository, entityService, buildService);
+                                           athenzDb, nameService, artifactRepository, entityService, buildService,
+                                           metricsService);
 
         // Make root logger use time from manual clock
         Logger.getLogger("").getHandlers()[0].setFilter(
@@ -138,10 +144,12 @@ public final class ControllerTester {
 
     public MockBuildService buildService() { return buildService; }
 
+    public MockMetricsService metricsService() { return metricsService; }
+
     /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */
     public final void createNewController() {
         controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb,
-                                      nameService, artifactRepository, entityService, buildService);
+                                      nameService, artifactRepository, entityService, buildService, metricsService);
     }
 
     /** Creates the given tenant and application and deploys it */
@@ -241,6 +249,10 @@ public final class ControllerTester {
                                            new DeployOptions(false, Optional.empty(), false, deployCurrentVersion));
     }
 
+    public Supplier<Application> application(ApplicationId application) {
+        return () -> controller().applications().require(application);
+    }
+
     /** Used by ApplicationSerializerTest to avoid breaking encapsulation. Should not be used by anything else */
     public static LockedApplication writable(Application application) {
         return new LockedApplication(application, new Lock("/test", new MockCurator()));
@@ -251,7 +263,7 @@ public final class ControllerTester {
                                                GitHubMock gitHub, ZoneRegistryMock zoneRegistryMock,
                                                AthenzDbMock athensDb, MemoryNameService nameService,
                                                ArtifactRepository artifactRepository, EntityService entityService,
-                                               BuildService buildService) {
+                                               BuildService buildService, MockMetricsService metricsService) {
         Controller controller = new Controller(curator,
                                                rotationsConfig,
                                                gitHub,
@@ -260,7 +272,7 @@ public final class ControllerTester {
                                                new MemoryGlobalRoutingService(),
                                                zoneRegistryMock,
                                                configServer,
-                                               new MockMetricsService(),
+                                               metricsService,
                                                nameService,
                                                new MockRoutingGenerator(),
                                                new ChefMock(),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java
index 88bbb582564..67a4139ecf1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 
 import java.util.HashMap;
@@ -10,16 +11,28 @@ import java.util.Map;
 /**
  * @author bratseth
  */
-public class MockMetricsService implements com.yahoo.vespa.hosted.controller.api.integration.MetricsService {
+public class MockMetricsService implements MetricsService {
+
+    private final Map<String, Double> metrics = new HashMap<>();
+
+    public MockMetricsService setMetric(String key, Double value) {
+        metrics.put(key, value);
+        return this;
+    }
 
     @Override
     public ApplicationMetrics getApplicationMetrics(ApplicationId application) {
-        return new ApplicationMetrics(0.5, 0.7);
+        return new ApplicationMetrics(metrics.getOrDefault("queryServiceQuality", 0.5),
+                                      metrics.getOrDefault("writeServiceQuality", 0.7));
     }
 
     @Override
     public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) {
-        return new DeploymentMetrics(1, 2, 3, 4, 5);
+        return new DeploymentMetrics(metrics.getOrDefault("queriesPerSecond", 1D),
+                                     metrics.getOrDefault("writesPerSecond", 2D),
+                                     metrics.getOrDefault("docoumentCount", 3D).longValue(),
+                                     metrics.getOrDefault("queryLatencyMillis", 4D),
+                                     metrics.getOrDefault("writeLatencyMillis", 5D));
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index 148d11e8b38..a651210767d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -12,37 +12,69 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.function.Supplier;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 /**
  * @author smorgrav
+ * @author mpolden
  */
 public class DeploymentMetricsMaintainerTest {
 
     @Test
     public void maintain() {
         ControllerTester tester = new ControllerTester();
-        ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
+        ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
+                                                  Environment.dev, 123).id();
+        DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(),
+                                                                                 Duration.ofDays(1),
+                                                                                 new JobControl(new MockCuratorDb()));
+        Supplier<Application> app = tester.application(appId);
+        Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
 
-        // Pre condition: no metric info on neither application nor deployment
-        assertEquals(0, tester.controller().applications().require(app).metrics().queryServiceQuality(), 0);
-        Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
-        assertEquals(0, deployment.metrics().documentCount(), 0);
+        // No metrics gathered yet
+        assertEquals(0, app.get().metrics().queryServiceQuality(), 0);
+        assertEquals(0, deployment.get().metrics().documentCount(), 0);
+        assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent());
+        assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent());
 
-        DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofMinutes(10), new JobControl(new MockCuratorDb()));
+        // Metrics are gathered and saved to application
         maintainer.maintain();
+        assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
+        assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
+        assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE);
+        assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE);
+        assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE);
+        assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE);
+        Instant t1 = tester.clock().instant();
+        assertEquals(t1, deployment.get().activity().lastQueried().get());
+        assertEquals(t1, deployment.get().activity().lastWritten().get());
 
-        // Post condition:
-        Application application = tester.controller().applications().require(app);
-        assertEquals(0.5, application.metrics().queryServiceQuality(), Double.MIN_VALUE);
-        assertEquals(0.7, application.metrics().writeServiceQuality(), Double.MIN_VALUE);
-        deployment = application.deployments().values().stream().findAny().get();
-        assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE);
-        assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE);
-        assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE);
-        assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE);
-        assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE);
+        // Time passes. Activity is updated as app is still receiving traffic
+        tester.clock().advance(Duration.ofHours(1));
+        Instant t2 = tester.clock().instant();
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t2, deployment.get().activity().lastWritten().get());
+
+        // Query traffic disappears. Query activity time is no longer updated
+        tester.clock().advance(Duration.ofHours(1));
+        Instant t3 = tester.clock().instant();
+        tester.metricsService().setMetric("queriesPerSecond", 0D);
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t3, deployment.get().activity().lastWritten().get());
+
+        // Feed traffic disappears. Feed activity time is no longer updated
+        tester.clock().advance(Duration.ofHours(1));
+        tester.metricsService().setMetric("writesPerSecond", 0D);
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t3, deployment.get().activity().lastWritten().get());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 071ec7b5c11..26cb68647e4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
@@ -65,9 +66,12 @@ public class ApplicationSerializerTest {
         ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31);
         ApplicationVersion applicationVersion2 = ApplicationVersion
                 .from(new SourceRevision("repo1", "branch1", "commit1"), 32);
+        Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z");
         deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils
         deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
-                createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6)));
+                                       createClusterUtils(3, 0.2), createClusterInfo(3, 4),
+                                       new DeploymentMetrics(2,3,4,5,6),
+                                       DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt))));
 
         OptionalLong projectId = OptionalLong.of(123L);
         List<JobStatus> statusList = new ArrayList<>();
@@ -108,6 +112,8 @@ public class ApplicationSerializerTest {
         assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version());
         assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at());
         assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at());
+        assertEquals(original.deployments().get(zone2).activity().lastQueried().get(), serialized.deployments().get(zone2).activity().lastQueried().get());
+        assertEquals(original.deployments().get(zone2).activity().lastWritten().get(), serialized.deployments().get(zone2).activity().lastWritten().get());
 
         assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId());
         assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size());
@@ -143,12 +149,11 @@ public class ApplicationSerializerTest {
         // Test metrics
         assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
         assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
-
-        assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
-        assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
-        assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
-        assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
-        assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
+        assertEquals(original.deployments().get(zone2).metrics().queriesPerSecond(), serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
+        assertEquals(original.deployments().get(zone2).metrics().writesPerSecond(), serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
+        assertEquals(original.deployments().get(zone2).metrics().documentCount(), serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
+        assertEquals(original.deployments().get(zone2).metrics().queryLatencyMillis(), serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
+        assertEquals(original.deployments().get(zone2).metrics().writeLatencyMillis(), serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
 
         { // test more deployment serialization cases
             Application original2 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 42)));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 8d734ec549c..545ee529635 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -57,6 +57,7 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1123,7 +1124,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
                     lockedApplication = lockedApplication
                             .withClusterInfo(deployment.zone(), clusterInfo)
                             .withClusterUtilization(deployment.zone(), clusterUtils)
-                            .with(deployment.zone(), metrics);
+                            .with(deployment.zone(), metrics)
+                            .recordActivityAt(Instant.parse("2018-06-01T10:15:30.00Z"), deployment.zone());
                 }
                 controllerTester.controller().applications().store(lockedApplication);
             });