Revert "Olaaun/metric maintaier use api (#7402)" (#7461)

This reverts commit 57ba28e794416ae7cc6e14660249459dac7df937.

6 files changed, 126 insertions, 188 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
index 44f29eb6113..135a05e2725 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
@@ -3,26 +3,16 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 
 import java.time.Duration;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.logging.Level;
 
 /**
  * Fetch utilization metrics and update applications with this data.
@@ -32,12 +22,10 @@ import java.util.logging.Level;
 public class ClusterUtilizationMaintainer extends Maintainer {
 
     private final Controller controller;
-    private final List<String> baseUris;
 
-    public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) {
+    public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) {
         super(controller, duration, jobControl);
         this.controller = controller;
-        this.baseUris = apiAuthorityConfig.authorities();
     }
 
     private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, ZoneId zone) {
@@ -55,45 +43,15 @@ public class ClusterUtilizationMaintainer extends Maintainer {
 
     @Override
     protected void maintain() {
-        try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
-            String uri = baseUris.get(0) + "metricforwarding/v1/clusterutilization"; // For now, we only feed to one controller
-            Slime slime = getMetricSlime();
-            ByteArrayEntity entity = new ByteArrayEntity(SlimeUtils.toJsonBytes(slime));
-            HttpPost httpPost = new HttpPost(uri);
-            httpPost.setEntity(entity);
-            httpClient.execute(httpPost);
-        } catch (Exception e) {
-            log.log(Level.WARNING, "Failed to update cluster utilization metrics", e);
-        }
-
-    }
-
-    private Slime getMetricSlime() {
-        Slime slime = new Slime();
-        Cursor cursor = slime.setArray();
         for (Application application : controller().applications().asList()) {
-            Cursor applicationCursor = cursor.addObject();
-            applicationCursor.setString("applicationId", application.id().serializedForm());
-            Cursor deploymentArray = applicationCursor.setArray("deployments");
             for (Deployment deployment : application.deployments().values()) {
-                Cursor deploymentEntry = deploymentArray.addObject();
-                deploymentEntry.setString("zoneId", deployment.zone().value());
-                Cursor clusterArray = deploymentEntry.setArray("clusterUtil");
+
                 Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone());
-                fillClusterUtilization(clusterArray, clusterUtilization);
+
+                controller().applications().lockIfPresent(application.id(), lockedApplication ->
+                    controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization)));
             }
         }
-        return slime;
     }
 
-    private void fillClusterUtilization(Cursor cursor, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) {
-        for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusterUtilization.entrySet()) {
-            Cursor clusterUtilCursor = cursor.addObject();
-            clusterUtilCursor.setString("clusterSpecId", entry.getKey().value());
-            clusterUtilCursor.setDouble("cpu", entry.getValue().getCpu());
-            clusterUtilCursor.setDouble("memory", entry.getValue().getMemory());
-            clusterUtilCursor.setDouble("disk", entry.getValue().getDisk());
-            clusterUtilCursor.setDouble("diskBusy", entry.getValue().getDiskBusy());
-        }
-    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 5d26ce16885..33555c08a43 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -65,8 +65,8 @@ public class ControllerMaintenance extends AbstractComponent {
         upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
         readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl);
         clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient);
-        clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl, apiAuthorityConfig);
-        deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl, apiAuthorityConfig);
+        clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl);
+        deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl);
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
         dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 67fb224f1ea..72936f8a679 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -2,23 +2,15 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.HostName;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.RotationStatus;
-import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.yolean.Exceptions;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 
-import java.io.IOException;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
@@ -45,12 +37,10 @@ public class DeploymentMetricsMaintainer extends Maintainer {
     private static final int applicationsToUpdateInParallel = 10;
 
     private final ApplicationController applications;
-    private final List<String> baseUris;
 
-    public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) {
+    public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) {
         super(controller, duration, jobControl);
         this.applications = controller.applications();
-        baseUris = apiAuthorityConfig.authorities();
     }
 
     @Override
@@ -61,27 +51,37 @@ public class DeploymentMetricsMaintainer extends Maintainer {
 
         // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used
         ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel);
-        Slime slime = new Slime();
-        Cursor cursor = slime.setArray();
         pool.submit(() -> {
             applicationList.parallelStream().forEach(application -> {
-                Cursor applicationCursor = cursor.addObject();
-                applicationCursor.setString("applicationId", application.id().serializedForm());
-                Cursor applicationMetrics = applicationCursor.setObject("applicationMetrics");
-                fillApplicationMetrics(applicationMetrics, application);
-                Cursor rotationStatus = applicationCursor.setArray("rotationStatus");
-                fillRotationStatus(rotationStatus, application);
-                Cursor deploymentArray = applicationCursor.setArray("deploymentMetrics");
-                for (Deployment deployment : application.deployments().values()) {
-                    Cursor deploymentEntry = deploymentArray.addObject();
-                    fillDeploymentMetrics(deploymentEntry, application, deployment);
+                try {
+                    applications.lockIfPresent(application.id(), locked ->
+                            applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id()))));
+
+                    applications.lockIfPresent(application.id(), locked ->
+                            applications.store(locked.withRotationStatus(rotationStatus(application))));
+
+                    for (Deployment deployment : application.deployments().values()) {
+                        MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
+                                                                                         .getDeploymentMetrics(application.id(), deployment.zone());
+                        DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(),
+                                                                             deploymentMetrics.writesPerSecond(),
+                                                                             deploymentMetrics.documentCount(),
+                                                                             deploymentMetrics.queryLatencyMillis(),
+                                                                             deploymentMetrics.writeLatencyMillis());
+
+                        applications.lockIfPresent(application.id(), locked ->
+                                applications.store(locked.with(deployment.zone(), newMetrics)
+                                                         .recordActivityAt(controller().clock().instant(), deployment.zone())));
+                    }
+                } catch (Exception e) {
+                    failures.incrementAndGet();
+                    lastException.set(e);
                 }
             });
         });
         pool.shutdown();
         try {
             pool.awaitTermination(30, TimeUnit.MINUTES);
-            feedMetrics(slime);
             if (lastException.get() != null) {
                 log.log(Level.WARNING, String.format("Failed to query metrics service for %d/%d applications. Last error: %s. Retrying in %s",
                                                      failures.get(),
@@ -91,8 +91,6 @@ public class DeploymentMetricsMaintainer extends Maintainer {
             }
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
-        } catch (IOException e) {
-            log.log(Level.WARNING, "Unable to feed metrics to API", e);
         }
     }
 
@@ -108,40 +106,6 @@ public class DeploymentMetricsMaintainer extends Maintainer {
                            .orElseGet(Collections::emptyMap);
     }
 
-    private void fillApplicationMetrics(Cursor applicationCursor, Application application) {
-        MetricsService.ApplicationMetrics metrics = controller().metricsService().getApplicationMetrics(application.id());
-        applicationCursor.setDouble("queryServiceQuality", metrics.queryServiceQuality());
-        applicationCursor.setDouble("writeServiceQuality", metrics.writeServiceQuality());
-    }
-
-    private void fillRotationStatus(Cursor rotationStatusCursor, Application application) {
-        Map<HostName, RotationStatus> rotationStatus = rotationStatus(application);
-        for (Map.Entry<HostName, RotationStatus> entry : rotationStatus.entrySet()) {
-            Cursor rotationStatusEntry = rotationStatusCursor.addObject();
-            rotationStatusEntry.setString("hostname", entry.getKey().value());
-            rotationStatusEntry.setString("rotationStatus", entry.getValue().toString());
-        }
-    }
-
-    private void fillDeploymentMetrics(Cursor deploymentCursor, Application application, Deployment deployment) {
-        MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
-                .getDeploymentMetrics(application.id(), deployment.zone());
-        deploymentCursor.setString("zoneId", deployment.zone().value());
-        deploymentCursor.setDouble("queriesPerSecond", deploymentMetrics.queriesPerSecond());
-        deploymentCursor.setDouble("writesPerSecond", deploymentMetrics.writesPerSecond());
-        deploymentCursor.setDouble("documentCount", deploymentMetrics.documentCount());
-        deploymentCursor.setDouble("queryLatencyMillis", deploymentMetrics.queryLatencyMillis());
-        deploymentCursor.setDouble("writeLatencyMillis", deploymentMetrics.writeLatencyMillis());
-    }
-
-    private void feedMetrics(Slime slime) throws IOException {
-        String uri = baseUris.get(0) + "/metricforwarding/v1/deploymentmetrics/"; // For now, we only feed to one controller
-        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
-        HttpPost httpPost = new HttpPost(uri);
-        httpPost.setEntity(new ByteArrayEntity(SlimeUtils.toJsonBytes(slime)));
-        httpClient.execute(httpPost);
-    }
-
     private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
         switch (status) {
             case IN: return RotationStatus.in;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
index 71d2690bb4a..66be2d09797 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
@@ -3,45 +3,24 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-import com.github.tomakehurst.wiremock.junit.WireMockRule;
-import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
-import com.github.tomakehurst.wiremock.verification.LoggedRequest;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Assert;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.time.Duration;
-import java.util.List;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
-import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
-import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
-import static com.github.tomakehurst.wiremock.client.WireMock.post;
-import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
-import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
-import static org.junit.Assert.assertEquals;
 
 /**
  * @author smorgrav
  */
 public class ClusterUtilizationMaintainerTest {
 
-    @Rule
-    public WireMockRule wireMockRule = new WireMockRule(4443);
-
     @Test
     public void maintain() {
-        wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/clusterutilization"))
-                .willReturn(aResponse().withStatus(200)));
         ControllerTester tester = new ControllerTester();
         ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
 
@@ -49,16 +28,12 @@ public class ClusterUtilizationMaintainerTest {
         Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
         Assert.assertEquals(0, deployment.clusterUtils().size());
 
-        ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
-        ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
-        ClusterUtilizationMaintainer maintainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()), apiAuthorityConfig);
-        maintainer.maintain();
+        ClusterUtilizationMaintainer mainainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()));
+        mainainer.maintain();
 
-        List<ServeEvent> allServeEvents = getAllServeEvents();
-        assertEquals(allServeEvents.size(), 1);
-        LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/clusterutilization"))).get(0);
-        String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"deployments\":[{\"zoneId\":\"dev.us-east-1\",\"clusterUtil\":[{\"clusterSpecId\":\"default\",\"cpu\":0.5554,\"memory\":0.6990000000000001,\"disk\":0.34590000000000004,\"diskBusy\":0.0}]}]}]";
-        assertEquals(expectedBody, new String(request.getBody()));
+        deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
+        Assert.assertEquals(1, deployment.clusterUtils().size());
+        Assert.assertEquals(0.5554, deployment.clusterUtils().get(ClusterSpec.Id.from("default")).getCpu(), Double.MIN_VALUE);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index b0d28e05744..e11440a372c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -1,30 +1,27 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.github.tomakehurst.wiremock.junit.WireMockRule;
-import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
-import com.github.tomakehurst.wiremock.verification.LoggedRequest;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.time.Duration;
-import java.util.List;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
-import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
-import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
-import static com.github.tomakehurst.wiremock.client.WireMock.post;
-import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
-import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import java.time.Instant;
+import java.util.function.Supplier;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 /**
  * @author smorgrav
@@ -32,16 +29,69 @@ import static org.junit.Assert.assertEquals;
  */
 public class DeploymentMetricsMaintainerTest {
 
-    @Rule
-    public WireMockRule wireMockRule = new WireMockRule(4443);
+    @Test
+    public void updates_metrics() {
+        ControllerTester tester = new ControllerTester();
+        ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
+                                                  Environment.dev, 123).id();
+        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
+        Supplier<Application> app = tester.application(appId);
+        Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
+
+        // No metrics gathered yet
+        assertEquals(0, app.get().metrics().queryServiceQuality(), 0);
+        assertEquals(0, deployment.get().metrics().documentCount(), 0);
+        assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent());
+        assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent());
+
+        // Metrics are gathered and saved to application
+        maintainer.maintain();
+        assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
+        assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
+        assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE);
+        assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE);
+        assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE);
+        assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE);
+        Instant t1 = tester.clock().instant().truncatedTo(MILLIS);
+        assertEquals(t1, deployment.get().activity().lastQueried().get());
+        assertEquals(t1, deployment.get().activity().lastWritten().get());
+
+        // Time passes. Activity is updated as app is still receiving traffic
+        tester.clock().advance(Duration.ofHours(1));
+        Instant t2 = tester.clock().instant().truncatedTo(MILLIS);
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t2, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
+
+        // Query traffic disappears. Query activity stops updating
+        tester.clock().advance(Duration.ofHours(1));
+        Instant t3 = tester.clock().instant().truncatedTo(MILLIS);
+        tester.metricsService().setMetric("queriesPerSecond", 0D);
+        tester.metricsService().setMetric("writesPerSecond", 5D);
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t3, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
+
+        // Feed traffic disappears. Feed activity stops updating
+        tester.clock().advance(Duration.ofHours(1));
+        tester.metricsService().setMetric("writesPerSecond", 0D);
+        maintainer.maintain();
+        assertEquals(t2, deployment.get().activity().lastQueried().get());
+        assertEquals(t3, deployment.get().activity().lastWritten().get());
+        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
+        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
+    }
 
     @Test
-    public void maintain() {
+    public void updates_rotation_status() {
         DeploymentTester tester = new DeploymentTester();
         MetricsServiceMock metricsService = tester.controllerTester().metricsService();
-        ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
-        ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
-        DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), apiAuthorityConfig);
+        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
         Application application = tester.createApplication("app1", "tenant1", 1, 1L);
         ZoneId zone1 = ZoneId.from("prod", "us-west-1");
         ZoneId zone2 = ZoneId.from("prod", "us-east-3");
@@ -55,22 +105,32 @@ public class DeploymentMetricsMaintainerTest {
                 .build();
         tester.deployCompletely(application, applicationPackage);
 
+        Supplier<Application> app = () -> tester.application(application.id());
+        Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
+        Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
         String assignedRotation = "rotation-fqdn-01";
         tester.controllerTester().metricsService().addRotation(assignedRotation);
 
+        // No status gathered yet
+        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get()));
+
         // One rotation out, one in
         metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test");
         metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test");
+        maintainer.maintain();
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get()));
 
-        wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))
-                .willReturn(aResponse().withStatus(200)));
+        // All rotations in
+        metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test");
         maintainer.maintain();
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get()));
+    }
 
-        List<ServeEvent> allServeEvents = getAllServeEvents();
-        assertEquals(1, allServeEvents.size());
-        LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))).get(0);
-        String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"applicationMetrics\":{\"queryServiceQuality\":0.5,\"writeServiceQuality\":0.7},\"rotationStatus\":[{\"hostname\":\"proxy.prod.us-east-3.vip.test\",\"rotationStatus\":\"out\"},{\"hostname\":\"proxy.prod.us-west-1.vip.test\",\"rotationStatus\":\"in\"}],\"deploymentMetrics\":[{\"zoneId\":\"prod.us-west-1\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0},{\"zoneId\":\"prod.us-east-3\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0}]}]";
-        assertEquals(expectedBody, new String(request.getBody()));
+    private static DeploymentMetricsMaintainer maintainer(Controller controller) {
+        return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index f840c188a34..027fc9745f2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.slime.Cursor;
@@ -20,7 +19,6 @@ import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.OktaAccessToken;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -43,10 +41,8 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock;
-import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
@@ -79,7 +75,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TreeMap;
 import java.util.function.Supplier;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
@@ -1278,22 +1273,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         String vipName = "proxy." + zone.value() + ".vip.test";
         metricsService().addRotation(rotationName)
                         .setZoneIn(rotationName, vipName);
-        ApplicationController applicationController = controllerTester.controller().applications();
-        List<Application> applicationList = applicationController.asList();
-        applicationList.stream().forEach(application -> {
-                applicationController.lockIfPresent(application.id(), locked ->
-                        applicationController.store(locked.withRotationStatus(rotationStatus(application))));
-        });}
-
-    private Map<HostName, RotationStatus> rotationStatus(Application application) {
-        return controllerTester.controller().applications().rotationRepository().getRotation(application)
-                .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name()))
-                .map(rotationStatus -> {
-                    Map<HostName, RotationStatus> result = new TreeMap<>();
-                    rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationStatus.in));
-                    return result;
-                })
-                .orElseGet(Collections::emptyMap);
+
+        new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run();
     }
 
     private void updateContactInformation() {