Olaaun/metric maintaier use api (#7402)

* Metric maintainers now feed to API instead of writing directly to DB

* Only feed once per maintain run

3 files changed, 77 insertions, 93 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
index 66be2d09797..71d2690bb4a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
@@ -3,24 +3,45 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
+import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static org.junit.Assert.assertEquals;
 
 /**
  * @author smorgrav
  */
 public class ClusterUtilizationMaintainerTest {
 
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(4443);
+
     @Test
     public void maintain() {
+        wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/clusterutilization"))
+                .willReturn(aResponse().withStatus(200)));
         ControllerTester tester = new ControllerTester();
         ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
 
@@ -28,12 +49,16 @@ public class ClusterUtilizationMaintainerTest {
         Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
         Assert.assertEquals(0, deployment.clusterUtils().size());
 
-        ClusterUtilizationMaintainer mainainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()));
-        mainainer.maintain();
+        ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
+        ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
+        ClusterUtilizationMaintainer maintainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()), apiAuthorityConfig);
+        maintainer.maintain();
 
-        deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
-        Assert.assertEquals(1, deployment.clusterUtils().size());
-        Assert.assertEquals(0.5554, deployment.clusterUtils().get(ClusterSpec.Id.from("default")).getCpu(), Double.MIN_VALUE);
+        List<ServeEvent> allServeEvents = getAllServeEvents();
+        assertEquals(allServeEvents.size(), 1);
+        LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/clusterutilization"))).get(0);
+        String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"deployments\":[{\"zoneId\":\"dev.us-east-1\",\"clusterUtil\":[{\"clusterSpecId\":\"default\",\"cpu\":0.5554,\"memory\":0.6990000000000001,\"disk\":0.34590000000000004,\"diskBusy\":0.0}]}]}]";
+        assertEquals(expectedBody, new String(request.getBody()));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index e11440a372c..b0d28e05744 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -1,27 +1,30 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.ApplicationId;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.RotationStatus;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
+import org.junit.Rule;
 import org.junit.Test;
 
 import java.time.Duration;
-import java.time.Instant;
-import java.util.function.Supplier;
-
-import static java.time.temporal.ChronoUnit.MILLIS;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
+import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 /**
  * @author smorgrav
@@ -29,69 +32,16 @@ import static org.junit.Assert.assertFalse;
  */
 public class DeploymentMetricsMaintainerTest {
 
-    @Test
-    public void updates_metrics() {
-        ControllerTester tester = new ControllerTester();
-        ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
-                                                  Environment.dev, 123).id();
-        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
-        Supplier<Application> app = tester.application(appId);
-        Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
-
-        // No metrics gathered yet
-        assertEquals(0, app.get().metrics().queryServiceQuality(), 0);
-        assertEquals(0, deployment.get().metrics().documentCount(), 0);
-        assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent());
-        assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent());
-
-        // Metrics are gathered and saved to application
-        maintainer.maintain();
-        assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
-        assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
-        assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE);
-        assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE);
-        assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE);
-        assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE);
-        assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE);
-        Instant t1 = tester.clock().instant().truncatedTo(MILLIS);
-        assertEquals(t1, deployment.get().activity().lastQueried().get());
-        assertEquals(t1, deployment.get().activity().lastWritten().get());
-
-        // Time passes. Activity is updated as app is still receiving traffic
-        tester.clock().advance(Duration.ofHours(1));
-        Instant t2 = tester.clock().instant().truncatedTo(MILLIS);
-        maintainer.maintain();
-        assertEquals(t2, deployment.get().activity().lastQueried().get());
-        assertEquals(t2, deployment.get().activity().lastWritten().get());
-        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
-        assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
-
-        // Query traffic disappears. Query activity stops updating
-        tester.clock().advance(Duration.ofHours(1));
-        Instant t3 = tester.clock().instant().truncatedTo(MILLIS);
-        tester.metricsService().setMetric("queriesPerSecond", 0D);
-        tester.metricsService().setMetric("writesPerSecond", 5D);
-        maintainer.maintain();
-        assertEquals(t2, deployment.get().activity().lastQueried().get());
-        assertEquals(t3, deployment.get().activity().lastWritten().get());
-        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
-        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
-
-        // Feed traffic disappears. Feed activity stops updating
-        tester.clock().advance(Duration.ofHours(1));
-        tester.metricsService().setMetric("writesPerSecond", 0D);
-        maintainer.maintain();
-        assertEquals(t2, deployment.get().activity().lastQueried().get());
-        assertEquals(t3, deployment.get().activity().lastWritten().get());
-        assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
-        assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
-    }
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(4443);
 
     @Test
-    public void updates_rotation_status() {
+    public void maintain() {
         DeploymentTester tester = new DeploymentTester();
         MetricsServiceMock metricsService = tester.controllerTester().metricsService();
-        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
+        ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
+        ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
+        DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), apiAuthorityConfig);
         Application application = tester.createApplication("app1", "tenant1", 1, 1L);
         ZoneId zone1 = ZoneId.from("prod", "us-west-1");
         ZoneId zone2 = ZoneId.from("prod", "us-east-3");
@@ -105,32 +55,22 @@ public class DeploymentMetricsMaintainerTest {
                 .build();
         tester.deployCompletely(application, applicationPackage);
 
-        Supplier<Application> app = () -> tester.application(application.id());
-        Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
-        Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
         String assignedRotation = "rotation-fqdn-01";
         tester.controllerTester().metricsService().addRotation(assignedRotation);
 
-        // No status gathered yet
-        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get()));
-
         // One rotation out, one in
         metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test");
         metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test");
-        maintainer.maintain();
-        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get()));
 
-        // All rotations in
-        metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test");
+        wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))
+                .willReturn(aResponse().withStatus(200)));
         maintainer.maintain();
-        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get()));
-    }
 
-    private static DeploymentMetricsMaintainer maintainer(Controller controller) {
-        return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
+        List<ServeEvent> allServeEvents = getAllServeEvents();
+        assertEquals(1, allServeEvents.size());
+        LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))).get(0);
+        String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"applicationMetrics\":{\"queryServiceQuality\":0.5,\"writeServiceQuality\":0.7},\"rotationStatus\":[{\"hostname\":\"proxy.prod.us-east-3.vip.test\",\"rotationStatus\":\"out\"},{\"hostname\":\"proxy.prod.us-west-1.vip.test\",\"rotationStatus\":\"in\"}],\"deploymentMetrics\":[{\"zoneId\":\"prod.us-west-1\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0},{\"zoneId\":\"prod.us-east-3\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0}]}]";
+        assertEquals(expectedBody, new String(request.getBody()));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 027fc9745f2..f840c188a34 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.slime.Cursor;
@@ -19,6 +20,7 @@ import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.OktaAccessToken;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -41,8 +43,10 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
@@ -75,6 +79,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.TreeMap;
 import java.util.function.Supplier;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
@@ -1273,8 +1278,22 @@ public class ApplicationApiTest extends ControllerContainerTest {
         String vipName = "proxy." + zone.value() + ".vip.test";
         metricsService().addRotation(rotationName)
                         .setZoneIn(rotationName, vipName);
-
-        new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run();
+        ApplicationController applicationController = controllerTester.controller().applications();
+        List<Application> applicationList = applicationController.asList();
+        applicationList.stream().forEach(application -> {
+                applicationController.lockIfPresent(application.id(), locked ->
+                        applicationController.store(locked.withRotationStatus(rotationStatus(application))));
+        });}
+
+    private Map<HostName, RotationStatus> rotationStatus(Application application) {
+        return controllerTester.controller().applications().rotationRepository().getRotation(application)
+                .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name()))
+                .map(rotationStatus -> {
+                    Map<HostName, RotationStatus> result = new TreeMap<>();
+                    rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationStatus.in));
+                    return result;
+                })
+                .orElseGet(Collections::emptyMap);
     }
 
     private void updateContactInformation() {