Decouple deployment metrics from the metricservice - to avoid external calls to metric providers when asking for deployment info

7 files changed, 133 insertions, 25 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 672f50f83d7..1df0b1bca51 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -334,9 +334,10 @@ public class ApplicationController {
             Deployment previousDeployment = application.deployments().get(zone);
             Deployment newDeployment = previousDeployment;
             if (previousDeployment == null) {
-                newDeployment = new Deployment(zone, revision, version, clock.instant(), new HashMap<>(), new HashMap<>());
+                newDeployment = new Deployment(zone, revision, version, clock.instant());
             } else {
-                newDeployment = new Deployment(zone, revision, version, clock.instant(), previousDeployment.clusterUtils(), previousDeployment.clusterInfo());
+                newDeployment = new Deployment(zone, revision, version, clock.instant(),
+                        previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics());
             }
 
             application = application.with(newDeployment);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index 01219e940a3..05a7f9667ba 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -14,6 +14,7 @@ import java.util.Objects;
  * A deployment of an application in a particular zone.
  * 
  * @author bratseth
+ * @author smorgrav
  */
 public class Deployment {
 
@@ -23,24 +24,28 @@ public class Deployment {
     private final Instant deployTime;
     private final Map<Id, ClusterUtilization> clusterUtils;
     private final Map<Id, ClusterInfo> clusterInfo;
+    private final DeploymentMetrics metrics;
 
     public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) {
-        this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>());
+        this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics());
     }
 
-    public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo) {
+    public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime,
+                      Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics) {
         Objects.requireNonNull(zone, "zone cannot be null");
         Objects.requireNonNull(revision, "revision cannot be null");
         Objects.requireNonNull(version, "version cannot be null");
         Objects.requireNonNull(deployTime, "deployTime cannot be null");
         Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null");
         Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null");
+        Objects.requireNonNull(clusterInfo, "deployment metrics cannot be null");
         this.zone = zone;
         this.revision = revision;
         this.version = version;
         this.deployTime = deployTime;
         this.clusterUtils = clusterUtils;
         this.clusterInfo = clusterInfo;
+        this.metrics = metrics;
     }
 
     /** Returns the zone this was deployed to */
@@ -64,11 +69,20 @@ public class Deployment {
     }
 
     public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) {
-        return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo);
+        return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo, metrics);
     }
 
     public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) {
-        return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo);
+        return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo, metrics);
+    }
+
+    public Deployment withMetrics(DeploymentMetrics metrics) {
+        return new Deployment(zone, revision, version, deployTime, clusterUtils, clusterInfo, metrics);
+    }
+
+    /** @return Key metrics for the deployment (application level) like QPS and document count */
+    public DeploymentMetrics metrics() {
+        return metrics;
     }
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java
new file mode 100644
index 00000000000..6812e4cb468
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java
@@ -0,0 +1,50 @@
+package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @author smorgrav
+ */
+public class DeploymentMetrics {
+
+    private final double queriesPerSecond;
+    private final double writesPerSecond;
+    private final double documentCount;
+    private final double queryLatencyMillis;
+    private final double writeLatencyMills;
+
+    DeploymentMetrics() {
+        this.queriesPerSecond = 0;
+        this.writesPerSecond = 0;
+        this.documentCount = 0;
+        this.queryLatencyMillis = 0;
+        this.writeLatencyMills = 0;
+    }
+
+    public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, double documentCount,
+                      double queryLatencyMillis, double writeLatencyMills) {
+        this.queriesPerSecond = queriesPerSecond;
+        this.writesPerSecond = writesPerSecond;
+        this.documentCount = documentCount;
+        this.queryLatencyMillis = queryLatencyMillis;
+        this.writeLatencyMills = writeLatencyMills;
+    }
+
+    public double queriesPerSecond() {
+        return queriesPerSecond;
+    }
+
+    public double writesPerSecond() {
+        return writesPerSecond;
+    }
+
+    public double documentCount() {
+        return documentCount;
+    }
+
+    public double queryLatencyMillis() {
+        return queryLatencyMillis;
+    }
+
+    public double writeLatencyMillis() {
+        return writeLatencyMills;
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 859e322b227..494c4bb7936 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 
@@ -95,6 +96,14 @@ public class ApplicationSerializer {
     private final String clusterUtilsDiskField = "disk";
     private final String clusterUtilsDiskBusyField = "diskbusy";
 
+    // Deployment metrics fields
+    private final String deploymentMetricsField = "metrics";
+    private final String deploymentMetricsQPSField = "queriesPerSecond";
+    private final String deploymentMetricsWPSField = "writesPerSecond";
+    private final String deploymentMetricsDocsField = "documentCount";
+    private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis";
+    private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis";
+
     
     // ------------------ Serialization
     
@@ -123,6 +132,16 @@ public class ApplicationSerializer {
         toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField));
         clusterInfoToSlime(deployment.clusterInfo(), object);
         clusterUtilsToSlime(deployment.clusterUtils(), object);
+        metricsToSlime(deployment.metrics(), object);
+    }
+
+    private void metricsToSlime(DeploymentMetrics metrics, Cursor object) {
+        Cursor root = object.setObject(deploymentMetricsField);
+        root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond());
+        root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond());
+        root.setDouble(deploymentMetricsDocsField, metrics.documentCount());
+        root.setDouble(deploymentMetricsQueryLatencyField, metrics.queryLatencyMillis());
+        root.setDouble(deploymentMetricsWriteLatencyField, metrics.writeLatencyMillis());
     }
 
     private void clusterInfoToSlime(Map<ClusterSpec.Id, ClusterInfo> clusters, Cursor object) {
@@ -246,7 +265,20 @@ public class ApplicationSerializer {
                               Version.fromString(deploymentObject.field(versionField).asString()),
                               Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
                 clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)),
-                clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)));
+                clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
+                deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)));
+    }
+
+    private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) {
+
+        double queriesPerSecond = object.field(deploymentMetricsQPSField).asDouble();
+        double writesPerSecond = object.field(deploymentMetricsWPSField).asDouble();
+        double documentCount = object.field(deploymentMetricsDocsField).asDouble();
+        double queryLatencyMillis = object.field(deploymentMetricsQueryLatencyField).asDouble();
+        double writeLatencyMills = object.field(deploymentMetricsWriteLatencyField).asDouble();
+
+        return new DeploymentMetrics(queriesPerSecond, writesPerSecond,
+                documentCount, queryLatencyMillis, writeLatencyMills);
     }
 
     private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime(Inspector object) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 42e89e7893f..379581e46fb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -62,8 +62,9 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterCost;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentCost;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
@@ -433,20 +434,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         toSlime(appCost, costObject);
 
         // Metrics
-        com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, instanceName);
-        Zone zoneId = new Zone(Environment.from(environment), RegionName.from(region));
-        try {
-            MetricsService.DeploymentMetrics metrics = controller.metricsService().getDeploymentMetrics(applicationId, zoneId);
-            Cursor metricsObject = response.setObject("metrics");
-            metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond());
-            metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond());
-            metricsObject.setDouble("documentCount", metrics.documentCount());
-            metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis());
-            metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis());
-        }
-        catch (RuntimeException e) {
-            log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e));
-        }
+        DeploymentMetrics metrics = deployment.metrics();
+        Cursor metricsObject = response.setObject("metrics");
+        metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond());
+        metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond());
+        metricsObject.setDouble("documentCount", metrics.documentCount());
+        metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis());
+        metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis());
 
         return new SlimeJsonResponse(slime);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 3e73bf4445b..991f762271f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import org.junit.Test;
@@ -61,7 +62,7 @@ public class ApplicationSerializerTest {
         ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1"));
         deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils
         deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
-                createClusterUtils(3, 0.2), createClusterInfo(3, 4)));
+                createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6)));
 
         Optional<Long> projectId = Optional.of(123L);
         List<JobStatus> statusList = new ArrayList<>();
@@ -123,6 +124,13 @@ public class ApplicationSerializerTest {
         assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor());
         assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size());
 
+        // Test metrics
+        assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
+        assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
+        assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
+        assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
+        assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
+
         { // test more deployment serialization cases
             Application original2 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1"))));
             Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 13b1165ccb2..e48afbe8920 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
@@ -173,7 +174,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET),
                               new File("application.json"));
         // GET an application deployment
-        addMockObservedApplicationCost(controllerTester);
+        setDeploymentMaintainedInfo(controllerTester);
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET),
                               new File("deployment.json"));
         // POST a 'restart application' command
@@ -732,7 +733,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
         controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest);
     }
 
-    private void addMockObservedApplicationCost(ContainerControllerTester controllerTester) {
+    /**
+     * Cluster info, utilization and deployment metrics are maintained async by maintainers.
+     *
+     * This sets these values as if the maintainers has been ran.
+     *
+     * @param controllerTester
+     */
+    private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) {
         for (Application application : controllerTester.controller().applications().asList()) {
             try (Lock lock = controllerTester.controller().applications().lock(application.id())) {
                 for (Deployment deployment : application.deployments().values()) {
@@ -745,6 +753,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                     clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3));
                     deployment = deployment.withClusterInfo(clusterInfo);
                     deployment = deployment.withClusterUtils(clusterUtils);
+                    deployment = deployment.withMetrics(new DeploymentMetrics(1,2,3,4,5));
                     application = application.with(deployment);
                     controllerTester.controller().applications().store(application, lock);
                 }