From 82d187a6f7e3f6f6adbd4430951791c17af17558 Mon Sep 17 00:00:00 2001 From: toby Date: Fri, 13 Oct 2017 12:51:25 +0200 Subject: Decouple deployment metrics from the metricservice - to avoid external calls to metric providers when asking for deployment info --- .../hosted/controller/ApplicationController.java | 5 ++- .../hosted/controller/application/Deployment.java | 22 ++++++++-- .../controller/application/DeploymentMetrics.java | 50 ++++++++++++++++++++++ .../persistence/ApplicationSerializer.java | 34 ++++++++++++++- .../restapi/application/ApplicationApiHandler.java | 24 ++++------- .../persistence/ApplicationSerializerTest.java | 10 ++++- .../restapi/application/ApplicationApiTest.java | 13 +++++- 7 files changed, 133 insertions(+), 25 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 672f50f83d7..1df0b1bca51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -334,9 +334,10 @@ public class ApplicationController { Deployment previousDeployment = application.deployments().get(zone); Deployment newDeployment = previousDeployment; if (previousDeployment == null) { - newDeployment = new Deployment(zone, revision, version, clock.instant(), new HashMap<>(), new HashMap<>()); + newDeployment = new Deployment(zone, revision, version, clock.instant()); } else { - newDeployment = new Deployment(zone, revision, version, clock.instant(), previousDeployment.clusterUtils(), previousDeployment.clusterInfo()); + newDeployment = new Deployment(zone, revision, version, clock.instant(), + previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics()); } application = application.with(newDeployment); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 01219e940a3..05a7f9667ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -14,6 +14,7 @@ import java.util.Objects; * A deployment of an application in a particular zone. * * @author bratseth + * @author smorgrav */ public class Deployment { @@ -23,24 +24,28 @@ public class Deployment { private final Instant deployTime; private final Map clusterUtils; private final Map clusterInfo; + private final DeploymentMetrics metrics; public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) { - this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>()); + this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics()); } - public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, Map clusterUtils, Map clusterInfo) { + public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, + Map clusterUtils, Map clusterInfo, DeploymentMetrics metrics) { Objects.requireNonNull(zone, "zone cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); Objects.requireNonNull(version, "version cannot be null"); Objects.requireNonNull(deployTime, "deployTime cannot be null"); Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null"); Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); + Objects.requireNonNull(clusterInfo, "deployment metrics cannot be null"); this.zone = zone; this.revision = revision; this.version = version; this.deployTime = deployTime; this.clusterUtils = clusterUtils; this.clusterInfo = clusterInfo; + this.metrics = metrics; } /** Returns the zone this was deployed to */ @@ -64,11 +69,20 @@ public class Deployment { } public Deployment withClusterUtils(Map clusterUtilization) { - return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo); + return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo, metrics); } public Deployment withClusterInfo(Map newClusterInfo) { - return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo); + return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo, metrics); + } + + public Deployment withMetrics(DeploymentMetrics metrics) { + return new Deployment(zone, revision, version, deployTime, clusterUtils, clusterInfo, metrics); + } + + /** @return Key metrics for the deployment (application level) like QPS and document count */ + public DeploymentMetrics metrics() { + return metrics; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java new file mode 100644 index 00000000000..6812e4cb468 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -0,0 +1,50 @@ +package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +public class DeploymentMetrics { + + private final double queriesPerSecond; + private final double writesPerSecond; + private final double documentCount; + private final double queryLatencyMillis; + private final double writeLatencyMills; + + DeploymentMetrics() { + this.queriesPerSecond = 0; + this.writesPerSecond = 0; + this.documentCount = 0; + this.queryLatencyMillis = 0; + this.writeLatencyMills = 0; + } + + public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, double documentCount, + double queryLatencyMillis, double writeLatencyMills) { + this.queriesPerSecond = queriesPerSecond; + this.writesPerSecond = writesPerSecond; + this.documentCount = documentCount; + this.queryLatencyMillis = queryLatencyMillis; + this.writeLatencyMills = writeLatencyMills; + } + + public double queriesPerSecond() { + return queriesPerSecond; + } + + public double writesPerSecond() { + return writesPerSecond; + } + + public double documentCount() { + return documentCount; + } + + public double queryLatencyMillis() { + return queryLatencyMillis; + } + + public double writeLatencyMillis() { + return writeLatencyMills; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 859e322b227..494c4bb7936 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -95,6 +96,14 @@ public class ApplicationSerializer { private final String clusterUtilsDiskField = "disk"; private final String clusterUtilsDiskBusyField = "diskbusy"; + // Deployment metrics fields + private final String deploymentMetricsField = "metrics"; + private final String deploymentMetricsQPSField = "queriesPerSecond"; + private final String deploymentMetricsWPSField = "writesPerSecond"; + private final String deploymentMetricsDocsField = "documentCount"; + private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis"; + private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis"; + // ------------------ Serialization @@ -123,6 +132,16 @@ public class ApplicationSerializer { toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); clusterInfoToSlime(deployment.clusterInfo(), object); clusterUtilsToSlime(deployment.clusterUtils(), object); + metricsToSlime(deployment.metrics(), object); + } + + private void metricsToSlime(DeploymentMetrics metrics, Cursor object) { + Cursor root = object.setObject(deploymentMetricsField); + root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond()); + root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond()); + root.setDouble(deploymentMetricsDocsField, metrics.documentCount()); + root.setDouble(deploymentMetricsQueryLatencyField, metrics.queryLatencyMillis()); + root.setDouble(deploymentMetricsWriteLatencyField, metrics.writeLatencyMillis()); } private void clusterInfoToSlime(Map clusters, Cursor object) { @@ -246,7 +265,20 @@ public class ApplicationSerializer { Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), - clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField))); + clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), + deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField))); + } + + private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { + + double queriesPerSecond = object.field(deploymentMetricsQPSField).asDouble(); + double writesPerSecond = object.field(deploymentMetricsWPSField).asDouble(); + double documentCount = object.field(deploymentMetricsDocsField).asDouble(); + double queryLatencyMillis = object.field(deploymentMetricsQueryLatencyField).asDouble(); + double writeLatencyMills = object.field(deploymentMetricsWriteLatencyField).asDouble(); + + return new DeploymentMetrics(queriesPerSecond, writesPerSecond, + documentCount, queryLatencyMillis, writeLatencyMills); } private Map clusterInfoMapFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 42e89e7893f..379581e46fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -62,8 +62,9 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; @@ -433,20 +434,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { toSlime(appCost, costObject); // Metrics - com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, instanceName); - Zone zoneId = new Zone(Environment.from(environment), RegionName.from(region)); - try { - MetricsService.DeploymentMetrics metrics = controller.metricsService().getDeploymentMetrics(applicationId, zoneId); - Cursor metricsObject = response.setObject("metrics"); - metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); - metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); - metricsObject.setDouble("documentCount", metrics.documentCount()); - metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); - metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e)); - } + DeploymentMetrics metrics = deployment.metrics(); + Cursor metricsObject = response.setObject("metrics"); + metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); + metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); + metricsObject.setDouble("documentCount", metrics.documentCount()); + metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); + metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 3e73bf4445b..991f762271f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import org.junit.Test; @@ -61,7 +62,7 @@ public class ApplicationSerializerTest { ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), - createClusterUtils(3, 0.2), createClusterInfo(3, 4))); + createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6))); Optional projectId = Optional.of(123L); List statusList = new ArrayList<>(); @@ -123,6 +124,13 @@ public class ApplicationSerializerTest { assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor()); assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size()); + // Test metrics + assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE); + assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE); + assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE); + assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE); + assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE); + { // test more deployment serialization cases Application original2 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1")))); Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 13b1165ccb2..e48afbe8920 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -173,7 +174,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), new File("application.json")); // GET an application deployment - addMockObservedApplicationCost(controllerTester); + setDeploymentMaintainedInfo(controllerTester); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET), new File("deployment.json")); // POST a 'restart application' command @@ -732,7 +733,14 @@ public class ApplicationApiTest extends ControllerContainerTest { controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest); } - private void addMockObservedApplicationCost(ContainerControllerTester controllerTester) { + /** + * Cluster info, utilization and deployment metrics are maintained async by maintainers. + * + * This sets these values as if the maintainers has been ran. + * + * @param controllerTester + */ + private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) { for (Application application : controllerTester.controller().applications().asList()) { try (Lock lock = controllerTester.controller().applications().lock(application.id())) { for (Deployment deployment : application.deployments().values()) { @@ -745,6 +753,7 @@ public class ApplicationApiTest extends ControllerContainerTest { clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); deployment = deployment.withClusterInfo(clusterInfo); deployment = deployment.withClusterUtils(clusterUtils); + deployment = deployment.withMetrics(new DeploymentMetrics(1,2,3,4,5)); application = application.with(deployment); controllerTester.controller().applications().store(application, lock); } -- cgit v1.2.3