Average deployment duration metric

1 files changed, 61 insertions, 4 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index d57f01bd132..93fdf3618a7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -26,11 +26,14 @@ import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Clock;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.Map;
 
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -54,8 +57,9 @@ public class MetricsReporterTest {
 
     @Test
     public void test_chef_metrics() {
+        Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault());;
         ControllerTester tester = new ControllerTester();
-        MetricsReporter metricsReporter = createReporter(tester.controller(), metrics, SystemName.cd);
+        MetricsReporter metricsReporter = createReporter(clock, tester.controller(), metrics, SystemName.cd);
         metricsReporter.maintain();
         assertEquals(2, metrics.getMetrics().size());
 
@@ -95,7 +99,7 @@ public class MetricsReporterTest {
         assertEquals(0.0, metrics.getMetric(MetricsReporter.deploymentFailMetric));
 
         // 1 app fails system-test
-        tester.jobCompletion(component).application(app4).submit();
+        tester.jobCompletion(component).application(app4).nextBuildNumber().uploadArtifact(applicationPackage).submit();
         tester.deployAndNotify(app4, applicationPackage, false, systemTest);
 
         metricsReporter.maintain();
@@ -110,11 +114,66 @@ public class MetricsReporterTest {
         assertNull(metricContext.getDimensions().get("zone"));
     }
 
+    @Test
+    public void test_deployment_average_duration() {
+        DeploymentTester tester = new DeploymentTester();
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .region("us-west-1")
+                .build();
+
+        MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.cd);
+
+        Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+        tester.deployCompletely(app, applicationPackage);
+        reporter.maintain();
+        assertEquals(Duration.ZERO, getAverageDeploymentDuration(app)); // An exceptionally fast deployment :-)
+
+        // App spends 3 hours deploying
+        tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit();
+        tester.clock().advance(Duration.ofHours(1));
+        tester.deployAndNotify(app, applicationPackage, true, systemTest);
+
+        tester.clock().advance(Duration.ofMinutes(30));
+        tester.deployAndNotify(app, applicationPackage, true, stagingTest);
+
+        tester.clock().advance(Duration.ofMinutes(90));
+        tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
+        reporter.maintain();
+
+        // Average time is 1 hour
+        assertEquals(Duration.ofHours(1), getAverageDeploymentDuration(app));
+
+        // Another deployment starts and stalls for 12 hours
+        tester.jobCompletion(component).application(app).nextBuildNumber(2).uploadArtifact(applicationPackage).submit();
+        tester.clock().advance(Duration.ofHours(12));
+        reporter.maintain();
+
+        assertEquals(Duration.ofHours(12) // hanging system-test
+                             .plus(Duration.ofMinutes(30)) // previous staging-test
+                             .plus(Duration.ofMinutes(90)) // previous production job
+                             .dividedBy(3), // Total number of orchestrated jobs
+                     getAverageDeploymentDuration(app));
+    }
+
+    private Duration getAverageDeploymentDuration(Application application) {
+        return metrics.getMetric((dimension, value) -> dimension.equals("application") &&
+                                                       value.equals(application.id().toString()),
+                                 MetricsReporter.deploymentAverageDuration)
+                      .map(seconds -> Duration.ofSeconds(seconds.longValue()))
+                      .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + application.id()));
+    }
+
     private void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) {
         assertEquals(expectedValue, metricContext.getDimensions().get(dimensionName));
     }
 
     private MetricsReporter createReporter(Controller controller, MetricsMock metricsMock, SystemName system) {
+        return createReporter(controller.clock(), controller, metricsMock, system);
+    }
+
+    private MetricsReporter createReporter(Clock clock, Controller controller, MetricsMock metricsMock,
+                                           SystemName system) {
         Chef client = Mockito.mock(Chef.class);
         PartialNodeResult result;
         try {
@@ -126,8 +185,6 @@ public class MetricsReporterTest {
         }
         when(client.partialSearchNodes(anyString(), anyListOf(AttributeMapping.class))).thenReturn(result);
 
-        Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault());
-
         return new MetricsReporter(controller, metricsMock, client, clock, new JobControl(new MockCuratorDb()), system);
     }