Store summary of convergence of application deployment in run

6 files changed, 192 insertions, 15 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java
new file mode 100644
index 00000000000..9a3296fa7d5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java
@@ -0,0 +1,120 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import java.util.Objects;
+
+/**
+ * Summary of node and service status during a deployment job.
+ *
+ * @author jonmv
+ */
+public class ConvergenceSummary {
+
+    private final long nodes;
+    private final long down;
+    private final long upgradingOs;
+    private final long needPlatformUpgrade;
+    private final long upgradingPlatform;
+    private final long needReboot;
+    private final long rebooting;
+    private final long needRestart;
+    private final long restarting;
+    private final long services;
+    private final long upgradingApplication;
+
+    public ConvergenceSummary(long nodes, long down, long upgradingOs, long needPlatformUpgrade, long upgradingPlatform,
+                              long needReboot, long rebooting, long needRestart, long restarting, long services, long upgradingApplication) {
+        this.nodes = nodes;
+        this.down = down;
+        this.upgradingOs = upgradingOs;
+        this.needPlatformUpgrade = needPlatformUpgrade;
+        this.upgradingPlatform = upgradingPlatform;
+        this.needReboot = needReboot;
+        this.rebooting = rebooting;
+        this.needRestart = needRestart;
+        this.restarting = restarting;
+        this.services = services;
+        this.upgradingApplication = upgradingApplication;
+    }
+
+    /** Number of nodes in the application. */
+    public long nodes() {
+        return nodes;
+    }
+
+    /** Number of nodes allowed to be down. */
+    public long down() {
+        return down;
+    }
+
+    /** Number of nodes down for OS upgrade. */
+    public long upgradingOs() {
+        return upgradingOs;
+    }
+
+    /** Number of nodes in need of a platform upgrade. */
+    public long needPlatformUpgrade() {
+        return needPlatformUpgrade;
+    }
+
+    /** Number of nodes down for platform upgrade. */
+    public long upgradingPlatform() {
+        return upgradingPlatform;
+    }
+
+    /** Number of nodes in need of a reboot. */
+    public long needReboot() {
+        return needReboot;
+    }
+
+    /** Number of nodes down for reboot. */
+    public long rebooting() {
+        return rebooting;
+    }
+
+    /** Number of nodes in need of a restart. */
+    public long needRestart() {
+        return needRestart;
+    }
+
+    /** Number of nodes down for restart. */
+    public long restarting() {
+        return restarting;
+    }
+
+    /** Number of services in the application. */
+    public long services() {
+        return services;
+    }
+
+    /** Number of services with outdated config generation. */
+    public long upgradingApplication() {
+        return upgradingApplication;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ConvergenceSummary that = (ConvergenceSummary) o;
+        return nodes == that.nodes &&
+               down == that.down &&
+               upgradingOs == that.upgradingOs &&
+               needPlatformUpgrade == that.needPlatformUpgrade &&
+               upgradingPlatform == that.upgradingPlatform &&
+               needReboot == that.needReboot &&
+               rebooting == that.rebooting &&
+               needRestart == that.needRestart &&
+               restarting == that.restarting &&
+               services == that.services &&
+               upgradingApplication == that.upgradingApplication;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodes, down, upgradingOs, needPlatformUpgrade, upgradingPlatform, needReboot, rebooting, needRestart, restarting, services, upgradingApplication);
+    }
+
+}
+
+
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index 31922f883cd..3a677ce5719 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -36,12 +36,13 @@ public class Run {
     private final long lastTestRecord;
     private final Instant lastVespaLogTimestamp;
     private final Optional<Instant> noNodesDownSince;
+    private final Optional<ConvergenceSummary> convergenceSummary;
     private final Optional<X509Certificate> testerCertificate;
 
     // For deserialisation only -- do not use!
-    public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start,
-               Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp,
-               Optional<Instant> noNodesDownSince, Optional<X509Certificate> testerCertificate) {
+    public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end,
+               RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince,
+               Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) {
         this.id = id;
         this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
         this.versions = versions;
@@ -51,6 +52,7 @@ public class Run {
         this.lastTestRecord = lastTestRecord;
         this.lastVespaLogTimestamp = lastVespaLogTimestamp;
         this.noNodesDownSince = noNodesDownSince;
+        this.convergenceSummary = convergenceSummary;
         this.testerCertificate = testerCertificate;
     }
 
@@ -58,7 +60,7 @@ public class Run {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class);
         JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step)));
         return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running,
-                       -1, Instant.EPOCH, Optional.empty(), Optional.empty());
+                       -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
     }
 
     /** Returns a new Run with the status of the given completed step set accordingly. */
@@ -72,7 +74,7 @@ public class Run {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), stepInfo.with(Step.Status.of(status)));
         return new Run(id, steps, versions, start, end, this.status == running ? status : this.status,
-                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, testerCertificate);
+                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate);
     }
 
     /** Returns a new Run with a new start time*/
@@ -87,43 +89,49 @@ public class Run {
         steps.put(step.get(), stepInfo.with(startTime));
 
         return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
-                       noNodesDownSince, testerCertificate);
+                       noNodesDownSince, convergenceSummary, testerCertificate);
     }
 
     public Run finished(Instant now) {
         requireActive();
         return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status,
-                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.empty());
+                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty());
     }
 
     public Run aborted() {
         requireActive();
         return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp,
-                       noNodesDownSince, testerCertificate);
+                       noNodesDownSince, convergenceSummary, testerCertificate);
     }
 
     public Run with(long lastTestRecord) {
         requireActive();
         return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
-                       noNodesDownSince, testerCertificate);
+                       noNodesDownSince, convergenceSummary, testerCertificate);
     }
 
     public Run with(Instant lastVespaLogTimestamp) {
         requireActive();
         return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
-                       noNodesDownSince, testerCertificate);
+                       noNodesDownSince, convergenceSummary, testerCertificate);
     }
 
     public Run noNodesDownSince(Instant noNodesDownSince) {
         requireActive();
         return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
-                       Optional.of(noNodesDownSince), testerCertificate);
+                       Optional.of(noNodesDownSince), convergenceSummary, testerCertificate);
+    }
+
+    public Run with(ConvergenceSummary convergenceSummary) {
+        requireActive();
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, Optional.of(convergenceSummary), testerCertificate);
     }
 
     public Run with(X509Certificate testerCertificate) {
         requireActive();
         return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
-                       noNodesDownSince, Optional.of(testerCertificate));
+                       noNodesDownSince, convergenceSummary, Optional.of(testerCertificate));
     }
 
     /** Returns the id of this run. */
@@ -204,6 +212,11 @@ public class Run {
         return noNodesDownSince;
     }
 
+    /** Returns a summary of convergence status during an application deployment — staging or upgrade. */
+    public Optional<ConvergenceSummary> convergenceSummary() {
+        return convergenceSummary;
+    }
+
     /** Returns the tester certificate for this run, or empty. */
     public Optional<X509Certificate> testerCertificate() {
         return testerCertificate;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 74ac37c3c0c..33ef5ca49f0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
@@ -93,6 +94,7 @@ class RunSerializer {
     private static final String lastTestRecordField = "lastTestRecord";
     private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp";
     private static final String noNodesDownSinceField = "noNodesDownSince";
+    private static final String convergenceSummaryField = "convergenceSummary";
     private static final String testerCertificateField = "testerCertificate";
 
     Run runFromSlime(Slime slime) {
@@ -135,6 +137,7 @@ class RunSerializer {
                        runObject.field(lastTestRecordField).asLong(),
                        Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS),
                        Serializers.optionalInstant(runObject.field(noNodesDownSinceField)),
+                       convergenceSummaryFrom(runObject.field(convergenceSummaryField)),
                        Optional.of(runObject.field(testerCertificateField))
                                .filter(Inspector::valid)
                                .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())));
@@ -172,6 +175,27 @@ class RunSerializer {
                                       compileVersion, buildTime, sourceUrl, commit);
     }
 
+    // Don't change this — introduce a separate array instead.
+    private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) {
+        if ( ! summaryArray.valid())
+            return Optional.empty();
+
+        if (summaryArray.entries() != 11)
+            throw new IllegalArgumentException("Convergence summary must have 11 entries");
+
+        return Optional.of(new ConvergenceSummary(summaryArray.entry(0).asLong(),
+                                                  summaryArray.entry(1).asLong(),
+                                                  summaryArray.entry(2).asLong(),
+                                                  summaryArray.entry(3).asLong(),
+                                                  summaryArray.entry(4).asLong(),
+                                                  summaryArray.entry(5).asLong(),
+                                                  summaryArray.entry(6).asLong(),
+                                                  summaryArray.entry(7).asLong(),
+                                                  summaryArray.entry(8).asLong(),
+                                                  summaryArray.entry(9).asLong(),
+                                                  summaryArray.entry(10).asLong()));
+    }
+
     Slime toSlime(Iterable<Run> runs) {
         Slime slime = new Slime();
         Cursor runArray = slime.setArray();
@@ -195,6 +219,7 @@ class RunSerializer {
         runObject.setLong(lastTestRecordField, run.lastTestLogEntry());
         runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS));
         run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli()));
+        run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField)));
         run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate)));
 
         Cursor stepsObject = runObject.setObject(stepsField);
@@ -231,6 +256,21 @@ class RunSerializer {
         applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit));
     }
 
+    // Don't change this — introduce a separate array with new values if needed.
+    private void toSlime(ConvergenceSummary summary, Cursor summaryArray) {
+        summaryArray.addLong(summary.nodes());
+        summaryArray.addLong(summary.down());
+        summaryArray.addLong(summary.upgradingOs());
+        summaryArray.addLong(summary.needPlatformUpgrade());
+        summaryArray.addLong(summary.upgradingPlatform());
+        summaryArray.addLong(summary.needReboot());
+        summaryArray.addLong(summary.rebooting());
+        summaryArray.addLong(summary.needRestart());
+        summaryArray.addLong(summary.restarting());
+        summaryArray.addLong(summary.services());
+        summaryArray.addLong(summary.upgradingApplication());
+    }
+
     static String valueOf(Step step) {
         switch (step) {
             case deployInitialReal  : return "deployInitialReal";
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
index 87a4fdbb4ef..06d5a42f9c0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
@@ -26,13 +26,13 @@ public class BadgesTest {
 
     private static final ApplicationId id = ApplicationId.from("tenant", "application", "default");
     private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty(), Optional.empty());
+                                               null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
 
     private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty(), Optional.empty());
+                                               null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
 
     private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty(), Optional.empty());
+                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
 
     @Test
     public void test() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index a569a55eb9e..3839e2103cd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
@@ -99,6 +100,8 @@ public class RunSerializerTest {
                                                                 "badb17"),
                                              122),
                      run.versions().sourceApplication().get());
+        assertEquals(Optional.of(new ConvergenceSummary(1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89)),
+                     run.convergenceSummary());
         assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" +
                                                   "MIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\n" +
                                                   "ZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index 962b6baa88d..a66b9d3e955 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -8,6 +8,7 @@
     "lastTestRecord": 3,
     "lastVespaLogTimestamp": 1196676930000432,
     "noNodesDownSince": 321321321321,
+    "convergenceSummary": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89],
     "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----",
     "steps": {
       "deployInitialReal": "unfinished",