Expose number of retiring nodes in deployment log

8 files changed, 58 insertions, 17 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java
index d874a8042f2..1c204159d2c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java
@@ -22,9 +22,10 @@ public class ConvergenceSummary {
     private final long restarting;
     private final long services;
     private final long needNewConfig;
+    private final long retiring;
 
     public ConvergenceSummary(long nodes, long down, long upgradingOs, long upgradingFirmware, long needPlatformUpgrade, long upgradingPlatform,
-                              long needReboot, long rebooting, long needRestart, long restarting, long services, long needNewConfig) {
+                              long needReboot, long rebooting, long needRestart, long restarting, long services, long needNewConfig, long retiring) {
         this.nodes = nodes;
         this.down = down;
         this.upgradingOs = upgradingOs;
@@ -37,6 +38,7 @@ public class ConvergenceSummary {
         this.restarting = restarting;
         this.services = services;
         this.needNewConfig = needNewConfig;
+        this.retiring = retiring;
     }
 
     /** Number of nodes in the application. */
@@ -99,6 +101,11 @@ public class ConvergenceSummary {
         return needNewConfig;
     }
 
+    /** Number of nodes that are retiring. */
+    public long retiring() {
+        return retiring;
+    }
+
     /** Whether the convergence is done. */
     public boolean converged() {
         return     nodes > 0
@@ -125,12 +132,13 @@ public class ConvergenceSummary {
                needRestart == that.needRestart &&
                restarting == that.restarting &&
                services == that.services &&
-               needNewConfig == that.needNewConfig;
+               needNewConfig == that.needNewConfig &&
+               retiring == that.retiring;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(nodes, down, upgradingOs, upgradingFirmware, needPlatformUpgrade, upgradingPlatform, needReboot, rebooting, needRestart, restarting, services, needNewConfig);
+        return Objects.hash(nodes, down, upgradingOs, upgradingFirmware, needPlatformUpgrade, upgradingPlatform, needReboot, rebooting, needRestart, restarting, services, needNewConfig, retiring);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
index 080d600005f..3c3cbfe2b2a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
@@ -87,6 +87,12 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
         return matching(NodeWithServices::needsNewConfig);
     }
 
+    /** The nodes that are retiring. */
+    public NodeList retiring() {
+        return matching(node -> node.node().retired());
+    }
+
+
     /** Returns a summary of the convergence status of the nodes in this list. */
     public ConvergenceSummary summary() {
         NodeList allowedDown = expectedDown();
@@ -101,7 +107,8 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
                                       needsRestart().size(),
                                       allowedDown.needsRestart().size(),
                                       asList().stream().mapToLong(node -> node.services().size()).sum(),
-                                      asList().stream().mapToLong(node -> node.services().stream().filter(service -> wantedConfigGeneration > service.currentGeneration()).count()).sum());
+                                      asList().stream().mapToLong(node -> node.services().stream().filter(service -> wantedConfigGeneration > service.currentGeneration()).count()).sum(),
+                                      retiring().size());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 0ecd86a4a38..91d85e62fcb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -95,7 +95,8 @@ class RunSerializer {
     private static final String lastTestRecordField = "lastTestRecord";
     private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp";
     private static final String noNodesDownSinceField = "noNodesDownSince";
-    private static final String convergenceSummaryField = "convergenceSummary";
+    private static final String oldConvergenceSummaryField = "convergenceSummary"; // TODO (freva): Remove after 7.410
+    private static final String convergenceSummaryField = "convergenceSummaryV2";
     private static final String testerCertificateField = "testerCertificate";
 
     Run runFromSlime(Slime slime) {
@@ -136,7 +137,8 @@ class RunSerializer {
                        runObject.field(lastTestRecordField).asLong(),
                        Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS),
                        Serializers.optionalInstant(runObject.field(noNodesDownSinceField)),
-                       convergenceSummaryFrom(runObject.field(convergenceSummaryField)),
+                       convergenceSummaryFrom(runObject.field(convergenceSummaryField))
+                               .or(() ->convergenceSummaryFrom(runObject.field(oldConvergenceSummaryField))),
                        Optional.of(runObject.field(testerCertificateField))
                                .filter(Inspector::valid)
                                .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())));
@@ -178,11 +180,10 @@ class RunSerializer {
 
     // Don't change this — introduce a separate array instead.
     private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) {
-        if ( ! summaryArray.valid() || summaryArray.entries() == 11) // TODO jonmv: fix
-            return Optional.empty();
+        if ( ! summaryArray.valid()) return Optional.empty();
 
-        if (summaryArray.entries() != 12)
-            throw new IllegalArgumentException("Convergence summary must have 12 entries");
+        if (summaryArray.entries() != 12 && summaryArray.entries() != 13)
+            throw new IllegalArgumentException("Convergence summary must have 13 entries");
 
         return Optional.of(new ConvergenceSummary(summaryArray.entry(0).asLong(),
                                                   summaryArray.entry(1).asLong(),
@@ -195,7 +196,8 @@ class RunSerializer {
                                                   summaryArray.entry(8).asLong(),
                                                   summaryArray.entry(9).asLong(),
                                                   summaryArray.entry(10).asLong(),
-                                                  summaryArray.entry(11).asLong()));
+                                                  summaryArray.entry(11).asLong(),
+                                                  summaryArray.entry(12).asLong()));
     }
 
     Slime toSlime(Iterable<Run> runs) {
@@ -221,7 +223,10 @@ class RunSerializer {
         runObject.setLong(lastTestRecordField, run.lastTestLogEntry());
         runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS));
         run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli()));
-        run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField)));
+        run.convergenceSummary().ifPresent(convergenceSummary -> {
+            toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField), false);
+            toSlime(convergenceSummary, runObject.setArray(oldConvergenceSummaryField), true);
+        });
         run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate)));
 
         Cursor stepsObject = runObject.setObject(stepsField);
@@ -258,7 +263,7 @@ class RunSerializer {
     }
 
     // Don't change this - introduce a separate array with new values if needed.
-    private void toSlime(ConvergenceSummary summary, Cursor summaryArray) {
+    private void toSlime(ConvergenceSummary summary, Cursor summaryArray, boolean oldFormat) {
         summaryArray.addLong(summary.nodes());
         summaryArray.addLong(summary.down());
         summaryArray.addLong(summary.upgradingOs());
@@ -271,6 +276,8 @@ class RunSerializer {
         summaryArray.addLong(summary.restarting());
         summaryArray.addLong(summary.services());
         summaryArray.addLong(summary.needNewConfig());
+        if (!oldFormat)
+            summaryArray.addLong(summary.retiring());
     }
 
     static String valueOf(Step step) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 622a7033519..dba618d139c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -213,6 +213,7 @@ class JobControllerApiHandlerHelper {
         summaryObject.setLong("upgradingFirmware", summary.upgradingFirmware());
         summaryObject.setLong("services", summary.services());
         summaryObject.setLong("needNewConfig", summary.needNewConfig());
+        summaryObject.setLong("retiring", summary.retiring());
     }
 
     private static void toSlime(Cursor entryArray, List<LogEntry> entries) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index b0a0cdf6293..8df2fd87398 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -19,12 +19,14 @@ import com.yahoo.vespa.hosted.controller.deployment.StepInfo;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Optional;
+import java.util.function.BiConsumer;
 
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
@@ -101,7 +103,7 @@ public class RunSerializerTest {
                                                                 "badb17"),
                                              122),
                      run.versions().sourceApplication().get());
-        assertEquals(Optional.of(new ConvergenceSummary(1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144)),
+        assertEquals(Optional.of(new ConvergenceSummary(1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233)),
                      run.convergenceSummary());
         assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" +
                                                   "MIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\n" +
@@ -153,4 +155,18 @@ public class RunSerializerTest {
         assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial)));
     }
 
+    @Test
+    public void convergenceSummaryMigrationTest() throws IOException {
+        String data = Files.readString(runFile);
+        BiConsumer<String, ConvergenceSummary> replaceAndAssert = (replace, convergenceSummaryOrNull) -> {
+            byte[] newData = data.replace("\"convergenceSummaryV2\": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233],", replace).getBytes(StandardCharsets.UTF_8);
+            assertEquals(convergenceSummaryOrNull, serializer.runsFromSlime(SlimeUtils.jsonToSlime(newData)).get(id).convergenceSummary().orElse(null));
+        };
+
+        replaceAndAssert.accept("", null);
+        replaceAndAssert.accept("\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0));
+        replaceAndAssert.accept("\"convergenceSummaryV2\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],\n" +
+                                "\"convergenceSummary\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],", new ConvergenceSummary(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13));
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index a7e5d249a9d..0f40dd27664 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -8,7 +8,7 @@
     "lastTestRecord": 3,
     "lastVespaLogTimestamp": 1196676930000432,
     "noNodesDownSince": 321321321321,
-    "convergenceSummary": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144],
+    "convergenceSummaryV2": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233],
     "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----",
     "steps": {
       "deployInitialReal": "unfinished",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
index 895b49157db..72411d155c7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
@@ -70,7 +70,8 @@
         "upgradingOs": 0,
         "upgradingFirmware": 0,
         "services": 1,
-        "needNewConfig": 1
+        "needNewConfig": 1,
+        "retiring": 0
       }
     },
     "copyVespaLogs": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
index 5aa13f1cf3c..2cf846ab6bb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
@@ -149,7 +149,8 @@
         "upgradingOs": 0,
         "upgradingFirmware": 0,
         "services": 1,
-        "needNewConfig": 1
+        "needNewConfig": 1,
+        "retiring": 0
       }
     },
     "startStagingSetup": {