Store instant since which no nodes were allowed to be down in run

5 files changed, 38 insertions, 19 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index 2851d4ccbdb..31922f883cd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -35,12 +35,13 @@ public class Run {
     private final RunStatus status;
     private final long lastTestRecord;
     private final Instant lastVespaLogTimestamp;
+    private final Optional<Instant> noNodesDownSince;
     private final Optional<X509Certificate> testerCertificate;
 
     // For deserialisation only -- do not use!
     public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start,
                Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp,
-               Optional<X509Certificate> testerCertificate) {
+               Optional<Instant> noNodesDownSince, Optional<X509Certificate> testerCertificate) {
         this.id = id;
         this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
         this.versions = versions;
@@ -49,6 +50,7 @@ public class Run {
         this.status = status;
         this.lastTestRecord = lastTestRecord;
         this.lastVespaLogTimestamp = lastVespaLogTimestamp;
+        this.noNodesDownSince = noNodesDownSince;
         this.testerCertificate = testerCertificate;
     }
 
@@ -56,7 +58,7 @@ public class Run {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class);
         JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step)));
         return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running,
-                       -1, Instant.EPOCH, Optional.empty());
+                       -1, Instant.EPOCH, Optional.empty(), Optional.empty());
     }
 
     /** Returns a new Run with the status of the given completed step set accordingly. */
@@ -70,7 +72,7 @@ public class Run {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), stepInfo.with(Step.Status.of(status)));
         return new Run(id, steps, versions, start, end, this.status == running ? status : this.status,
-                       lastTestRecord, lastVespaLogTimestamp, testerCertificate);
+                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, testerCertificate);
     }
 
     /** Returns a new Run with a new start time*/
@@ -84,37 +86,44 @@ public class Run {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), stepInfo.with(startTime));
 
-        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, testerCertificate);
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, testerCertificate);
     }
 
     public Run finished(Instant now) {
         requireActive();
         return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status,
-                       lastTestRecord, lastVespaLogTimestamp, Optional.empty());
+                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.empty());
     }
 
     public Run aborted() {
         requireActive();
-        return new Run(id, steps, versions, start, end, aborted,
-                       lastTestRecord, lastVespaLogTimestamp, testerCertificate);
+        return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, testerCertificate);
     }
 
     public Run with(long lastTestRecord) {
         requireActive();
-        return new Run(id, steps, versions, start, end, status,
-                       lastTestRecord, lastVespaLogTimestamp, testerCertificate);
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, testerCertificate);
     }
 
     public Run with(Instant lastVespaLogTimestamp) {
         requireActive();
-        return new Run(id, steps, versions, start, end, status,
-                       lastTestRecord, lastVespaLogTimestamp, testerCertificate);
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, testerCertificate);
+    }
+
+    public Run noNodesDownSince(Instant noNodesDownSince) {
+        requireActive();
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       Optional.of(noNodesDownSince), testerCertificate);
     }
 
     public Run with(X509Certificate testerCertificate) {
         requireActive();
-        return new Run(id, steps, versions, start, end, status,
-                       lastTestRecord, lastVespaLogTimestamp, Optional.of(testerCertificate));
+        return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+                       noNodesDownSince, Optional.of(testerCertificate));
     }
 
     /** Returns the id of this run. */
@@ -190,6 +199,11 @@ public class Run {
         return lastVespaLogTimestamp;
     }
 
+    /** Returns the timestamp of the last time no nodes were allowed to be down. */
+    public Optional<Instant> noNodesDownSince() {
+        return noNodesDownSince;
+    }
+
     /** Returns the tester certificate for this run, or empty. */
     public Optional<X509Certificate> testerCertificate() {
         return testerCertificate;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 8ffa6e65a42..74ac37c3c0c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -92,6 +92,7 @@ class RunSerializer {
     private static final String sourceField = "source";
     private static final String lastTestRecordField = "lastTestRecord";
     private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp";
+    private static final String noNodesDownSinceField = "noNodesDownSince";
     private static final String testerCertificateField = "testerCertificate";
 
     Run runFromSlime(Slime slime) {
@@ -129,12 +130,11 @@ class RunSerializer {
                        steps,
                        versionsFromSlime(runObject.field(versionsField)),
                        Instant.ofEpochMilli(runObject.field(startField).asLong()),
-                       Optional.of(runObject.field(endField))
-                               .filter(Inspector::valid)
-                               .map(end -> Instant.ofEpochMilli(end.asLong())),
+                       Serializers.optionalInstant(runObject.field(endField)),
                        runStatusOf(runObject.field(statusField).asString()),
                        runObject.field(lastTestRecordField).asLong(),
                        Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS),
+                       Serializers.optionalInstant(runObject.field(noNodesDownSinceField)),
                        Optional.of(runObject.field(testerCertificateField))
                                .filter(Inspector::valid)
                                .map(certificate -> X509CertificateUtils.fromPem(certificate.asString())));
@@ -194,6 +194,7 @@ class RunSerializer {
         runObject.setString(statusField, valueOf(run.status()));
         runObject.setLong(lastTestRecordField, run.lastTestLogEntry());
         runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS));
+        run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli()));
         run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate)));
 
         Cursor stepsObject = runObject.setObject(stepsField);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
index b2a6a75b937..87a4fdbb4ef 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java
@@ -26,13 +26,13 @@ public class BadgesTest {
 
     private static final ApplicationId id = ApplicationId.from("tenant", "application", "default");
     private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty());
+                                               null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty(), Optional.empty());
 
     private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty());
+                                               null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty(), Optional.empty());
 
     private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())),
-                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty());
+                                               null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty(), Optional.empty());
 
     @Test
     public void test() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index e1017173fcb..a569a55eb9e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -75,6 +75,7 @@ public class RunSerializerTest {
 
         assertEquals(id, run.id());
         assertEquals(start, run.start());
+        assertEquals(Optional.of(Instant.ofEpochMilli(321321321321L)), run.noNodesDownSince());
         assertFalse(run.hasEnded());
         assertEquals(running, run.status());
         assertEquals(3, run.lastTestLogEntry());
@@ -127,6 +128,7 @@ public class RunSerializerTest {
 
         run = run.with(1L << 50)
                  .with(Instant.now().truncatedTo(MILLIS))
+                 .noNodesDownSince(Instant.now().truncatedTo(MILLIS))
                  .aborted()
                  .finished(Instant.now().truncatedTo(MILLIS));
         assertEquals(aborted, run.status());
@@ -138,6 +140,7 @@ public class RunSerializerTest {
         assertEquals(run.end(), phoenix.end());
         assertEquals(run.status(), phoenix.status());
         assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry());
+        assertEquals(run.noNodesDownSince(), phoenix.noNodesDownSince());
         assertEquals(run.testerCertificate(), phoenix.testerCertificate());
         assertEquals(run.versions(), phoenix.versions());
         assertEquals(run.steps(), phoenix.steps());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index 201192280fe..962b6baa88d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -7,6 +7,7 @@
     "status": "running",
     "lastTestRecord": 3,
     "lastVespaLogTimestamp": 1196676930000432,
+    "noNodesDownSince": 321321321321,
     "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----",
     "steps": {
       "deployInitialReal": "unfinished",