Merge pull request #11951 from vespa-engine/jvenstad/adaptive-deployment-job-timeout

Jvenstad/adaptive deployment job timeout

4 files changed, 17 insertions, 7 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 674ff733bb8..e0b83670027 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -380,12 +380,12 @@ public class InternalStepRunner implements StepRunner {
         }
 
         if ( ! firstTick)
-            logger.log(nodeList.allowedDown().asList().stream()
+            logger.log(nodeList.expectedDown().asList().stream()
                                .flatMap(node -> nodeDetails(node, false))
                                .collect(toList()));
 
         controller.jobController().locked(id, lockedRun -> {
-            Instant noNodesDownSince = summary.down() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null;
+            Instant noNodesDownSince = nodeList.allowedDown().size() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null;
             return lockedRun.noNodesDownSince(noNodesDownSince).withSummary(summary);
         });
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index c8cfc8ac286..0c2b6ee1744 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -1,6 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -226,9 +227,14 @@ public class JobController {
 
     /** Returns an immutable map of all known runs for the given application and job type. */
     public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) {
-        NavigableMap<RunId, Run> runs = curator.readHistoricRuns(id, type);
-        last(id, type).ifPresent(run -> runs.put(run.id(), run));
-        return Collections.unmodifiableNavigableMap(runs);
+        ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number));
+        Optional<Run> last = last(id, type);
+        curator.readHistoricRuns(id, type).forEach((runId, run) -> {
+            if (last.isEmpty() || ! runId.equals(last.get().id()))
+                runs.put(runId, run);
+        });
+        last.ifPresent(run -> runs.put(run.id(), run));
+        return runs.build();
     }
 
     /** Returns the run with the given id, if it exists. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
index d3533fc5200..09c672cf3a0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
@@ -68,6 +68,11 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
 
     /** The nodes currently allowed to be down. */
     public NodeList allowedDown() {
+        return matching(node -> node.isAllowedDown());
+    }
+
+    /** The nodes currently expected to be down. */
+    public NodeList expectedDown() {
         return matching(node -> node.isAllowedDown() || node.isNewlyProvisioned());
     }
 
@@ -83,7 +88,7 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
 
     /** Returns a summary of the convergence status of the nodes in this list. */
     public ConvergenceSummary summary() {
-        NodeList allowedDown = allowedDown();
+        NodeList allowedDown = expectedDown();
         return new ConvergenceSummary(size(),
                                       allowedDown.size(),
                                       withParentDown().needsOsUpgrade().size(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 6d212afb72a..2c51a9bdc00 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -387,7 +387,6 @@ public class CuratorDb {
 
     public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) {
         Path path = runsPath(id, type);
-        cachedHistoricRuns.remove(path);
         curator.set(path, asJson(runSerializer.toSlime(runs)));
     }