Merge pull request #6352 from vespa-engine/jvenstad/deployment-pipeline

Expand Node interface and check for pending re[boot|start|s

2 files changed, 61 insertions, 8 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index d75f0e90d63..90864730a15 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
+import org.jetbrains.annotations.TestOnly;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -13,24 +14,43 @@ import java.util.Optional;
  * A node in hosted Vespa.
  *
  * @author mpolden
+ * @author jonmv
  */
 public class Node {
 
     private final HostName hostname;
     private final State state;
-    private final Optional<ApplicationId> owner;
     private final NodeType type;
+    private final Optional<ApplicationId> owner;
     private final Version currentVersion;
     private final Version wantedVersion;
+    private final ServiceState serviceState;
+    private final long restartGeneration;
+    private final long wantedRestartGeneration;
+    private final long rebootGeneration;
+    private final long wantedRebootGeneration;
 
-    public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner, Version currentVersion,
-                Version wantedVersion) {
+    public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner,
+                Version currentVersion, Version wantedVersion, ServiceState serviceState,
+                long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration) {
         this.hostname = hostname;
         this.state = state;
         this.type = type;
         this.owner = owner;
         this.currentVersion = currentVersion;
         this.wantedVersion = wantedVersion;
+        this.serviceState = serviceState;
+        this.restartGeneration = restartGeneration;
+        this.wantedRestartGeneration = wantedRestartGeneration;
+        this.rebootGeneration = rebootGeneration;
+        this.wantedRebootGeneration = wantedRebootGeneration;
+    }
+
+    @TestOnly
+    public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner,
+                Version currentVersion, Version wantedVersion) {
+        this(hostname, state, type, owner, currentVersion, wantedVersion,
+             ServiceState.unorchestrated, 0, 0, 0, 0);
     }
 
     public HostName hostname() {
@@ -55,6 +75,26 @@ public class Node {
         return wantedVersion;
     }
 
+    public ServiceState serviceState() {
+        return serviceState;
+    }
+
+    public long restartGeneration() {
+        return restartGeneration;
+    }
+
+    public long wantedRestartGeneration() {
+        return wantedRestartGeneration;
+    }
+
+    public long rebootGeneration() {
+        return rebootGeneration;
+    }
+
+    public long wantedRebootGeneration() {
+        return wantedRebootGeneration;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -80,4 +120,11 @@ public class Node {
         parked
     }
 
+    /** Known node states with regards to service orchestration */
+    public enum ServiceState {
+        expectedUp,
+        allowedDown,
+        unorchestrated
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index e6f2263b8eb..77b6baa6bea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -261,12 +261,18 @@ public class InternalStepRunner implements StepRunner {
         List<Node> nodes = controller.configServer().nodeRepository().list(zone(type), id, Arrays.asList(active, reserved));
         for (Node node : nodes)
             // TODO jvenstad: Add ALLOWED_TO_BE_DOWN and reboot and restart generation information as well.
-            logger.get().log(String.format("%70s: %s%s",
+            logger.get().log(String.format("%70s: %-12s%-25s%-32s%s",
                                            node.hostname(),
-                                           node.wantedVersion(),
-                                           node.currentVersion().equals(node.wantedVersion()) ? "" : " <-- " + node.currentVersion()));
-
-        return nodes.stream().allMatch(node -> target.equals(node.currentVersion()));
+                                           node.serviceState(),
+                                           node.wantedVersion() + (node.currentVersion().equals(node.wantedVersion()) ? "" : " <-- " + node.currentVersion()),
+                                           node.restartGeneration() == node.wantedRestartGeneration() ? ""
+                                                   : "restart pending (" + node.wantedRestartGeneration() + " <-- " + node.restartGeneration() + ")",
+                                           node.rebootGeneration() == node.wantedRebootGeneration() ? ""
+                                                   : "reboot pending (" + node.wantedRebootGeneration() + " <-- " + node.rebootGeneration() + ")"));
+
+        return nodes.stream().allMatch(node ->    node.currentVersion().equals(target)
+                                               && node.restartGeneration() == node.wantedRestartGeneration()
+                                               && node.rebootGeneration() == node.wantedRebootGeneration());
     }
 
     private boolean servicesConverged(ApplicationId id, JobType type) {