Allow no progress for 2 hours while redistributing docs

11 files changed, 87 insertions, 81 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index af36708ccd1..3d04c29ac51 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -121,7 +121,8 @@ public class InternalStepRunner implements StepRunner {
     static final Duration endpointTimeout = Duration.ofMinutes(15);
     static final Duration endpointCertificateTimeout = Duration.ofMinutes(15);
     static final Duration testerTimeout = Duration.ofMinutes(30);
-    static final Duration installationTimeout = Duration.ofMinutes(60);
+    static final Duration nodesDownTimeout = Duration.ofMinutes(60);
+    static final Duration noNodesDownTimeout = Duration.ofMinutes(120);
     static final Duration certificateTimeout = Duration.ofMinutes(300);
 
     private final Controller controller;
@@ -358,17 +359,17 @@ public class InternalStepRunner implements StepRunner {
 
         String failureReason = null;
 
-        NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(installationTimeout));
+        NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(nodesDownTimeout));
         if ( ! suspendedTooLong.isEmpty()) {
-            failureReason = "Some nodes have been suspended for more than " + installationTimeout.toMinutes() + " minutes:\n" +
+            failureReason = "Some nodes have been suspended for more than " + nodesDownTimeout.toMinutes() + " minutes:\n" +
                             suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n"));
         }
 
         if (run.noNodesDownSince()
-               .map(since -> since.isBefore(controller.clock().instant().minus(installationTimeout)))
+               .map(since -> since.isBefore(controller.clock().instant().minus(noNodesDownTimeout)))
                .orElse(false)) {
             if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0)
-                failureReason ="No nodes allowed to suspend to progress installation for " + installationTimeout.toMinutes() + " minutes.";
+                failureReason = "No nodes allowed to suspend to progress installation for " + noNodesDownTimeout.toMinutes() + " minutes.";
             else
                 failureReason = "Nodes not able to start with new application package.";
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 36ab6a9aa60..4550b29d2fd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -364,7 +364,7 @@ public class DeploymentContext {
         triggerJobs();
         RunId id = currentRun(job).id();
         doDeploy(job);
-        tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1));
+        tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plusSeconds(1));
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasFailed());
         assertTrue(jobs.run(id).get().hasEnded());
@@ -378,7 +378,7 @@ public class DeploymentContext {
         RunId id = currentRun(job).id();
         doDeploy(job);
         doUpgrade(job);
-        tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1));
+        tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plusSeconds(1));
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasFailed());
         assertTrue(jobs.run(id).get().hasEnded());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 44fd677a4d3..a3d1b9adb35 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -159,7 +159,7 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal));
 
-        tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plus(Duration.ofSeconds(1)));
         tester.runner().run();
         assertEquals(installationFailed, tester.jobs().run(id).get().status());
     }
@@ -211,7 +211,7 @@ public class InternalStepRunnerTest {
 
         Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()),
                                                                           app.instanceId()).iterator().next();
-        tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.noNodesDownTimeout.minus(Duration.ofSeconds(1)));
         tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()),
                                                              new Node.Builder(systemTestNode)
                                                                      .serviceState(Node.ServiceState.allowedDown)
@@ -226,7 +226,7 @@ public class InternalStepRunnerTest {
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
         assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal));
 
-        tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(3)));
+        tester.clock().advance(InternalStepRunner.nodesDownTimeout.minus(Duration.ofSeconds(3)));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 6b71968822f..17d234b0c0c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -82,7 +82,7 @@ public class JobControllerApiHandlerHelperTest {
         assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status());
 
         tester.runner().run();
-        tester.clock().advance(Duration.ofHours(1).plusSeconds(1));
+        tester.clock().advance(Duration.ofHours(2).plusSeconds(1));
         tester.runner().run();
         assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status());
         assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
index 65db44cb4e2..868809978de 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
@@ -128,8 +128,8 @@
         {
           "id": 3,
           "url": "https://some.url:43/instance/default/job/system-test/run/3",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
@@ -320,9 +320,9 @@
       "dependencies": [],
       "declared": true,
       "instance": "default",
-      "readyAt": 4353000,
-      "delayedUntil": 4353000,
-      "coolingDownUntil": 4353000,
+      "readyAt": 7953000,
+      "delayedUntil": 7953000,
+      "coolingDownUntil": 7953000,
       "jobName": "staging-test",
       "url": "https://some.url:43/instance/default/job/staging-test",
       "environment": "staging",
@@ -351,8 +351,8 @@
         {
           "id": 5,
           "url": "https://some.url:43/instance/default/job/staging-test/run/5",
-          "start": 3703000,
-          "end": 3703000,
+          "start": 7303000,
+          "end": 7303000,
           "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
@@ -432,8 +432,8 @@
         {
           "id": 4,
           "url": "https://some.url:43/instance/default/job/staging-test/run/4",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
@@ -513,8 +513,8 @@
         {
           "id": 3,
           "url": "https://some.url:43/instance/default/job/staging-test/run/3",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
@@ -756,7 +756,7 @@
       ],
       "declared": true,
       "instance": "default",
-      "readyAt": 3603000,
+      "readyAt": 7203000,
       "jobName": "production-us-central-1",
       "url": "https://some.url:43/instance/default/job/production-us-central-1",
       "environment": "prod",
@@ -773,7 +773,7 @@
         {
           "id": 3,
           "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3",
-          "start": 3603000,
+          "start": 7203000,
           "status": "running",
           "versions": {
             "targetPlatform": "6.1.0",
@@ -1050,7 +1050,7 @@
           "id": 2,
           "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2",
           "start": 1000,
-          "end": 3602000,
+          "end": 7202000,
           "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
index f7185d373c4..2053b5a80b1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
@@ -2,8 +2,8 @@
   "1": {
     "id": 1,
     "status": "success",
-    "start": 3703000,
-    "end": 3703000,
+    "start": 7303000,
+    "end": 7303000,
     "wantedPlatform": "7.1",
     "wantedApplication": {
       "hash": "unknown"
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
index e97c76668d3..1e43c6e2953 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
@@ -1,8 +1,8 @@
 {
-  "devJobs": {},
-  "deployments": [],
   "lastVersions": {},
   "deploying": {},
+  "deployments": [],
   "jobs": {},
+  "devJobs": {},
   "deployment": []
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
index aaa9127bdfd..d46b396b8cd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
@@ -30,8 +30,8 @@
         {
           "id": 1,
           "status": "success",
-          "start": 3703000,
-          "end": 3703000,
+          "start": 7303000,
+          "end": 7303000,
           "wantedPlatform": "7.1",
           "wantedApplication": {
             "hash": "unknown"
@@ -58,8 +58,8 @@
         {
           "id": 1,
           "url": "https://some.url:43/root//run/1",
-          "start": 3703000,
-          "end": 3703000,
+          "start": 7303000,
+          "end": 7303000,
           "status": "success",
           "versions": {
             "targetPlatform": "7.1.0",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
index 1e1a4549006..948d46b1a6b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
@@ -37,7 +37,7 @@
   "deployments": [
     {
       "us-central-1": {
-        "at": 3603000,
+        "at": 7203000,
         "platform": "6.1",
         "application": {
           "hash": "1.0.3-commit1",
@@ -97,8 +97,8 @@
         {
           "id": 3,
           "status": "success",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -265,8 +265,8 @@
         {
           "id": 5,
           "status": "installationFailed",
-          "start": 3703000,
-          "end": 3703000,
+          "start": 7303000,
+          "end": 7303000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -313,8 +313,8 @@
         {
           "id": 4,
           "status": "installationFailed",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -361,8 +361,8 @@
         {
           "id": 3,
           "status": "success",
-          "start": 3603000,
-          "end": 3603000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -510,7 +510,7 @@
         {
           "id": 3,
           "status": "running",
-          "start": 3603000,
+          "start": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -774,7 +774,7 @@
           "id": 2,
           "status": "installationFailed",
           "start": 1000,
-          "end": 3602000,
+          "end": 7202000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.2-commit1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
index a7825681f4b..2ad35968732 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
@@ -94,8 +94,8 @@
   "3": {
     "id": 3,
     "status": "success",
-    "start": 3603000,
-    "end": 3603000,
+    "start": 7203000,
+    "end": 7203000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
@@ -146,8 +146,8 @@
   "4": {
     "id": 4,
     "status": "installationFailed",
-    "start": 3603000,
-    "end": 3603000,
+    "start": 7203000,
+    "end": 7203000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
@@ -194,8 +194,8 @@
   "5": {
     "id": 5,
     "status": "installationFailed",
-    "start": 3703000,
-    "end": 3703000,
+    "start": 7303000,
+    "end": 7303000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
index 8ade8795c86..ba51471d467 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
@@ -4,144 +4,149 @@
   "log": {
     "deployTester": [
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "No services requiring restart."
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment successful."
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "foo"
       }
     ],
     "installTester": [
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       }
     ],
     "deployInitialReal": [
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..."
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "No services requiring restart."
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment successful."
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "foo"
       }
     ],
     "installInitialReal": [
       {
-        "at": 3703000,
+        "at": 7303000,
+        "type": "info",
+        "message": "######## Details for all nodes ########"
+      },
+      {
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment expired before installation was successful."
       }
     ],
     "deactivateReal": [
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..."
       }
     ],
     "deactivateTester": [
       {
-        "at": 3703000,
+        "at": 7303000,
         "type": "info",
         "message": "Deactivating tester of tenant.application in staging.us-east-3 ..."
       }
     ]
   },
-  "lastId": 22,
+  "lastId": 23,
   "steps": {
     "deployTester": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "installTester": {
       "status": "unfinished",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "deployInitialReal": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "installInitialReal": {
       "status": "failed",
-      "startMillis": 3703000,
+      "startMillis": 7303000,
       "convergence": {
         "nodes": 1,
         "down": 0,
@@ -177,19 +182,19 @@
     },
     "copyVespaLogs": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "deactivateReal": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "deactivateTester": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     },
     "report": {
       "status": "succeeded",
-      "startMillis": 3703000
+      "startMillis": 7303000
     }
   }
 }