diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2020-03-09 13:33:26 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2020-03-09 13:33:26 +0100 |
commit | 999ae31a782aac8529b7abfafe119756676f8e85 (patch) | |
tree | a804d3f41fe94da28f4712092fd3fc2f6bc1597d /controller-server | |
parent | bbc74ec90bcba32b2de15b3cb5f17b0ed221ca33 (diff) |
Allow no progress for 2 hours while redistributing docs
Diffstat (limited to 'controller-server')
11 files changed, 87 insertions, 81 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index af36708ccd1..3d04c29ac51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -121,7 +121,8 @@ public class InternalStepRunner implements StepRunner { static final Duration endpointTimeout = Duration.ofMinutes(15); static final Duration endpointCertificateTimeout = Duration.ofMinutes(15); static final Duration testerTimeout = Duration.ofMinutes(30); - static final Duration installationTimeout = Duration.ofMinutes(60); + static final Duration nodesDownTimeout = Duration.ofMinutes(60); + static final Duration noNodesDownTimeout = Duration.ofMinutes(120); static final Duration certificateTimeout = Duration.ofMinutes(300); private final Controller controller; @@ -358,17 +359,17 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(installationTimeout)); + NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(nodesDownTimeout)); if ( ! suspendedTooLong.isEmpty()) { - failureReason = "Some nodes have been suspended for more than " + installationTimeout.toMinutes() + " minutes:\n" + + failureReason = "Some nodes have been suspended for more than " + nodesDownTimeout.toMinutes() + " minutes:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } if (run.noNodesDownSince() - .map(since -> since.isBefore(controller.clock().instant().minus(installationTimeout))) + .map(since -> since.isBefore(controller.clock().instant().minus(noNodesDownTimeout))) .orElse(false)) { if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0) - failureReason ="No nodes allowed to suspend to progress installation for " + installationTimeout.toMinutes() + " minutes."; + failureReason = "No nodes allowed to suspend to progress installation for " + noNodesDownTimeout.toMinutes() + " minutes."; else failureReason = "Nodes not able to start with new application package."; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 36ab6a9aa60..4550b29d2fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -364,7 +364,7 @@ public class DeploymentContext { triggerJobs(); RunId id = currentRun(job).id(); doDeploy(job); - tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); + tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); @@ -378,7 +378,7 @@ public class DeploymentContext { RunId id = currentRun(job).id(); doDeploy(job); doUpgrade(job); - tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); + tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 44fd677a4d3..a3d1b9adb35 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -159,7 +159,7 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); - tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.noNodesDownTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(installationFailed, tester.jobs().run(id).get().status()); } @@ -211,7 +211,7 @@ public class InternalStepRunnerTest { Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), app.instanceId()).iterator().next(); - tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.noNodesDownTimeout.minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()), new Node.Builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) @@ -226,7 +226,7 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); - tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(3))); + tester.clock().advance(InternalStepRunner.nodesDownTimeout.minus(Duration.ofSeconds(3))); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 6b71968822f..17d234b0c0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -82,7 +82,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); tester.runner().run(); - tester.clock().advance(Duration.ofHours(1).plusSeconds(1)); + tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index 65db44cb4e2..868809978de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -128,8 +128,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/system-test/run/3", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -320,9 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", - "readyAt": 4353000, - "delayedUntil": 4353000, - "coolingDownUntil": 4353000, + "readyAt": 7953000, + "delayedUntil": 7953000, + "coolingDownUntil": 7953000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -351,8 +351,8 @@ { "id": 5, "url": "https://some.url:43/instance/default/job/staging-test/run/5", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -432,8 +432,8 @@ { "id": 4, "url": "https://some.url:43/instance/default/job/staging-test/run/4", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -513,8 +513,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/staging-test/run/3", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -756,7 +756,7 @@ ], "declared": true, "instance": "default", - "readyAt": 3603000, + "readyAt": 7203000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -773,7 +773,7 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", - "start": 3603000, + "start": 7203000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -1050,7 +1050,7 @@ "id": 2, "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 3602000, + "end": 7202000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index f7185d373c4..2053b5a80b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index e97c76668d3..1e43c6e2953 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -1,8 +1,8 @@ { - "devJobs": {}, - "deployments": [], "lastVersions": {}, "deploying": {}, + "deployments": [], "jobs": {}, + "devJobs": {}, "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index aaa9127bdfd..d46b396b8cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -30,8 +30,8 @@ { "id": 1, "status": "success", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" @@ -58,8 +58,8 @@ { "id": 1, "url": "https://some.url:43/root//run/1", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "status": "success", "versions": { "targetPlatform": "7.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index 1e1a4549006..948d46b1a6b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -37,7 +37,7 @@ "deployments": [ { "us-central-1": { - "at": 3603000, + "at": 7203000, "platform": "6.1", "application": { "hash": "1.0.3-commit1", @@ -97,8 +97,8 @@ { "id": 3, "status": "success", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -265,8 +265,8 @@ { "id": 5, "status": "installationFailed", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -313,8 +313,8 @@ { "id": 4, "status": "installationFailed", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -361,8 +361,8 @@ { "id": 3, "status": "success", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -510,7 +510,7 @@ { "id": 3, "status": "running", - "start": 3603000, + "start": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -774,7 +774,7 @@ "id": 2, "status": "installationFailed", "start": 1000, - "end": 3602000, + "end": 7202000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.2-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index a7825681f4b..2ad35968732 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 3603000, - "end": 3603000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 3703000, - "end": 3703000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 8ade8795c86..ba51471d467 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,144 +4,149 @@ "log": { "deployTester": [ { - "at": 3703000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 3703000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 3703000, + "at": 7303000, + "type": "info", + "message": "######## Details for all nodes ########" + }, + { + "at": 7303000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 3703000, + "at": 7303000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } ] }, - "lastId": 22, + "lastId": 23, "steps": { "deployTester": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 }, "installTester": { "status": "unfinished", - "startMillis": 3703000 + "startMillis": 7303000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 }, "installInitialReal": { "status": "failed", - "startMillis": 3703000, + "startMillis": 7303000, "convergence": { "nodes": 1, "down": 0, @@ -177,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 }, "report": { "status": "succeeded", - "startMillis": 3703000 + "startMillis": 7303000 } } } |