diff options
author | jonmv <venstad@gmail.com> | 2022-08-10 20:53:33 +0200 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2022-08-10 20:53:33 +0200 |
commit | 28e2649e349798456a489bc30d8458d7c2145431 (patch) | |
tree | d4b68db5f9d3a37b2933de42c0b603e5b7f5dd23 | |
parent | 33543be1fc9ea28cdc727b62155aa72ea0fc2dd3 (diff) |
Re-run inconclusive tests on ever 30min instead
2 files changed, 10 insertions, 6 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 38d7b6d3a2b..3133682fe04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -656,10 +656,13 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestReport(id); return Optional.of(testFailure); case INCONCLUSIVE: - long sleepMinutes = Math.max(15, Math.min(120, Duration.between(deployment.get().at(), controller.clock().instant()).toMinutes() / 20)); - logger.log("Tests were inconclusive, and will run again in " + sleepMinutes + " minutes."); controller.jobController().updateTestReport(id); - controller.jobController().locked(id, run -> run.sleepingUntil(controller.clock().instant().plusSeconds(60 * sleepMinutes))); + controller.jobController().locked(id, run -> { + Instant nextAttemptAt = run.start(); + while ( ! nextAttemptAt.isAfter(controller.clock().instant())) nextAttemptAt = nextAttemptAt.plusSeconds(1800); + logger.log("Tests were inconclusive, and will run again at " + nextAttemptAt + "."); + return run.sleepingUntil(nextAttemptAt); + }); return Optional.of(reset); case ERROR: logger.log(INFO, "Tester failed running its tests!"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index bf488198126..493c0945ecc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -358,7 +358,8 @@ public class InternalStepRunnerTest { // Test sleeps for a while. tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).stepStatuses().get(Step.deployTester)); - tester.clock().advance(Duration.ofSeconds(899)); + Instant nextAttemptAt = tester.clock().instant().plusSeconds(1800); + tester.clock().advance(Duration.ofSeconds(1799)); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).stepStatuses().get(Step.deployTester)); @@ -380,8 +381,8 @@ public class InternalStepRunnerTest { assertTestLogEntries(id, Step.endTests, new LogEntry(lastId1 + 1, Instant.ofEpochMilli(123), info, "Not enough data!"), - new LogEntry(lastId1 + 2, instant1, info, "Tests were inconclusive, and will run again in 15 minutes."), - new LogEntry(lastId1 + 15, instant1, info, "### Run will reset, and start over at " + instant1.plusSeconds(900).truncatedTo(SECONDS)), + new LogEntry(lastId1 + 2, instant1, info, "Tests were inconclusive, and will run again at " + nextAttemptAt + "."), + new LogEntry(lastId1 + 15, instant1, info, "### Run will reset, and start over at " + nextAttemptAt), new LogEntry(lastId1 + 16, instant1, info, ""), new LogEntry(lastId2 + 1, tester.clock().instant(), info, "Tests completed successfully.")); |