aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjonmv <venstad@gmail.com>2022-08-10 20:53:33 +0200
committerjonmv <venstad@gmail.com>2022-08-10 20:53:33 +0200
commit28e2649e349798456a489bc30d8458d7c2145431 (patch)
treed4b68db5f9d3a37b2933de42c0b603e5b7f5dd23
parent33543be1fc9ea28cdc727b62155aa72ea0fc2dd3 (diff)
Re-run inconclusive tests on ever 30min instead
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java7
2 files changed, 10 insertions, 6 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 38d7b6d3a2b..3133682fe04 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -656,10 +656,13 @@ public class InternalStepRunner implements StepRunner {
controller.jobController().updateTestReport(id);
return Optional.of(testFailure);
case INCONCLUSIVE:
- long sleepMinutes = Math.max(15, Math.min(120, Duration.between(deployment.get().at(), controller.clock().instant()).toMinutes() / 20));
- logger.log("Tests were inconclusive, and will run again in " + sleepMinutes + " minutes.");
controller.jobController().updateTestReport(id);
- controller.jobController().locked(id, run -> run.sleepingUntil(controller.clock().instant().plusSeconds(60 * sleepMinutes)));
+ controller.jobController().locked(id, run -> {
+ Instant nextAttemptAt = run.start();
+ while ( ! nextAttemptAt.isAfter(controller.clock().instant())) nextAttemptAt = nextAttemptAt.plusSeconds(1800);
+ logger.log("Tests were inconclusive, and will run again at " + nextAttemptAt + ".");
+ return run.sleepingUntil(nextAttemptAt);
+ });
return Optional.of(reset);
case ERROR:
logger.log(INFO, "Tester failed running its tests!");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index bf488198126..493c0945ecc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -358,7 +358,8 @@ public class InternalStepRunnerTest {
// Test sleeps for a while.
tester.runner().run();
assertEquals(unfinished, tester.jobs().run(id).stepStatuses().get(Step.deployTester));
- tester.clock().advance(Duration.ofSeconds(899));
+ Instant nextAttemptAt = tester.clock().instant().plusSeconds(1800);
+ tester.clock().advance(Duration.ofSeconds(1799));
tester.runner().run();
assertEquals(unfinished, tester.jobs().run(id).stepStatuses().get(Step.deployTester));
@@ -380,8 +381,8 @@ public class InternalStepRunnerTest {
assertTestLogEntries(id, Step.endTests,
new LogEntry(lastId1 + 1, Instant.ofEpochMilli(123), info, "Not enough data!"),
- new LogEntry(lastId1 + 2, instant1, info, "Tests were inconclusive, and will run again in 15 minutes."),
- new LogEntry(lastId1 + 15, instant1, info, "### Run will reset, and start over at " + instant1.plusSeconds(900).truncatedTo(SECONDS)),
+ new LogEntry(lastId1 + 2, instant1, info, "Tests were inconclusive, and will run again at " + nextAttemptAt + "."),
+ new LogEntry(lastId1 + 15, instant1, info, "### Run will reset, and start over at " + nextAttemptAt),
new LogEntry(lastId1 + 16, instant1, info, ""),
new LogEntry(lastId2 + 1, tester.clock().instant(), info, "Tests completed successfully."));