From 017caeefc9c5b96dcef1ef2da8b2c09f20753f9f Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 11 Feb 2022 12:27:34 +0100 Subject: Add unit tests for all upgrade rollout setting, with revision catching upp --- .../deployment/DeploymentTriggerTest.java | 141 +++++++++++++++++++-- 1 file changed, 131 insertions(+), 10 deletions(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 0ac1691829a..aa29f58a909 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1263,28 +1263,149 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app3.instance().change()); } - // TODO test new revision allowed exactly when dependent instance is failing // TODO test multi-tier pipeline with various policies // TODO test multi-tier pipeline with upgrade after new version is the candidate @Test public void testRevisionJoinsUpgradeWithSeparateRollout() { - var app = tester.newDeploymentContext().submit().deploy(); - var version = new Version("7.1"); - tester.controllerTester().upgradeSystem(version); + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("separate") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); tester.clock().advance(Duration.ofMinutes(1)); - app.submit(); - assertEquals(Change.of(version).with(app.lastSubmission().get()), app.instance().change()); + // Revision starts rolling, but stays behind. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); - app.runJob(productionUsEast3).runJob(productionUsWest1); - tester.triggerJobs(); - assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); + // Upgrade got here first, so attempts to proceed alone, but the upgrade fails. + app.triggerJobs(); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + app.timeOutConvergence(productionUsEast3); + + // Revision is allowed to join. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version1), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + app.runJob(productionUsEast3); + + // Platform and revision now proceed together. + app.runJob(stagingTest); + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testRevisionJoinsUpgradeWithLeadingRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("leading") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); + + // Revision starts rolling, and catches up. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + + // Upgrade got here first, and has triggered, but is now obsolete. + app.triggerJobs(); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); - app.runJob(productionUsEast3).runJob(productionUsWest1); + // Once staging tests verify the joint upgrade, the job is replaced with that. + app.runJob(stagingTest); + app.triggerJobs(); + app.jobAborted(productionUsEast3).runJob(productionUsEast3); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + + // Platform and revision now proceed together. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + } + + @Test + public void testRevisionPassesUpgradeWithSimultaneousRollout() { + var appPackage = new ApplicationPackageBuilder().region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .upgradeRollout("simultaneous") + .build(); + var app = tester.newDeploymentContext().submit(appPackage).deploy(); + + // Platform rolls through first production zone. + var version0 = tester.controller().readSystemVersion(); + var version1 = new Version("7.1"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); + + // Revision starts rolling, and catches up. + var revision0 = app.lastSubmission(); + app.submit(appPackage); + var revision1 = app.lastSubmission(); + assertEquals(Change.of(version1).with(revision1.get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + + // Upgrade got here first, and has triggered, but is now obsolete. + app.triggerJobs(); + app.assertRunning(productionUsEast3); + assertEquals(new Versions(version1, revision0.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + assertEquals(RunStatus.running, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); + + // Once staging tests verify the joint upgrade, the job is replaced with that. + app.runJob(stagingTest); + app.triggerJobs(); + app.jobAborted(productionUsEast3).runJob(productionUsEast3); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsEast3).get().versions()); + + // Revision now proceeds alone. + app.runJob(systemTest).runJob(stagingTest); + app.triggerJobs(); + assertEquals(new Versions(version0, revision1.get(), Optional.of(version0), revision0), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); + + // Upgrade follows. + app.triggerJobs(); + assertEquals(new Versions(version1, revision1.get(), Optional.of(version0), revision1), + tester.jobs().last(app.instanceId(), productionUsWest1).get().versions()); + app.runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); } -- cgit v1.2.3