diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2023-10-02 09:15:38 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-10-02 09:15:38 +0200 |
commit | 2f718b83022b1158be18ef14e16c4628b3e606a1 (patch) | |
tree | c64186037d74922ad593d1531eecfea90fedea45 /controller-server | |
parent | 3b5ff95e40afc294ec6dc02d7e865b85f45cf3b2 (diff) | |
parent | 4265888cd9ccea8aada319ed91ce3a8b0151ddee (diff) |
Merge pull request #28740 from vespa-engine/jonmv/require-at-least-half-apps-regardless-of-pins
Require at least 50% of all canaries for normal, of defaults for high
Diffstat (limited to 'controller-server')
2 files changed, 85 insertions, 3 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index b03098bf18f..8a415a1e7e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -53,9 +53,15 @@ public record VespaVersion(Version version, if (productionOnThis.with(UpgradePolicy.canary).unpinned().size() < all.withProductionDeployment().with(UpgradePolicy.canary).unpinned().size()) return Confidence.low; - // 'high' if 90% of all unpinned default upgrade applications upgraded - if (productionOnThis.with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() >= - all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() * 0.9) + // 'low' unless at least half of all canary applications are upgraded + if (productionOnThis.with(UpgradePolicy.canary).size() < all.withProductionDeployment().with(UpgradePolicy.canary).size() * 0.5) + return Confidence.low; + + // 'high' if 90% of all unpinned default upgrade applications, and 50% of all of them, have upgraded + if ( productionOnThis.with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() >= + all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).unpinned().groupingBy(TenantAndApplicationId::from).size() * 0.9 + && productionOnThis.with(UpgradePolicy.defaultPolicy).groupingBy(TenantAndApplicationId::from).size() >= + all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).groupingBy(TenantAndApplicationId::from).size() * 0.5) return Confidence.high; return Confidence.normal; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index abce1c309ae..cb74165fdbc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -16,7 +16,9 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -701,6 +703,80 @@ public class VersionStatusTest { } } + @Test + void testPinnedAppsAreIgnoredForIncreasingConfidenceWhenLessThanHalfArePinned() { + DeploymentContext canaries[] = new DeploymentContext[3]; + DeploymentContext defaults[] = new DeploymentContext[3]; + DeploymentTester tester = new DeploymentTester().atMondayMorning(); + Version version1 = new Version("6.2"); + tester.controllerTester().upgradeSystem(version1); + + for (int i = 0; i < 3; i++) { + canaries[i] = tester.newDeploymentContext("t" + i, "a", "default"); + canaries[i].submit(canaryApplicationPackage).deploy(); + defaults[i] = tester.newDeploymentContext("t" + i, "b", "default"); + defaults[i].submit(defaultApplicationPackage).deploy(); + } + + assertEquals(Confidence.high, confidence(tester.controller(), version1)); + + // All apps are pinned to version1, and then version2 releases. Initial confidence is low. + for (int i = 0; i < 3; i++) { + tester.deploymentTrigger().forceChange(canaries[i].instanceId(), Change.empty().withPlatformPin()); + tester.deploymentTrigger().forceChange(defaults[i].instanceId(), Change.empty().withPlatformPin()); + } + Version version2 = new Version("6.3"); + tester.controllerTester().upgradeSystem(version2); + tester.upgrader().maintain(); + tester.triggerJobs(); + assertEquals(List.of(), tester.jobs().active()); + assertEquals(Confidence.low, confidence(tester.controller(), version2)); + + // One canary and one default are unpinned and upgrade. Confidence remains low, + // as more than half the apps are pinned, and less tan 100%/90% have upgraded. + tester.deploymentTrigger().cancelChange(canaries[0].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().forceChange(canaries[0].instanceId(), Change.of(version2)); + canaries[0].deployPlatform(version2); + tester.deploymentTrigger().cancelChange(defaults[0].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().forceChange(defaults[0].instanceId(), Change.of(version2)); + defaults[0].deployPlatform(version2); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.low, confidence(tester.controller(), version2)); + + // All apps are unpinned, and another canary and default upgrade. Confidence still remains low, + // as less than half of the unpinned apps have upgraded. + tester.deploymentTrigger().cancelChange(canaries[1].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().cancelChange(canaries[2].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().forceChange(canaries[1].instanceId(), Change.of(version2)); + tester.deploymentTrigger().cancelChange(defaults[1].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().cancelChange(defaults[2].instanceId(), ChangesToCancel.ALL); + tester.deploymentTrigger().forceChange(defaults[1].instanceId(), Change.of(version2)); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.low, confidence(tester.controller(), version2)); + + // The second canary upgrades while the last is unpinned. + canaries[1].deployPlatform(version2); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.low, confidence(tester.controller(), version2)); + + // When the last remaining canary is pinned, less than half are pinned, and all have upgraded, + // so confidence finally increases to normal. + tester.deploymentTrigger().forceChange(canaries[2].instanceId(), Change.empty().withPlatformPin()); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.normal, confidence(tester.controller(), version2)); + + // The second default upgrades while the last is unpinned. + defaults[1].deployPlatform(version2); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.normal, confidence(tester.controller(), version2)); + + // When the last remaining default is pinned, less than half are pinned, and more than 90% have upgraded, + // so confidence increases to high. + tester.deploymentTrigger().forceChange(defaults[2].instanceId(), Change.empty().withPlatformPin()); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.high, confidence(tester.controller(), version2)); + } + private void assertOnVersion(Version version, ApplicationId instance, DeploymentTester tester) { var vespaVersion = tester.controller().readVersionStatus().version(version); assertNotNull(vespaVersion, "Statistics for version " + version + " exist"); |