From 3e321f1c24ce80153b1730f82c840c31f20ad9a2 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 15 Mar 2022 12:42:07 +0100 Subject: Handle incocmpatible versions for dev/perf deployments --- .../hosted/controller/ApplicationController.java | 4 +- .../controller/deployment/DeploymentTrigger.java | 5 +-- .../controller/deployment/JobController.java | 48 ++++++++++++++++----- .../controller/maintenance/DeploymentUpgrader.java | 19 +++++++-- .../hosted/controller/maintenance/Upgrader.java | 2 - .../hosted/controller/versions/VersionStatus.java | 11 +++++ .../vespa/hosted/controller/ControllerTest.java | 49 ++++++++++++++++++++++ .../application/ApplicationApiCloudTest.java | 3 ++ .../application/responses/deployment-overview.json | 3 +- .../responses/dev-aws-us-east-2a-runs.json | 3 +- .../application/responses/dev-overview.json | 9 ++-- .../restapi/application/responses/jobs.json | 9 ++-- .../responses/overview-user-instance.json | 3 +- 13 files changed, 138 insertions(+), 30 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a4aac26b973..61fce68b5d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -875,11 +875,9 @@ public class ApplicationController { /** Returns the latest known version within the given major, which is not newer than the system version. */ public Optional lastCompatibleVersion(int targetMajorVersion) { VersionStatus versions = controller.readVersionStatus(); - Version systemVersion = controller.systemVersion(versions); - return versions.versions().stream() + return versions.deployableVersions().stream() .map(VespaVersion::versionNumber) .filter(version -> version.getMajor() == targetMajorVersion) - .filter(version -> ! version.isAfter(systemVersion)) .max(naturalOrder()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index b0e51c8fe9d..03dfa0134cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -142,9 +142,8 @@ public class DeploymentTrigger { return List.of(systemVersion); VespaVersion.Confidence target = policy == DeploymentSpec.UpgradePolicy.defaultPolicy ? VespaVersion.Confidence.normal : VespaVersion.Confidence.high; - return versions.versions().stream() - .filter(version -> ! version.versionNumber().isAfter(systemVersion) - && version.confidence().equalOrHigherThan(target)) + return versions.deployableVersions().stream() + .filter(version -> version.confidence().equalOrHigherThan(target)) .map(VespaVersion::versionNumber) .sorted(reverseOrder()) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index abd5ca8891a..6413d9e0def 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSortedMap; +import com.yahoo.collections.Iterables; import com.yahoo.component.Version; +import com.yahoo.component.VersionCompatibility; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; @@ -26,13 +28,17 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -49,6 +55,7 @@ import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.stream.Stream; +import static com.yahoo.collections.Iterables.reversed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -533,8 +540,10 @@ public class JobController { lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L); - ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(), - Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty()); + ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), + applicationPackage.compileVersion(), + Optional.empty(), Optional.empty(), + Optional.empty(), true, Optional.empty()); byte[] diff = lastRun.map(run -> run.versions().targetApplication()) .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage)) @@ -542,16 +551,10 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff); + Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun)); start(id, type, - new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion() - .flatMap(controller.applications()::lastCompatibleVersion) - .or(() -> lastRun.map(run -> run.versions().targetPlatform()) - .filter(controller.readVersionStatus()::isActive)) - .orElseGet(controller::readSystemVersion)), - version, - lastRun.map(run -> run.versions().targetPlatform()), - lastRun.map(run -> run.versions().targetApplication())), + new Versions(targetPlatform, version, lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetApplication())), false, dryRun ? JobProfile.developmentDryRun : JobProfile.development, Optional.empty()); @@ -562,6 +565,31 @@ public class JobController { }); } + private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional lastRun) { + Optional major = applicationPackage.deploymentSpec().majorVersion(); + if (major.isPresent()) + return controller.applications().lastCompatibleVersion(major.get()) + .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + + "but no version on this major was found")); + + // Prefer previous platform if possible. + VersionStatus versionStatus = controller.readVersionStatus(); + VersionCompatibility compatibility = controller.applications().versionCompatibility(); + Optional target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive); + if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get()))) + return target.get(); + + // Otherwise, use newest, compatible version. + for (VespaVersion platform : reversed(versionStatus.deployableVersions())) + if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber()))) + return platform.versionNumber(); + + throw new IllegalArgumentException("no suitable platform version found" + + applicationPackage.compileVersion() + .map(version -> " for package compiled against " + version) + .orElse("")); + } + /** Aborts a run and waits for it complete. */ private void abortAndWait(RunId id) { abort(id, "replaced by new deployment"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index bf3b9b90f66..57a4a042203 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -18,6 +19,8 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; +import static com.yahoo.collections.Iterables.reversed; + /** * Upgrades instances in manually deployed zones to the system version, at a convenient time. * @@ -33,22 +36,32 @@ public class DeploymentUpgrader extends ControllerMaintainer { protected double maintain() { AtomicInteger attempts = new AtomicInteger(); AtomicInteger failures = new AtomicInteger(); - Version systemVersion = controller().readSystemVersion(); + + Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version. + for (VespaVersion platform : controller().readVersionStatus().deployableVersions()) + if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.low)) + targetPlatform = platform.versionNumber(); + + if (targetPlatform == null) + return 0; for (Application application : controller().applications().readable()) for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) try { - attempts.incrementAndGet(); JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; Run last = controller().jobController().last(job).get(); - Versions target = new Versions(systemVersion, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); + Versions target = new Versions(targetPlatform, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication())); + if (last.versions().targetApplication().compileVersion() + .map(version -> controller().applications().versionCompatibility().refuse(version, target.targetPlatform())) + .orElse(false)) continue; if ( ! deployment.version().isBefore(target.targetPlatform())) continue; if ( ! isLikelyNightFor(job)) continue; log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); + attempts.incrementAndGet(); controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index f723d8b5134..545f38bf6df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -25,10 +25,8 @@ import java.util.Random; import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; -import static java.util.Comparator.reverseOrder; /** * Maintenance job which schedules applications for Vespa version upgrade diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 238ab0b09fa..3fa440e694e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -69,6 +69,17 @@ public class VersionStatus { * Calling this is free, but the returned status is slightly out of date. */ public List versions() { return versions; } + + /** Lists all currently active Vespa versions, from lowest to highest number, which are not newer than the system version. */ + public List deployableVersions() { + List deployable = new ArrayList<>(); + for (VespaVersion version : versions) { + deployable.add(version); + if (version.isSystemVersion()) + return deployable; + } + return List.of(); + } /** Returns the given version, or null if it is not present */ public VespaVersion version(Version version) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 63c0193ba7f..ca4e56fca3a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -18,6 +18,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -721,6 +722,54 @@ public class ControllerTest { .get(tester.clock().instant())); } + @Test + public void testDevDeploymentWithIncompatibleVersions() { + Version version1 = new Version("7"); + Version version2 = new Version("7.5"); + Version version3 = new Version("8"); + var context = tester.newDeploymentContext(); + tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class); + tester.controllerTester().upgradeSystem(version2); + ZoneId zone = ZoneId.from("dev", "us-east-1"); + + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version1).build()); + assertEquals(version2, context.deployment(zone).version()); + assertEquals(Optional.of(version1), context.deployment(zone).applicationVersion().compileVersion()); + + try { + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version1).majorVersion(8).build()); + fail("Should fail when specifying a major that does not yet exist"); + } + catch (IllegalArgumentException e) { + assertEquals("major 8 specified in deployment.xml, but no version on this major was found", e.getMessage()); + } + + try { + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build()); + fail("Should fail when compiled against a version which does not yet exist"); + } + catch (IllegalArgumentException e) { + assertEquals("no suitable platform version found for package compiled against 8", e.getMessage()); + } + + tester.controllerTester().upgradeSystem(version3); + try { + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version1).majorVersion(8).build()); + fail("Should fail when specifying a major which is incompatible with compile version"); + } + catch (IllegalArgumentException e) { + assertEquals("Will not start a job with incompatible platform version (8) and compile versions (7)", e.getMessage()); + } + + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build()); + assertEquals(version3, context.deployment(zone).version()); + assertEquals(Optional.of(version3), context.deployment(zone).applicationVersion().compileVersion()); + + context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build()); + assertEquals(version3, context.deployment(zone).version()); + assertEquals(Optional.of(version3), context.deployment(zone).applicationVersion().compileVersion()); + } + @Test public void testSuspension() { var context = tester.newDeploymentContext(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index fa1483fd90f..c6c2fd51de8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -311,6 +312,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var application = ApplicationName.from("unique"); var applicationPackage = new ApplicationPackageBuilder().withoutAthenzIdentity().build(); + new ControllerTester(tester).upgradeSystem(new Version("6.1")); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); tester.assertResponse( @@ -372,6 +374,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .globalServiceId("foo") .region("aws-us-east-1c") .build(); + new ControllerTester(tester).upgradeSystem(new Version("6.1")); tester.controller().jobController().deploy(ApplicationId.from("scoober", "albums", "default"), JobType.productionAwsUsEast1c, Optional.empty(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index c0a6829b026..782819ef6c6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -518,7 +518,8 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" } }, "steps": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index dce73ad56cd..313737146fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -9,7 +9,8 @@ "versions": { "targetPlatform": "7.1.0", "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" } }, "steps": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index 63fd0845d1b..e2d7423f161 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -6,11 +6,13 @@ { "versions": { "targetApplication": { - "build": 2 + "build": 2, + "compileVersion": "6.1.0" }, "targetPlatform": "6.1.0", "sourceApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" }, "sourcePlatform": "6.1.0" }, @@ -36,7 +38,8 @@ { "versions": { "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" }, "targetPlatform": "6.1.0" }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 736e1fe082c..e8c2660f999 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -12,11 +12,13 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" }, "sourcePlatform":"6.1.0", "sourceApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" } }, "steps": [ @@ -43,7 +45,8 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" } }, "steps": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index f8aba54356b..b0556e39630 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -6,7 +6,8 @@ { "versions": { "targetApplication": { - "build": 1 + "build": 1, + "compileVersion": "6.1.0" }, "targetPlatform": "7.1.0" }, -- cgit v1.2.3