diff options
7 files changed, 84 insertions, 21 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java index 937df497133..e9fa196d44e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java @@ -18,6 +18,7 @@ public interface BuildService { /** * Returns whether the given job is currently running. */ + // TODO jvenstad: Change to enum State { idle, running, queued, disabled } boolean isRunning(BuildJob buildJob); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index b0f08770bd4..1f1402383cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -290,10 +290,10 @@ public class ApplicationController { if ( ! triggered.isPresent()) return unexpectedDeployment(applicationId, zone); platformVersion = preferOldestVersion - ? application.oldestDeployedPlatform().orElse(controller.systemVersion()) + ? triggered.get().sourcePlatform().orElse(triggered.get().platform()) : triggered.get().platform(); applicationVersion = preferOldestVersion - ? application.oldestDeployedApplication().orElse(triggered.get().application()) + ? triggered.get().sourceApplication().orElse(triggered.get().application()) : triggered.get().application(); applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage(application.id(), applicationVersion.id())); validateRun(application, zone, platformVersion, applicationVersion); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 33ae481e91c..510260434a2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -95,7 +95,7 @@ public class DeploymentTrigger { * Called each time a job completes (successfully or not) to record information used when deciding what to trigger. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.DEBUG, String.format("Got notified of %s for %s of %s (%d).", + log.log(LogLevel.INFO, String.format("Got notified of %s for %s of %s (%d).", report.jobError().map(JobError::toString).orElse("success"), report.jobType(), report.applicationId(), @@ -335,7 +335,8 @@ public class DeploymentTrigger { List<Job> jobs = new ArrayList<>(); for (Step step : steps.stream().filter(step -> step.deploysTo(test) || step.deploysTo(staging)).collect(toList())) { for (JobType jobType : step.zones().stream().map(order::toJob).collect(toList())) { - Optional<JobRun> completion = successOn(application, jobType, target); + Optional<JobRun> completion = successOn(application, jobType, target) + .filter(run -> jobType != stagingTest || sourcesMatchIfPresent(target, run)); if (completion.isPresent()) availableSince = completion.get().at(); else if (isTested(application, target, jobType)) @@ -355,10 +356,12 @@ public class DeploymentTrigger { return true; } + /** If the given state's sources are present and differ from its targets, returns whether they are equal to those of the given job run. */ private static boolean sourcesMatchIfPresent(State target, JobRun jobRun) { - return true; - //return ( (target.sourcePlatform.equals(jobRun.sourcePlatform()) || ! target.sourcePlatform.isPresent()) - //&& (target.sourceApplication.equals(jobRun.sourceApplication()) || ! target.sourceApplication.isPresent())); + return ( ! target.sourcePlatform.filter(version -> ! version.equals(target.targetPlatform)).isPresent() + || target.sourcePlatform.equals(jobRun.sourcePlatform())) + && ( ! target.sourceApplication.filter(version -> ! version.equals(target.targetApplication)).isPresent() + || target.sourceApplication.equals(jobRun.sourceApplication())); } private static boolean targetsMatch(State target, JobRun jobRun) { @@ -411,7 +414,7 @@ public class DeploymentTrigger { if (isRunning(application, job.jobType)) return false; - if (completedAt(job.change, application, job.jobType).isPresent()) + if (successOn(application, job.jobType, job.target).filter(run -> sourcesMatchIfPresent(job.target, run)).isPresent()) return false; // Job may have completed since it was computed. if ( ! job.jobType.isProduction()) @@ -508,11 +511,13 @@ public class DeploymentTrigger { @Override public String toString() { - return String.format("platform %s %s, application %s %s", + return String.format("platform %s%s, application %s%s", targetPlatform, - sourcePlatform.map(v -> "(from " + v + ")").orElse(""), + sourcePlatform.filter(version -> ! version.equals(targetPlatform)) + .map(v -> " (from " + v + ")").orElse(""), targetApplication.id(), - sourceApplication.map(v -> "(from " + v.id() + ")").orElse("")); + sourceApplication.filter(version -> ! version.equals(targetApplication)) + .map(v -> " (from " + v.id() + ")").orElse("")); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index a368640c13c..b19e2f637c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -439,7 +439,6 @@ public class DeploymentTriggerTest { assertEquals(v2, app.get().deployments().get(productionUsCentral1.zone(main).get()).version()); assertEquals((Long) 42L, app.get().deployments().get(productionUsCentral1.zone(main).get()).applicationVersion().buildNumber().get()); - // TODO jvenstad: Fails here now, because job isn't triggered any more, as deploy target is not verified. assertNotEquals(triggered, app.get().deploymentJobs().jobStatus().get(productionUsCentral1).lastTriggered().get().at()); // Change has a higher application version than what is deployed -- deployment should trigger. @@ -510,4 +509,45 @@ public class DeploymentTriggerTest { assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionUsEast3).lastSuccess().get().application().buildNumber().get().longValue()); } + @Test + public void eachDifferentUpgradeCombinationIsTested() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + Supplier<Application> app = () -> tester.application(application.id()); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-central-1") + .parallel("eu-west-1", "us-east-3") + .build(); + // Application version 42 and platform version 6.1. + tester.deployCompletely(application, applicationPackage); + + // Application partially upgrades, then a new version is released. + Version v1 = new Version("6.1"); + Version v2 = new Version("6.2"); + tester.upgradeSystem(v2); + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, empty(), true, productionUsCentral1); + tester.deployAndNotify(application, empty(), true, productionEuWest1); + tester.deployAndNotify(application, empty(), false, productionUsEast3); + assertEquals(v2, app.get().deployments().get(ZoneId.from("prod", "us-central-1")).version()); + assertEquals(v2, app.get().deployments().get(ZoneId.from("prod", "eu-west-1")).version()); + assertEquals(v1, app.get().deployments().get(ZoneId.from("prod", "us-east-3")).version()); + + Version v3 = new Version("6.3"); + tester.upgradeSystem(v3); + tester.deployAndNotify(application, empty(), false, productionUsEast3); + + // See that sources for staging are: first v2, then v1. + tester.deployAndNotify(application, empty(), true, systemTest); + tester.deployAndNotify(application, empty(), true, stagingTest); + assertEquals(v2, app.get().deploymentJobs().jobStatus().get(stagingTest).lastSuccess().get().sourcePlatform().get()); + tester.deployAndNotify(application, empty(), true, productionUsCentral1); + assertEquals(v1, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().sourcePlatform().get()); + tester.deployAndNotify(application, empty(), true, stagingTest); + tester.deployAndNotify(application, empty(), true, productionEuWest1); + tester.deployAndNotify(application, empty(), true, productionUsEast3); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 5b1c99ec727..9073c47fec1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -20,6 +20,7 @@ import java.time.Duration; import java.time.Instant; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; @@ -717,7 +718,7 @@ public class UpgraderTest { // Another hour pass, time is 20:00 and application upgrades tester.clock().advance(Duration.ofHours(1)); readyJobsTrigger.maintain(); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1); + tester.deployAndNotify(app, applicationPackage, true, productionUsCentral1); tester.deployAndNotify(app, applicationPackage, true, productionUsEast3); assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); } @@ -760,7 +761,7 @@ public class UpgraderTest { tester.deployAndNotify(app, applicationPackage, true, stagingTest); tester.deployAndNotify(app, applicationPackage, true, productionUsWest1); clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1); + tester.deployAndNotify(app, applicationPackage, true, productionUsCentral1); assertTrue(tester.buildService().jobs().isEmpty()); // Next job not triggered due to being in the block window // A day passes and we get a new version @@ -776,12 +777,12 @@ public class UpgraderTest { tester.clock().advance(Duration.ofHours(17)); // Monday, 10:00 tester.upgrader().maintain(); tester.readyJobTrigger().maintain(); - // We proceed with the new version in the expected order, not starting with the previously blocked version: - // Test jobs are run with the new version, but not production as we are in the block window tester.deployAndNotify(app, applicationPackage, true, systemTest); tester.deployAndNotify(app, applicationPackage, true, stagingTest); tester.deployAndNotify(app, applicationPackage, true, productionUsWest1); - tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1); + tester.deployAndNotify(app, applicationPackage, true, productionUsCentral1); + // us-east-3 has an older version than the other zones, and needs a new staging test run. + tester.deployAndNotify(app, applicationPackage, true, stagingTest); tester.deployAndNotify(app, applicationPackage, true, productionUsEast3); assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e99b1e9e425..22d487628e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -515,9 +515,25 @@ public class ApplicationApiTest extends ControllerContainerTest { .projectId(projectId) .submit(); + // staging-test again for us-east-3 + String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default", + id.tenant().value(), id.application().value()); + tester.assertResponse(request(stagingPath, POST) + .data(deployData) + .screwdriverIdentity(SCREWDRIVER_ID), + new File("deploy-result.json")); + tester.assertResponse(request(stagingPath, DELETE) + .screwdriverIdentity(SCREWDRIVER_ID), + "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); + controllerTester.jobCompletion(DeploymentJobs.JobType.stagingTest) + .application(id) + .projectId(projectId) + .submit(); + // us-east-3 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) - .data(deployData).screwdriverIdentity(SCREWDRIVER_ID), + .data(deployData) + .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); controllerTester.jobCompletion(DeploymentJobs.JobType.productionUsEast3) .application(id) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index 1a88a7691f1..89c6f3a8615 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-west-1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" }, "lastCompleted": { @@ -108,7 +108,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-west-1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" }, "lastSuccess": { @@ -122,7 +122,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-west-1", + "reason": "Testing deployment for production-us-east-3", "at": "(ignore)" } }, |