diff options
8 files changed, 149 insertions, 55 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 4bbc65eee9f..1b8f9f25a80 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -35,37 +35,71 @@ public enum JobType { productionUsEast3 ("production-us-east-3", Map.of(main, ZoneId.from("prod" , "us-east-3"))), + testUsEast3 ("test-us-east-3", + Map.of(main, ZoneId.from("prod" , "us-east-3")), true), + productionUsWest1 ("production-us-west-1", Map.of(main, ZoneId.from("prod" , "us-west-1"))), + testUsWest1 ("test-us-west-1", + Map.of(main, ZoneId.from("prod" , "us-west-1")), true), + productionUsCentral1 ("production-us-central-1", Map.of(main, ZoneId.from("prod" , "us-central-1"))), + testUsCentral1 ("test-us-central-1", + Map.of(main, ZoneId.from("prod" , "us-central-1")), true), + productionApNortheast1 ("production-ap-northeast-1", Map.of(main, ZoneId.from("prod" , "ap-northeast-1"))), + testApNortheast1 ("test-ap-northeast-1", + Map.of(main, ZoneId.from("prod" , "ap-northeast-1")), true), + productionApNortheast2 ("production-ap-northeast-2", Map.of(main, ZoneId.from("prod" , "ap-northeast-2"))), + testApNortheast2 ("test-ap-northeast-2", + Map.of(main, ZoneId.from("prod" , "ap-northeast-2")), true), + productionApSoutheast1 ("production-ap-southeast-1", Map.of(main, ZoneId.from("prod" , "ap-southeast-1"))), + testApSoutheast1 ("test-ap-southeast-1", + Map.of(main, ZoneId.from("prod" , "ap-southeast-1")), true), + productionEuWest1 ("production-eu-west-1", Map.of(main, ZoneId.from("prod" , "eu-west-1"))), + testEuWest1 ("test-eu-west-1", + Map.of(main, ZoneId.from("prod" , "eu-west-1")), true), + productionAwsUsEast1a ("production-aws-us-east-1a", Map.of(main, ZoneId.from("prod" , "aws-us-east-1a"))), + testAwsUsEast1a ("test-aws-us-east-1a", + Map.of(main, ZoneId.from("prod" , "aws-us-east-1a")), true), + productionAwsUsEast1c ("production-aws-us-east-1c", Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"), Public, ZoneId.from("prod", "aws-us-east-1c"))), + testAwsUsEast1c ("test-aws-us-east-1c", + Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"), + Public, ZoneId.from("prod", "aws-us-east-1c")), true), + productionAwsUsWest2a ("production-aws-us-west-2a", Map.of(main, ZoneId.from("prod" , "aws-us-west-2a"))), + testAwsUsWest2a ("test-aws-us-west-2a", + Map.of(main, ZoneId.from("prod" , "aws-us-west-2a")), true), + productionAwsUsEast1b ("production-aws-us-east-1b", Map.of(main, ZoneId.from("prod" , "aws-us-east-1b"))), + testAwsUsEast1b ("test-aws-us-east-1b", + Map.of(main, ZoneId.from("prod" , "aws-us-east-1b")), true), + devUsEast1 ("dev-us-east-1", Map.of(main, ZoneId.from("dev" , "us-east-1"))), @@ -75,9 +109,15 @@ public enum JobType { productionCdAwsUsEast1a("production-cd-aws-us-east-1a", Map.of(cd , ZoneId.from("prod" , "cd-aws-us-east-1a"))), + testCdAwsUsEast1a ("test-cd-aws-us-east-1a", + Map.of(cd , ZoneId.from("prod" , "cd-aws-us-east-1a")), true), + productionCdUsCentral1 ("production-cd-us-central-1", Map.of(cd , ZoneId.from("prod" , "cd-us-central-1"))), + testCdUsCentral1 ("test-cd-us-central-1", + Map.of(cd , ZoneId.from("prod" , "cd-us-central-1")), true), + // TODO: Cannot remove production-cd-us-central-2 until we know there are no serialized data in controller referencing it productionCdUsCentral2 ("production-cd-us-central-2", Map.of(cd , ZoneId.from("prod" , "cd-us-central-2"))), @@ -85,6 +125,9 @@ public enum JobType { productionCdUsWest1 ("production-cd-us-west-1", Map.of(cd , ZoneId.from("prod" , "cd-us-west-1"))), + testCdUsWest1 ("test-cd-us-west-1", + Map.of(cd , ZoneId.from("prod" , "cd-us-west-1")), true), + devCdUsCentral1 ("dev-cd-us-central-1", Map.of(cd , ZoneId.from("dev" , "cd-us-central-1"))), @@ -164,7 +207,7 @@ public enum JobType { } /** Returns the production test job type for the given environment and region or null if none */ - public static Optional<JobType> from(SystemName system, RegionName region) { + public static Optional<JobType> testFrom(SystemName system, RegionName region) { return from(system, ZoneId.from(Environment.prod, region), true); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 9594acc2931..45850669210 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -31,6 +31,8 @@ import java.util.stream.Stream; import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.collectingAndThen; @@ -132,39 +134,39 @@ public class DeploymentStatus { Map<JobId, List<Versions>> testJobs = new HashMap<>(); jobs.forEach((job, versions) -> { if ( ! job.type().isTest()) { - declaredTest(job.application(), JobType.systemTest).ifPresent(test -> { + declaredTest(job.application(), systemTest).ifPresent(test -> { testJobs.merge(test, versions.stream() .filter(version -> ! steps.get(test).isRunning(version)) .filter(version -> allJobs.successOn(version).get(test).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) .collect(toUnmodifiableList()), - DeploymentStatus::concat); + DeploymentStatus::union); }); - declaredTest(job.application(), JobType.stagingTest).ifPresent(test -> { + declaredTest(job.application(), stagingTest).ifPresent(test -> { testJobs.merge(test, versions.stream() .filter(version -> ! steps.get(test).isRunning(version)) .filter(version -> allJobs.successOn(version).get(test).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) .collect(toUnmodifiableList()), - DeploymentStatus::concat); + DeploymentStatus::union); }); } }); jobs.forEach((job, versions) -> { - if ( ! job.type().isTest() && ! testedOn(versions, JobType.systemTest, testJobs)) - testJobs.merge(new JobId(job.application(), JobType.systemTest), + if ( ! job.type().isTest() && ! testedOn(versions, systemTest, testJobs)) + testJobs.merge(new JobId(job.application(), systemTest), versions.stream() - .filter(version -> steps.keySet().stream().noneMatch(id -> id.type() == JobType.systemTest && steps.get(id).isRunning(version))) - .filter(version -> allJobs.successOn(version).type(JobType.systemTest).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) + .filter(version -> steps.keySet().stream().noneMatch(id -> id.type() == systemTest && steps.get(id).isRunning(version))) + .filter(version -> allJobs.successOn(version).type(systemTest).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) .collect(toUnmodifiableList()), - DeploymentStatus::concat); - if ( ! job.type().isTest() && ! testedOn(versions, JobType.stagingTest, testJobs)) - testJobs.merge(new JobId(job.application(), JobType.stagingTest), + DeploymentStatus::union); + if ( ! job.type().isTest() && ! testedOn(versions, stagingTest, testJobs)) + testJobs.merge(new JobId(job.application(), stagingTest), versions.stream() - .filter(version -> steps.keySet().stream().noneMatch(id -> id.type() == JobType.stagingTest && steps.get(id).isRunning(version))) - .filter(version -> allJobs.successOn(version).type(JobType.stagingTest).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) + .filter(version -> steps.keySet().stream().noneMatch(id -> id.type() == stagingTest && steps.get(id).isRunning(version))) + .filter(version -> allJobs.successOn(version).type(stagingTest).isEmpty() && allJobs.triggeredOn(version).get(job).isEmpty()) .collect(toUnmodifiableList()), - DeploymentStatus::concat); + DeploymentStatus::union); }); jobs.putAll(testJobs); } @@ -181,8 +183,8 @@ public class DeploymentStatus { .anyMatch(job -> job.type() == testJob && testJobs.get(job).containsAll(versions)); } - private static <T> List<T> concat(List<T> first, List<T> second) { - return Stream.concat(first.stream(), second.stream()).collect(toUnmodifiableList()); + private static <T> List<T> union(List<T> first, List<T> second) { + return Stream.concat(first.stream(), second.stream()).distinct().collect(toUnmodifiableList()); } private Optional<Deployment> deploymentFor(JobId job) { @@ -220,7 +222,7 @@ public class DeploymentStatus { previous.add(stepStatus); } else if (step.isTest()) { - jobType = JobType.from(system, ((DeclaredTest) step).region()) + jobType = JobType.testFrom(system, ((DeclaredTest) step).region()) .orElseThrow(() -> new IllegalStateException("No job is known for " + step + " in " + system)); JobType preType = JobType.from(system, prod, ((DeclaredTest) step).region()) .orElseThrow(() -> new IllegalStateException("No job is known for " + step + " in " + system)); @@ -258,13 +260,13 @@ public class DeploymentStatus { private boolean isTested(JobId job, Versions versions) { return allJobs.triggeredOn(versions).get(job).isPresent() - || ! declaredTest(job.application(), JobType.systemTest).map(__ -> allJobs.instance(job.application().instance())) + || ! declaredTest(job.application(), systemTest).map(__ -> allJobs.instance(job.application().instance())) .orElse(allJobs) - .type(JobType.systemTest) + .type(systemTest) .successOn(versions).isEmpty() - && ! declaredTest(job.application(), JobType.stagingTest).map(__ -> allJobs.instance(job.application().instance())) + && ! declaredTest(job.application(), stagingTest).map(__ -> allJobs.instance(job.application().instance())) .orElse(allJobs) - .type(JobType.stagingTest) + .type(stagingTest) .successOn(versions).isEmpty(); } @@ -426,11 +428,10 @@ public class DeploymentStatus { return new JobStepStatus(step, dependencies, job, status) { @Override public Optional<Instant> completedAt(Change change, Versions versions) { - Versions toVerify = Versions.from(change, status.application.require(instance).deployments().get(ZoneId.from(prod, step.region()))); return job.lastSuccess() - .filter(run -> toVerify.targetsMatch(run.versions())) + .filter(run -> versions.targetsMatch(run.versions())) .filter(run -> status.instanceJobs(instance).get(jobType).lastCompleted() - .map(last -> last.end().get().isBefore(run.start())).orElse(false)) + .map(last -> ! last.end().get().isAfter(run.start())).orElse(false)) .map(run -> run.end().get()); } }; @@ -509,5 +510,25 @@ public class DeploymentStatus { * */ + public static List<JobId> jobsFor(Application application, SystemName system) { + if (DeploymentSpec.empty.equals(application.deploymentSpec())) + return List.of(); + + return application.deploymentSpec().instances().stream() + .flatMap(spec -> Stream.concat(Stream.of(systemTest, stagingTest), + flatten(spec).filter(step -> step.concerns(prod)) + .map(step -> { + if (step instanceof DeclaredZone) + return JobType.from(system, prod, ((DeclaredZone) step).region().get()); + return JobType.testFrom(system, ((DeclaredTest) step).region()); + }) + .flatMap(Optional::stream)) + .map(type -> new JobId(application.id().instance(spec.name()), type))) + .collect(toUnmodifiableList()); + } + + private static Stream<DeploymentSpec.Step> flatten(DeploymentSpec.Step step) { + return step instanceof DeploymentSpec.Steps ? step.steps().stream().flatMap(DeploymentStatus::flatten) : Stream.of(step); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index b77f3a242c3..7bcdfd48265 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -265,7 +265,8 @@ public class DeploymentTrigger { status.jobsToRun().forEach((job, versionsList) -> { for (Versions versions : versionsList) status.stepStatus().get(job).readyAt(status.application().change(), versions) - .filter(readyAt -> ! clock.instant().isBefore(readyAt)).ifPresent(readyAt -> { + .filter(readyAt -> ! clock.instant().isBefore(readyAt)) + .ifPresent(readyAt -> { if ( ! ( isSuspendedInAnotherZone(status.application().require(job.application().instance()), job.type().zone(controller.system())) && job.type().environment() == Environment.prod)) @@ -461,22 +462,11 @@ public class DeploymentTrigger { private Change remainingChange(Application application) { Change change = application.change(); - if (application.deploymentSpec().instances().stream() - .allMatch(spec -> { - DeploymentSteps steps = new DeploymentSteps(spec, controller::system); - return (steps.productionJobs().isEmpty() ? steps.testJobs() : steps.productionJobs()) - .stream().allMatch(job -> isComplete(application.change().withoutApplication(), application.change(), application.require(spec.name()), job, jobs.jobStatus(new JobId(application.id().instance(spec.name()), job)))); - })) + DeploymentStatus status = jobs.deploymentStatus(application); + if (status.jobsToRun(status.application().change().withoutApplication()).isEmpty()) change = change.withoutPlatform(); - - if (application.deploymentSpec().instances().stream() - .allMatch(spec -> { - DeploymentSteps steps = new DeploymentSteps(spec, controller::system); - return (steps.productionJobs().isEmpty() ? steps.testJobs() : steps.productionJobs()) - .stream().allMatch(job -> isComplete(application.change().withoutPlatform(), application.change(), application.require(spec.name()), job, jobs.jobStatus(new JobId(application.id().instance(spec.name()), job)))); - })) + if (status.jobsToRun(status.application().change().withoutPlatform()).isEmpty()) change = change.withoutApplication(); - return change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 34af34c426f..81b33557044 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -289,15 +289,12 @@ public class JobController { return new JobStatus(id, runs(id)); } - /** Returns the job status of all declared jobs for the given instance id, indexed by job type. */ + /** Returns the deployment status of the given application. */ public DeploymentStatus deploymentStatus(Application application) { return new DeploymentStatus(application, - application.deploymentSpec().instances().stream() - .flatMap(spec -> new DeploymentSteps(spec, controller::system) - .jobs().stream() - .map(type -> jobStatus(new JobId(application.id().instance(spec.name()), type)))) - .collect(toUnmodifiableMap(status -> status.id(), - status -> status)), + DeploymentStatus.jobsFor(application, controller.system()).stream() + .collect(toUnmodifiableMap(job -> job, + job -> jobStatus(job))), controller.system(), controller.systemVersion()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 685fee92e7e..7bccbc0ebc9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -49,6 +49,13 @@ public enum JobProfile { EnumSet.of(deactivateTester, report)), + productionTest(EnumSet.of(deployTester, + installTester, + startTests, + endTests), + EnumSet.of(deactivateTester, + report)), + development(EnumSet.of(deployReal, installReal), EnumSet.of(copyVespaLogs)); @@ -67,7 +74,7 @@ public enum JobProfile { switch (type.environment()) { case test: return systemTest; case staging: return stagingTest; - case prod: return production; + case prod: return type.isTest() ? productionTest : production; case perf: case dev: return development; default: throw new AssertionError("Unexpected environment '" + type.environment() + "'!"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index c352fc5550f..7684aa9e887 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -99,6 +99,13 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder test(String regionName) { + environmentBody.append(" <test>"); + environmentBody.append(regionName); + environmentBody.append("</test>\n"); + return this; + } + public ApplicationPackageBuilder parallel(String... regionName) { environmentBody.append(" <parallel>\n"); Arrays.stream(regionName).forEach(this::region); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 65ebc0e67a0..ee65ffee62f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -302,10 +302,12 @@ public class DeploymentContext { var job = jobId(type); triggerJobs(); doDeploy(job); - doUpgrade(job); - doConverge(job); - if (job.type().environment().isManuallyDeployed()) - return this; + if ( ! job.type().isTest()) { + doUpgrade(job); + doConverge(job); + if (job.type().environment().isManuallyDeployed()) + return this; + } doInstallTester(job); doTests(job); doTeardown(job); @@ -514,7 +516,8 @@ public class DeploymentContext { ZoneId zone = zone(job); // All installation is complete and endpoints are ready, so tests may begin. - assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); + if ( ! job.type().isTest()) + assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installTester)); assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.startTests)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 88e0a8b82b3..2db50b37326 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -30,6 +30,10 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testEuWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsEast3; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static java.time.temporal.ChronoUnit.MILLIS; @@ -811,8 +815,30 @@ public class DeploymentTriggerTest { otherInstance.runJob(productionUsEast3); assertEquals(2, app.application().instances().size()); assertEquals(2, app.application().productionDeployments().values().stream() - .mapToInt(Collection::size) - .sum()); + .mapToInt(Collection::size) + .sum()); + } + + @Test + public void testDeclaredProductionTests() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region("us-east-3") + .delay(Duration.ofMinutes(1)) + .test("us-east-3") + .region("us-west-1") + .region("us-central-1") + .test("us-central-1") + .test("us-west-1") + .build(); + var app = tester.newDeploymentContext().submit(applicationPackage); + + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); + app.assertNotRunning(productionUsWest1); + + tester.clock().advance(Duration.ofMinutes(1)); + app.runJob(testUsEast3) + .runJob(productionUsWest1).runJob(productionUsCentral1) + .runJob(testUsCentral1).runJob(testUsWest1); } } |