diff options
author | Harald Musum <musum@oath.com> | 2017-11-08 09:21:29 +0100 |
---|---|---|
committer | Harald Musum <musum@oath.com> | 2017-11-08 09:21:29 +0100 |
commit | c90fa829e3593062ea397b6a8527178f7616d613 (patch) | |
tree | b73df0983eddb9b5930b7fcc548af3e644e7c146 | |
parent | 8fb495a3be28e17d3f712ab27f5723d96ed4bc70 (diff) | |
parent | 92fd730ade0c311ad42a1ebaa0154ec115452707 (diff) |
Merge branch 'master' into balder/add-filedist-rpc-to-configserver
19 files changed, 139 insertions, 199 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index f46fe39e8c4..818f7498af4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -90,9 +90,6 @@ public class DeploymentSpec { /** Adds missing required steps and reorders steps to a permissible order */ private static List<Step> completeSteps(List<Step> steps) { - // Ensure no duplicate deployments to the same zone - steps = new ArrayList<>(new LinkedHashSet<>(steps)); - // Add staging if required and missing if (steps.stream().anyMatch(step -> step.deploysTo(Environment.prod)) && steps.stream().noneMatch(step -> step.deploysTo(Environment.staging))) { diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 8bab2f83448..5050f88af31 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -97,12 +97,10 @@ public class DeploymentSpecTest { StringReader r = new StringReader( "<deployment version='1.0'>" + " <test/>" + - " <test/>" + " <staging/>" + " <prod>" + " <region active='false'>us-east1</region>" + - " <region active='false'>us-east1</region>" + - " <delay hours='3' minutes='30'/>" + + " <delay hours='3' minutes='30'/>" + " <region active='true'>us-west1</region>" + " </prod>" + "</deployment>" diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index f94b0f7284b..2d9bf45a39a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -105,12 +105,12 @@ public class DeploymentJobs { /** Returns whether this has some job status which is not a success */ public boolean hasFailures() { - return status.values().stream().anyMatch(jobStatus -> ! jobStatus.isSuccess()); + return JobList.from(status.values()).failing().anyMatch(); } /** Returns whether any job is currently in progress */ public boolean isRunning(Instant timeoutLimit) { - return status.values().stream().anyMatch(job -> job.isRunning(timeoutLimit)); + return JobList.from(status.values()).running(timeoutLimit).anyMatch(); } /** Returns whether the given job type is currently running and was started after timeoutLimit */ @@ -166,33 +166,44 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } + private static Optional<Long> requireId(Optional<Long> id, String message) { + Objects.requireNonNull(id, message); + if ( ! id.isPresent()) { + return id; + } + if (id.get() <= 0) { + throw new IllegalArgumentException(message); + } + return id; + } + /** Job types that exist in the build system */ public enum JobType { - component("component"), - systemTest("system-test", zone(SystemName.cd, "test", "cd-us-central-1"), zone("test", "us-east-1")), - stagingTest("staging-test", zone(SystemName.cd, "staging", "cd-us-central-1"), zone("staging", "us-east-3")), - productionCorpUsEast1("production-corp-us-east-1", zone("prod", "corp-us-east-1")), - productionUsEast3("production-us-east-3", zone("prod", "us-east-3")), - productionUsWest1("production-us-west-1", zone("prod", "us-west-1")), - productionUsCentral1("production-us-central-1", zone("prod", "us-central-1")), - productionApNortheast1("production-ap-northeast-1", zone("prod", "ap-northeast-1")), - productionApNortheast2("production-ap-northeast-2", zone("prod", "ap-northeast-2")), - productionApSoutheast1("production-ap-southeast-1", zone("prod", "ap-southeast-1")), - productionEuWest1("production-eu-west-1", zone("prod", "eu-west-1")), - productionCdUsCentral1("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), - productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); - - private final String id; + component ("component") , + systemTest ("system-test" , zone("test" , "us-east-1" ), zone(SystemName.cd, "test" , "cd-us-central-1")), + stagingTest ("staging-test" , zone("staging", "us-east-3" ), zone(SystemName.cd, "staging", "cd-us-central-1")), + productionCorpUsEast1 ("production-corp-us-east-1" , zone("prod" , "corp-us-east-1")), + productionUsEast3 ("production-us-east-3" , zone("prod" , "us-east-3" )), + productionUsWest1 ("production-us-west-1" , zone("prod" , "us-west-1" )), + productionUsCentral1 ("production-us-central-1" , zone("prod" , "us-central-1" )), + productionApNortheast1 ("production-ap-northeast-1" , zone("prod" , "ap-northeast-1")), + productionApNortheast2 ("production-ap-northeast-2" , zone("prod" , "ap-northeast-2")), + productionApSoutheast1 ("production-ap-southeast-1" , zone("prod" , "ap-southeast-1")), + productionEuWest1 ("production-eu-west-1" , zone("prod" , "eu-west-1" )), + productionCdUsCentral1 ("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), + productionCdUsCentral2 ("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); + + private final String jobName; private final ImmutableMap<SystemName, Zone> zones; - JobType(String id, Zone... zones) { - this.id = id; + JobType(String jobName, Zone... zones) { + this.jobName = jobName; this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(), zone -> zone))); } - public String id() { return id; } + public String jobName() { return jobName; } /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ public Optional<Zone> zone(SystemName system) { @@ -217,33 +228,17 @@ public class DeploymentJobs { return zone(system).map(Zone::region); } - public static JobType fromId(String id) { - switch (id) { - case "component" : return component; - case "system-test" : return systemTest; - case "staging-test" : return stagingTest; - case "production-corp-us-east-1" : return productionCorpUsEast1; - case "production-us-east-3" : return productionUsEast3; - case "production-us-west-1" : return productionUsWest1; - case "production-us-central-1" : return productionUsCentral1; - case "production-ap-northeast-1" : return productionApNortheast1; - case "production-ap-northeast-2" : return productionApNortheast2; - case "production-ap-southeast-1" : return productionApSoutheast1; - case "production-eu-west-1" : return productionEuWest1; - case "production-cd-us-central-1" : return productionCdUsCentral1; - case "production-cd-us-central-2" : return productionCdUsCentral2; - default : throw new IllegalArgumentException("Unknown job id '" + id + "'"); - } + public static JobType fromJobName(String jobName) { + return Stream.of(values()) + .filter(jobType -> jobType.jobName.equals(jobName)) + .findAny().orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); } /** Returns the job type for the given zone */ public static Optional<JobType> from(SystemName system, Zone zone) { - for (JobType job : values()) { - Optional<Zone> jobZone = job.zone(system); - if (jobZone.isPresent() && jobZone.get().equals(zone)) - return Optional.of(job); - } - return Optional.empty(); + return Stream.of(values()) + .filter(job -> job.zone(system).filter(zone::equals).isPresent()) + .findAny(); } /** Returns the job job type for the given environment and region or null if none */ @@ -290,7 +285,7 @@ public class DeploymentJobs { public JobType jobType() { return jobType; } public long projectId() { return projectId; } public long buildNumber() { return buildNumber; } - public boolean success() { return !jobError.isPresent(); } + public boolean success() { return ! jobError.isPresent(); } public Optional<JobError> jobError() { return jobError; } } @@ -298,23 +293,6 @@ public class DeploymentJobs { public enum JobError { unknown, outOfCapacity; - - public static Optional<JobError> from(boolean success) { - return Optional.of(success) - .filter(b -> !b) - .map(ignored -> unknown); - } - } - - private static Optional<Long> requireId(Optional<Long> id, String message) { - Objects.requireNonNull(id, message); - if (!id.isPresent()) { - return id; - } - if (id.get() <= 0) { - throw new IllegalArgumentException(message); - } - return id; } }
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index 8fb01577e2c..9201c74e761 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -16,19 +15,16 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.logging.Logger; -import java.util.stream.Collector; -import java.util.stream.Collectors; +import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toList; /** * This class determines the order of deployments according to an application's deployment spec. @@ -64,7 +60,7 @@ public class DeploymentOrder { // At this point we have deployed to system test, so deployment spec is available List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); Optional<DeploymentSpec.Step> currentStep = fromJob(job, application); - if (!currentStep.isPresent()) { + if ( ! currentStep.isPresent()) { return Collections.emptyList(); } @@ -75,13 +71,13 @@ public class DeploymentOrder { } // Postpone if step hasn't completed all its jobs for this change - if (!completedSuccessfully(currentStep.get(), application.deploying().get(), application)) { + if ( ! completedSuccessfully(currentStep.get(), application.deploying().get(), application)) { return Collections.emptyList(); } // Postpone next job if delay has not passed yet Duration delay = delayAfter(currentStep.get(), application); - if (postponeDeployment(delay, job, application)) { + if (shouldPostponeDeployment(delay, job, application)) { log.info(String.format("Delaying next job after %s of %s by %s", job, application, delay)); return Collections.emptyList(); } @@ -89,11 +85,11 @@ public class DeploymentOrder { DeploymentSpec.Step nextStep = deploymentSteps.get(currentIndex + 1); return nextStep.zones().stream() .map(this::toJob) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } - /** Returns whether the given job is first in a deployment */ - public boolean isFirst(JobType job) { + /** Returns whether the given job causes an application change */ + public boolean givesApplicationChange(JobType job) { return job == JobType.component; } @@ -113,35 +109,33 @@ public class DeploymentOrder { public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) { return deploymentSpec.steps().stream() .flatMap(step -> jobsFrom(step).stream()) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns job status sorted according to deployment spec */ - public Map<JobType, JobStatus> sortBy(DeploymentSpec deploymentSpec, Map<JobType, JobStatus> jobStatus) { - List<DeploymentJobs.JobType> jobs = jobsFrom(deploymentSpec); - return jobStatus.entrySet().stream() - .sorted(Comparator.comparingInt(kv -> jobs.indexOf(kv.getKey()))) - .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue), - Collections::unmodifiableMap)); + public List<JobStatus> sortBy(DeploymentSpec deploymentSpec, Collection<JobStatus> jobStatus) { + List<DeploymentJobs.JobType> sortedJobs = jobsFrom(deploymentSpec); + return jobStatus.stream() + .sorted(comparingInt(job -> sortedJobs.indexOf(job.type()))) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns deployments sorted according to declared zones */ - public Map<Zone, Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Map<Zone, Deployment> deployments) { + public List<Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Collection<Deployment> deployments) { List<Zone> productionZones = zones.stream() - .filter(z -> z.environment() == Environment.prod && z.region().isPresent()) + .filter(z -> z.region().isPresent()) .map(z -> new Zone(z.environment(), z.region().get())) - .collect(Collectors.toList()); - return deployments.entrySet().stream() - .sorted(Comparator.comparingInt(kv -> productionZones.indexOf(kv.getKey()))) - .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue), - Collections::unmodifiableMap)); + .collect(toList()); + return deployments.stream() + .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns jobs for the given step */ private List<JobType> jobsFrom(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns whether all jobs have completed successfully for given step */ @@ -167,7 +161,7 @@ public class DeploymentOrder { } /** Returns whether deployment should be postponed according to delay */ - private boolean postponeDeployment(Duration delay, JobType job, Application application) { + private boolean shouldPostponeDeployment(Duration delay, JobType job, Application application) { Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(job)) .flatMap(JobStatus::lastSuccess) .map(JobStatus.JobRun::at); @@ -177,9 +171,8 @@ public class DeploymentOrder { /** Find all steps that deploy to one or more zones */ private static List<DeploymentSpec.Step> deploymentSteps(Application application) { return application.deploymentSpec().steps().stream() - .filter(step -> step instanceof DeploymentSpec.DeclaredZone || - step instanceof DeploymentSpec.ParallelZones) - .collect(Collectors.toList()); + .filter(step -> ! step.zones().isEmpty()) + .collect(toList()); } /** Determines the delay that should pass after the given step */ @@ -200,13 +193,4 @@ public class DeploymentOrder { return totalDelay; } - private static <T, K, U> Collector<T, ?, Map<K,U>> toLinkedMap(Function<? super T, ? extends K> keyMapper, - Function<? super T, ? extends U> valueMapper) { - return Collectors.toMap(keyMapper, valueMapper, - (u, v) -> { - throw new IllegalStateException(String.format("Duplicate key %s", u)); - }, - LinkedHashMap::new); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e5c7b7f1c2f..2cd273c82f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -82,7 +82,7 @@ public class DeploymentTrigger { // Handle successful starting and ending if (report.success()) { - if (order.isFirst(report.jobType())) { // the first job tells us that a change occurred + if (order.givesApplicationChange(report.jobType())) { if (acceptNewRevisionNow(application)) { // Set this as the change we are doing, unless we are already pushing a platform change if ( ! ( application.deploying().isPresent() && diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java index 56b4023f932..e25db10a8cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java @@ -87,9 +87,9 @@ public class PolledBuildSystem implements BuildSystem { Optional<Long> projectId = projectId(application); if (projectId.isPresent()) { - jobsToRun.add(new BuildJob(projectId.get(), jobType.id())); + jobsToRun.add(new BuildJob(projectId.get(), jobType.jobName())); } else { - log.warning("Not queuing " + jobType.id() + " for " + application.toShortString() + + log.warning("Not queuing " + jobType.jobName() + " for " + application.toShortString() + " because project ID is missing"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 3bd1abdf607..2745862f68a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -215,7 +215,7 @@ public class ApplicationSerializer { } private void toSlime(JobStatus jobStatus, Cursor object) { - object.setString(jobTypeField, jobStatus.type().id()); + object.setString(jobTypeField, jobStatus.type().jobName()); if (jobStatus.jobError().isPresent()) object.setString(errorField, jobStatus.jobError().get().name()); @@ -373,7 +373,7 @@ public class ApplicationSerializer { } private JobStatus jobStatusFromSlime(Inspector object) { - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromId(object.field(jobTypeField).asString()); + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromJobName(object.field(jobTypeField).asString()); Optional<JobError> jobError = Optional.empty(); if (object.field(errorField).valid()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a259e221a1e..50b80e34788 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -354,14 +354,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // Jobs sorted according to deployment spec - Map<DeploymentJobs.JobType, JobStatus> jobStatus = controller.applications().deploymentTrigger() + List<JobStatus> jobStatus = controller.applications().deploymentTrigger() .deploymentOrder() - .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus()); + .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus().values()); Cursor deploymentsArray = response.setArray("deploymentJobs"); - for (JobStatus job : jobStatus.values()) { + for (JobStatus job : jobStatus) { Cursor jobObject = deploymentsArray.addObject(); - jobObject.setString("type", job.type().id()); + jobObject.setString("type", job.type().jobName()); jobObject.setBool("success", job.isSuccess()); job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered"))); @@ -382,11 +382,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { globalRotationsArray.addString(rotation.toString()); // Deployments sorted according to deployment spec - Map<Zone, Deployment> deployments = controller.applications().deploymentTrigger() + List<Deployment> deployments = controller.applications().deploymentTrigger() .deploymentOrder() - .sortBy(application.deploymentSpec().zones(), application.deployments()); + .sortBy(application.deploymentSpec().zones(), application.deployments().values()); Cursor instancesArray = response.setArray("instances"); - for (Deployment deployment : deployments.values()) { + for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); deploymentObject.setString("environment", deployment.zone().environment().value()); deploymentObject.setString("region", deployment.zone().region().value()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index bcc72245f0b..394d8a8af26 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -101,7 +101,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> { Cursor applicationObject = failingArray.addObject(); toSlime(applicationObject, application, request); - applicationObject.setString("failing", firstFailing.type().id()); + applicationObject.setString("failing", firstFailing.type().jobName()); }); }); } @@ -124,7 +124,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> { Cursor applicationObject = runningArray.addObject(); toSlime(applicationObject, application, request); - applicationObject.setString("running", lastDeploying.type().id()); + applicationObject.setString("running", lastDeploying.type().jobName()); }); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 3dbff0b4aa3..be30d0d4ac3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -108,7 +108,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { LockedApplication application = controller.applications().require(applicationId, lock); JobType jobType = Optional.of(asString(request.getData())) .filter(s -> !s.isEmpty()) - .map(JobType::fromId) + .map(JobType::fromJobName) .orElse(JobType.component); // Since this is a manual operation we likely want it to trigger as soon as possible so we add it at to the // front of the queue @@ -120,7 +120,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor cursor = slime.setObject(); - cursor.setString("message", "Triggered " + jobType.id() + " for " + applicationId); + cursor.setString("message", "Triggered " + jobType.jobName() + " for " + applicationId); return new SlimeJsonResponse(slime); } } @@ -174,7 +174,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { report.field("tenant").asString(), report.field("application").asString(), report.field("instance").asString()), - JobType.fromId(report.field("jobName").asString()), + JobType.fromJobName(report.field("jobName").asString()), report.field("projectId").asLong(), report.field("buildNumber").asLong(), jobError diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index aea66f3cd67..f6a650e71be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -94,7 +94,7 @@ public class ControllerTest { // staging job - succeeding Version version1 = Version.fromString("6.1"); // Set in config server mock Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); assertFalse("Revision is currently not known", ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); tester.deployAndNotify(app1, applicationPackage, true, systemTest); @@ -136,7 +136,7 @@ public class ControllerTest { tester.clock().advance(Duration.ofSeconds(1)); // system and staging test job - succeeding - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); assertStatus(JobStatus.initial(systemTest) .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) @@ -163,7 +163,7 @@ public class ControllerTest { .environment(Environment.prod) .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); try { tester.deploy(systemTest, app1, applicationPackage); fail("Expected exception due to unallowed production deployment removal"); @@ -205,7 +205,7 @@ public class ControllerTest { Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); // First deployment: An application change - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); @@ -228,7 +228,7 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); @@ -425,7 +425,7 @@ public class ControllerTest { // app1: staging-test job fails with out of capacity and is added to the front of the queue tester.deploy(stagingTest, app1, applicationPackage); tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); - assertEquals(stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(project1, buildSystem.jobs().get(0).projectId()); // app2 and app3: Completes deployment @@ -461,15 +461,15 @@ public class ControllerTest { List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); assertEquals(2, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project2, nextJobs.get(0).projectId()); - assertEquals(stagingTest.id(), nextJobs.get(1).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(1).jobName()); assertEquals(project3, nextJobs.get(1).projectId()); // And finally the requeued job for app1 nextJobs = buildSystem.takeJobsToRun(); assertEquals(1, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project1, nextJobs.get(0).projectId()); } @@ -480,20 +480,6 @@ public class ControllerTest { assertEquals(expectedStatus, existingStatus); } - private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError) { - return new JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } - - private JobReport mockReport(Application application, JobType jobType, boolean success) { - return mockReport(application, jobType, JobError.from(success)); - } - @Test public void testGlobalRotations() throws IOException { // Setup tester and app def diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 9e228e0becb..773ecf313cc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.unknown; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -140,6 +141,20 @@ public class DeploymentTester { completeDeployment(application, applicationPackage, Optional.empty(), true); } + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, boolean success) { + return jobReport(application, jobType, Optional.ofNullable(success ? null : unknown)); + } + + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { + return new DeploymentJobs.JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 42, + jobError + ); + } + /** Deploy application using the given application package, but expecting to stop after test phases */ public void deployTestOnly(Application application, ApplicationPackage applicationPackage) { notifyJobCompletion(JobType.component, application, true); @@ -172,7 +187,7 @@ public class DeploymentTester { } public void notifyJobCompletion(JobType jobType, Application application, boolean success) { - notifyJobCompletion(jobType, application, DeploymentJobs.JobError.from(success)); + notifyJobCompletion(jobType, application, Optional.ofNullable(success ? null : unknown)); } public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) { @@ -228,7 +243,7 @@ public class DeploymentTester { for (JobType job : jobs) { BuildService.BuildJob buildJob = findJob(application, job); assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId()); - assertEquals(job.id(), buildJob.jobName()); + assertEquals(job.jobName(), buildJob.jobName()); } if (expectOnlyTheseJobs) assertEquals(jobs.length, countJobsOf(application)); @@ -237,7 +252,7 @@ public class DeploymentTester { private BuildService.BuildJob findJob(Application application, JobType jobType) { for (BuildService.BuildJob job : buildSystem().jobs()) - if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id())) + if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.jobName())) return job; throw new NoSuchElementException(jobType + " is not scheduled for " + application); } @@ -247,15 +262,6 @@ public class DeploymentTester { .filter(job -> job.projectId() == application.deploymentJobs().projectId().get()) .count(); } - private DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } private static ApplicationPackage applicationPackage(String upgradePolicy) { return new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 022fa705def..5a753617761 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -72,7 +72,7 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); tester.failureRedeployer().maintain(); assertEquals("Retried dead job", 1, tester.buildSystem().jobs().size()); - assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), tester.buildSystem().jobs().get(0).jobName()); } @Test @@ -131,7 +131,7 @@ public class DeploymentTriggerTest { // Consume us-west-1 job without reporting completion assertEquals(1, buildSystem.jobs().size()); - assertEquals(JobType.productionUsWest1.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), buildSystem.jobs().get(0).jobName()); buildSystem.takeJobsToRun(); // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress @@ -184,8 +184,8 @@ public class DeploymentTriggerTest { // Deploys in two regions in parallel assertEquals(2, tester.buildSystem().jobs().size()); - assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName()); - assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName()); + assertEquals(JobType.productionUsEast3.jobName(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), tester.buildSystem().jobs().get(1).jobName()); tester.buildSystem().takeJobsToRun(); tester.deploy(JobType.productionUsWest1, application, applicationPackage, false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java index 0293ea08d65..1b1a4feaa4e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.util.EnumMap; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.QUEUED; @@ -161,11 +162,11 @@ public class MockBuildService implements BuildService { jobType, projectId, 42, - JobError.from(success) + Optional.ofNullable(success ? null : JobError.unknown) )); } - private BuildJob buildJob() { return new BuildJob(projectId, jobType.id()); } + private BuildJob buildJob() { return new BuildJob(projectId, jobType.jobName()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 2782dd6ec3b..87ef7ed07b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -70,7 +70,7 @@ public class FailureRedeployerTest { tester.clock().advance(Duration.ofMinutes(1)); tester.failureRedeployer().maintain(); assertFalse("Job is not retried", tester.buildSystem().jobs().stream() - .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id()))); + .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName()))); // Test environments pass tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); @@ -109,14 +109,14 @@ public class FailureRedeployerTest { tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); // staging-test starts, but does not complete - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); tester.failureRedeployer().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); // Just over 12 hours pass, job is retried tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); tester.failureRedeployer().maintain(); - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); // Deployment completes tester.deploy(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); @@ -212,7 +212,7 @@ public class FailureRedeployerTest { // Production job starts, but does not complete assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.jobName(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); // Failure re-deployer runs diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 4886eba40b6..13636122cfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -49,7 +49,7 @@ public class OutstandingChangeDeployerTest { List<BuildService.BuildJob> jobs = tester.buildSystem().jobs(); assertEquals(1, jobs.size()); assertEquals(11, jobs.get(0).projectId()); - assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName()); + assertEquals(DeploymentJobs.JobType.systemTest.jobName(), jobs.get(0).jobName()); assertFalse(tester.application("app1").hasOutstandingChange()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 0414cda3f55..836e9756c24 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -643,7 +643,7 @@ public class UpgraderTest { tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); tester.upgrader().maintain(); assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Upgrade is rescheduled", DeploymentJobs.JobType.systemTest.id(), + assertEquals("Upgrade is rescheduled", DeploymentJobs.JobType.systemTest.jobName(), tester.buildSystem().jobs().get(0).jobName()); tester.deployCompletely(default4, applicationPackage); assertEquals(version, tester.application(default4.id()).deployedVersion().get()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 1638a2845ed..ee8ac5e3b8e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -96,14 +96,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Response response; response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.GET)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); // Check that GET didn't affect the enqueued jobs. response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); Thread.sleep(50); @@ -163,7 +163,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + app.id().tenant().value() + "/application/" + app.id().application().value(), "invalid".getBytes(StandardCharsets.UTF_8), Request.Method.POST), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job id 'invalid'\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job name 'invalid'\"}"); // component is triggered if no job is specified in request body assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + @@ -172,7 +172,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { 200, "{\"message\":\"Triggered component for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.component.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.component.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); buildSystem.takeJobsToRun(); @@ -182,7 +182,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { "staging-test".getBytes(StandardCharsets.UTF_8), Request.Method.POST), 200, "{\"message\":\"Triggered staging-test for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); } @@ -197,14 +197,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Optional<JobError> jobError) { return "{\n" + - " \"projectId\" : " + projectId + ",\n" + - " \"jobName\" :\"" + jobType.id() + "\",\n" + - " \"buildNumber\" : " + buildNumber + ",\n" + - jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + - " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + - " \"application\" :\"" + applicationId.application().value() + "\",\n" + - " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + - "}"; + " \"projectId\" : " + projectId + ",\n" + + " \"jobName\" :\"" + jobType.jobName() + "\",\n" + + " \"buildNumber\" : " + buildNumber + ",\n" + + jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + + " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + + " \"application\" :\"" + applicationId.application().value() + "\",\n" + + " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + + "}"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 1157863f009..e97742aba5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -294,8 +294,8 @@ public class VersionStatusTest { Version versionWithUnknownTag = new Version("6.1.2"); Application app = tester.createAndDeploy("tenant1", "domain1","application1", Environment.test, 11); - applications.notifyJobCompletion(mockReport(app, component, true)); - applications.notifyJobCompletion(mockReport(app, systemTest, true)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, component, true)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, systemTest, true)); List<VespaVersion> vespaVersions = VersionStatus.compute(tester.controller()).versions(); @@ -312,14 +312,4 @@ public class VersionStatusTest { .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); } - private DeploymentJobs.JobReport mockReport(Application application, DeploymentJobs.JobType jobType, boolean success) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - JobError.from(success) - ); - } - } |