diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2017-08-28 14:52:02 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-28 14:52:02 +0200 |
commit | 622361fbcb6314ca23d18f64248d4f5f84230803 (patch) | |
tree | 4a92e0cba01ad937e25a3d720c52e079d94f455d | |
parent | cbc229afc827385775de59d5067d85de8c916e0c (diff) | |
parent | 43b7fdad9e9e718f1fff146d3e25ce898dcf577c (diff) |
Merge pull request #3228 from vespa-engine/mpolden/ignore-stale-job-data
Ignore stale job data in failure re-deployer
9 files changed, 430 insertions, 57 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 3fcd285e0fc..fa7a48c85c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -80,21 +80,11 @@ public class ApplicationList { return listOf(list.stream().filter(application -> ! failingOn(version, application))); } - /** Returns the subset of applications which have one or more deployment jobs failing for the current change */ - public ApplicationList hasDeploymentFailures() { - return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get()))); - } - /** Returns the subset of applications which have at least one deployment */ public ApplicationList hasDeployment() { return listOf(list.stream().filter(a -> !a.deployments().isEmpty())); } - /** Returns the subset of applications that are currently deploying a change */ - public ApplicationList isDeploying() { - return listOf(list.stream().filter(application -> application.deploying().isPresent())); - } - /** Returns the subset of applications which started failing after the given instant */ public ApplicationList startedFailingAfter(Instant instant) { return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant))); @@ -140,18 +130,6 @@ public class ApplicationList { return listOf(list.stream().filter(a -> !hasRunningJob(a, change))); } - /** Returns the subset of applications which currently do not have any job in progress */ - public ApplicationList notRunningJob() { - return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress())); - } - - /** Returns the subset of applications which has a job that started running before the given instant */ - public ApplicationList jobRunningSince(Instant instant) { - return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince() - .map(at -> at.isBefore(instant)) - .orElse(false))); - } - /** Returns the subset of applications which deploys to given environment and region */ public ApplicationList deploysTo(Environment environment, RegionName region) { return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region)))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index d9256f94086..d775dd2a356 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -118,11 +117,6 @@ public class DeploymentJobs { return status.values().stream().anyMatch(JobStatus::inProgress); } - /** Returns whether any job is failing for the given change */ - public boolean failingOn(Change change) { - return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change)); - } - /** Returns whether change can be deployed to the given environment */ public boolean isDeployableTo(Environment environment, Optional<Change> change) { if (environment == null || !change.isPresent()) { @@ -147,15 +141,6 @@ public class DeploymentJobs { return failingSince; } - /** Returns the time at which the oldest running job started */ - public Optional<Instant> runningSince() { - return jobStatus().values().stream() - .filter(JobStatus::inProgress) - .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at())) - .map(jobStatus -> jobStatus.lastTriggered().get().at()) - .findFirst(); - } - /** * Returns the id of the Screwdriver project running these deployment jobs * - or empty when this is not known or does not exist. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 2bc219dde62..ac84f3685ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -89,13 +89,13 @@ public class DeploymentTrigger { /** * Called periodically to cause triggering of jobs in the background */ - public void triggerFailing(ApplicationId applicationId) { + public void triggerFailing(ApplicationId applicationId, String cause) { try (Lock lock = applications().lock(applicationId)) { Application application = applications().require(applicationId); if (shouldRetryFromBeginning(application)) { // failed for a long time: Discard existing change and restart from the component job application = application.withDeploying(Optional.empty()); - application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock); + application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock); applications().store(application, lock); } else { // retry the failed job (with backoff) @@ -103,7 +103,7 @@ public class DeploymentTrigger { JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); if (isFailing(jobStatus)) { if (shouldRetryNow(jobStatus)) { - application = trigger(jobType, application, "Retrying failing job", lock); + application = trigger(jobType, application, "Retrying failing job: " + cause, lock); applications().store(application, lock); } break; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java index 9e8f902a8db..38d4a4a8a81 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java @@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Attempts redeployment of failed jobs and deployments. @@ -16,6 +19,8 @@ import java.util.List; * @author bratseth */ public class FailureRedeployer extends Maintainer { + + private final static Duration jobTimeout = Duration.ofHours(12); public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl); @@ -23,20 +28,61 @@ public class FailureRedeployer extends Maintainer { @Override public void maintain() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying(); - List<Application> toTrigger = new ArrayList<>(); + List<Application> applications = controller().applications().asList(); + retryFailingJobs(applications); + retryStuckJobs(applications); + } + + private void retryFailingJobs(List<Application> applications) { + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + if (application.deploymentJobs().inProgress()) { + continue; + } + Optional<Map.Entry<JobType, JobStatus>> failingJob = jobFailingFor(application); + failingJob.ifPresent(job -> triggerFailing(application, "Job " + job.getKey().id() + + " has been failing since " + job.getValue().lastCompleted().get())); + } + } - // Applications with deployment failures for current change and no running jobs - toTrigger.addAll(applications.hasDeploymentFailures() - .notRunningJob() - .asList()); + private void retryStuckJobs(List<Application> applications) { + Instant maxAge = controller().clock().instant().minus(jobTimeout); + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + Optional<Map.Entry<JobType, JobStatus>> job = oldestRunningJob(application); + if (!job.isPresent()) { + continue; + } + // Ignore job if it doesn't belong to a zone in this system + if (!job.get().getKey().zone(controller().system()).isPresent()) { + continue; + } + if (job.get().getValue().lastTriggered().get().at().isBefore(maxAge)) { + triggerFailing(application, "Job " + job.get().getKey().id() + + " has been running for more than " + jobTimeout); + } + } + } + + private Optional<Map.Entry<JobType, JobStatus>> jobFailingFor(Application application) { + return application.deploymentJobs().jobStatus().entrySet().stream() + .filter(e -> !e.getValue().isSuccess() && e.getValue().lastCompletedFor(application.deploying().get())) + .findFirst(); + } - // Applications with jobs that have been in progress for more than 12 hours - Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12)); - toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList()); + private Optional<Map.Entry<JobType, JobStatus>> oldestRunningJob(Application application) { + return application.deploymentJobs().jobStatus().entrySet().stream() + .filter(kv -> kv.getValue().inProgress()) + .sorted(Comparator.comparing(kv -> kv.getValue().lastTriggered().get().at())) + .findFirst(); + } - toTrigger.forEach(application -> controller().applications().deploymentTrigger() - .triggerFailing(application.id())); + private void triggerFailing(Application application, String cause) { + controller().applications().deploymentTrigger().triggerFailing(application.id(), cause); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 62b935842f7..bf21467bc8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -27,9 +27,11 @@ public class ZoneRegistryMock implements ZoneRegistry { deploymentTimeToLive.put(zone, duration); } + private SystemName system = SystemName.main; + @Override public SystemName system() { - return SystemName.main; + return system; } @Override @@ -71,4 +73,8 @@ public class ZoneRegistryMock implements ZoneRegistry { public URI getDashboardUri() { return URI.create("http://dashboard.test"); } + + public void setSystem(SystemName system) { + this.system = system; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 32d1714ea52..0e816be864d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -160,7 +160,7 @@ public class DeploymentTester { } public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { - job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); + job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); } public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index ce06910240b..0f48afc0ca4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -34,14 +34,14 @@ public class DeploymentTriggerTest { tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried job", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(7)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index cde511a9076..052ca87f791 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -3,13 +3,20 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import org.junit.Test; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import static org.junit.Assert.assertEquals; @@ -166,4 +173,60 @@ public class FailureRedeployerTest { assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); } + @Test + public void retryIgnoresStaleJobData() throws Exception { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + + // Current system version, matches version in test data + Version version = Version.fromString("6.141.117"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .region("cd-us-central-1") + .build(); + + // New version is released + version = Version.fromString("6.142.1"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true); + + tester.deploy(DeploymentJobs.JobType.stagingTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, application, true); + + // Production job starts, but does not complete + assertEquals(1, tester.buildSystem().jobs().size()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + tester.buildSystem().takeJobsToRun(); + + // Failure re-deployer runs + tester.failureRedeployer().maintain(); + assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); + + // Deployment completes + tester.notifyJobCompletion(DeploymentJobs.JobType.productionCdUsCentral1, application, true); + assertFalse("Change deployed", tester.application(application.id()).deploying().isPresent()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json new file mode 100644 index 00000000000..323889c7c45 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json @@ -0,0 +1,295 @@ +{ + "id": "vespa:canary:default", + "deploymentSpecField": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<deployment version=\"1.0\">\n <upgrade policy='canary'/>\n <test />\n <staging />\n <prod>\n <region active=\"true\">cd-us-central-1</region>\n </prod>\n</deployment>\n", + "validationOverrides": "<validation-overrides>\n <allow until=\"2017-04-27\">force-automatic-tenant-upgrade-test</allow>\n</validation-overrides>\n", + "deployments": [ + { + "zone": { + "environment": "prod", + "region": "cd-us-central-1" + }, + "version": "6.141.117", + "deployTime": 1503901783487, + "applicationPackageRevision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + } + } + ], + "deploymentJobs": { + "projectId": 191186, + "jobStatus": [ + { + "jobType": "production-eu-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034019032 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033995026 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033995026 + } + }, + { + "jobType": "production-cd-us-central-1", + "jobError": "unknown", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903384816 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "firstFailing": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903556127 + } + }, + { + "jobType": "component", + "lastTriggered": { + "version": "6.141.109", + "at": 1503867517105 + }, + "lastCompleted": { + "version": "6.141.109", + "at": 1503867704464 + }, + "lastSuccess": { + "version": "6.141.109", + "at": 1503867704464 + } + }, + { + "jobType": "production-corp-us-east-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034428590 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034114538 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034114538 + } + }, + { + "jobType": "production-ap-southeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034265146 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034097617 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034097617 + } + }, + { + "jobType": "production-us-central-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033800484 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034273753 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034273753 + } + }, + { + "jobType": "staging-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900683154 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + } + }, + { + "jobType": "system-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503899621243 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + } + }, + { + "jobType": "production-us-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034273768 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034019015 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034019015 + } + }, + { + "jobType": "production-ap-northeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033995045 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034257206 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034257206 + } + }, + { + "jobType": "production-ap-northeast-2", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034257222 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034265048 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034265048 + } + }, + { + "jobType": "production-us-east-3", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034114555 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033800469 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033800469 + } + } + ], + "selfTriggering": false + }, + "outstandingChangeField": false +} |