Merge branch 'master' into balder/move-databuffer-and-compression-to-vespalib

9 files changed, 430 insertions, 57 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index 3fcd285e0fc..fa7a48c85c2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -80,21 +80,11 @@ public class ApplicationList {
         return listOf(list.stream().filter(application -> ! failingOn(version, application)));
     }
 
-    /** Returns the subset of applications which have one or more deployment jobs failing for the current change */
-    public ApplicationList hasDeploymentFailures() {
-        return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get())));
-    }
-
     /** Returns the subset of applications which have at least one deployment */
     public ApplicationList hasDeployment() {
         return listOf(list.stream().filter(a -> !a.deployments().isEmpty()));
     }
 
-    /** Returns the subset of applications that are currently deploying a change */
-    public ApplicationList isDeploying() {
-        return listOf(list.stream().filter(application -> application.deploying().isPresent()));
-    }
-
     /** Returns the subset of applications which started failing after the given instant */
     public ApplicationList startedFailingAfter(Instant instant) {
         return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant)));
@@ -140,18 +130,6 @@ public class ApplicationList {
         return listOf(list.stream().filter(a -> !hasRunningJob(a, change)));
     }
 
-    /** Returns the subset of applications which currently do not have any job in progress */
-    public ApplicationList notRunningJob() {
-        return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress()));
-    }
-
-    /** Returns the subset of applications which has a job that started running before the given instant */
-    public ApplicationList jobRunningSince(Instant instant) {
-        return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince()
-                .map(at -> at.isBefore(instant))
-                .orElse(false)));
-    }
-
     /** Returns the subset of applications which deploys to given environment and region */
     public ApplicationList deploysTo(Environment environment, RegionName region) {
         return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region))));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index d9256f94086..d775dd2a356 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -118,11 +117,6 @@ public class DeploymentJobs {
         return status.values().stream().anyMatch(JobStatus::inProgress);
     }
 
-    /** Returns whether any job is failing for the given change */
-    public boolean failingOn(Change change) {
-        return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change));
-    }
-
     /** Returns whether change can be deployed to the given environment */
     public boolean isDeployableTo(Environment environment, Optional<Change> change) {
         if (environment == null || !change.isPresent()) {
@@ -147,15 +141,6 @@ public class DeploymentJobs {
         return failingSince;
     }
 
-    /** Returns the time at which the oldest running job started */
-    public Optional<Instant> runningSince() {
-        return jobStatus().values().stream()
-                .filter(JobStatus::inProgress)
-                .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at()))
-                .map(jobStatus -> jobStatus.lastTriggered().get().at())
-                .findFirst();
-    }
-
     /**
      * Returns the id of the Screwdriver project running these deployment jobs 
      * - or empty when this is not known or does not exist.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 2bc219dde62..ac84f3685ca 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -89,13 +89,13 @@ public class DeploymentTrigger {
     /**
      * Called periodically to cause triggering of jobs in the background
      */
-    public void triggerFailing(ApplicationId applicationId) {
+    public void triggerFailing(ApplicationId applicationId, String cause) {
         try (Lock lock = applications().lock(applicationId)) {
             Application application = applications().require(applicationId);
             if (shouldRetryFromBeginning(application)) {
                 // failed for a long time: Discard existing change and restart from the component job
                 application = application.withDeploying(Optional.empty());
-                application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock);
+                application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock);
                 applications().store(application, lock);
             } else {
                 // retry the failed job (with backoff)
@@ -103,7 +103,7 @@ public class DeploymentTrigger {
                     JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
                     if (isFailing(jobStatus)) {
                         if (shouldRetryNow(jobStatus)) {
-                            application = trigger(jobType, application, "Retrying failing job", lock);
+                            application = trigger(jobType, application, "Retrying failing job: " + cause, lock);
                             applications().store(application, lock);
                         }
                         break;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java
index 9e8f902a8db..38d4a4a8a81 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java
@@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 
 /**
  * Attempts redeployment of failed jobs and deployments.
@@ -16,6 +19,8 @@ import java.util.List;
  * @author bratseth
  */
 public class FailureRedeployer extends Maintainer {
+
+    private final static Duration jobTimeout = Duration.ofHours(12);
     
     public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) {
         super(controller, interval, jobControl);
@@ -23,20 +28,61 @@ public class FailureRedeployer extends Maintainer {
 
     @Override
     public void maintain() {
-        ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying();
-        List<Application> toTrigger = new ArrayList<>();
+        List<Application> applications = controller().applications().asList();
+        retryFailingJobs(applications);
+        retryStuckJobs(applications);
+    }
+
+    private void retryFailingJobs(List<Application> applications) {
+        for (Application application : applications) {
+            if (!application.deploying().isPresent()) {
+                continue;
+            }
+            if (application.deploymentJobs().inProgress()) {
+                continue;
+            }
+            Optional<Map.Entry<JobType, JobStatus>> failingJob = jobFailingFor(application);
+            failingJob.ifPresent(job -> triggerFailing(application, "Job " + job.getKey().id() +
+                    " has been failing since " + job.getValue().lastCompleted().get()));
+        }
+    }
 
-        // Applications with deployment failures for current change and no running jobs
-        toTrigger.addAll(applications.hasDeploymentFailures()
-                                 .notRunningJob()
-                                 .asList());
+    private void retryStuckJobs(List<Application> applications) {
+        Instant maxAge = controller().clock().instant().minus(jobTimeout);
+        for (Application application : applications) {
+            if (!application.deploying().isPresent()) {
+                continue;
+            }
+            Optional<Map.Entry<JobType, JobStatus>> job = oldestRunningJob(application);
+            if (!job.isPresent()) {
+                continue;
+            }
+            // Ignore job if it doesn't belong to a zone in this system
+            if (!job.get().getKey().zone(controller().system()).isPresent()) {
+                continue;
+            }
+            if (job.get().getValue().lastTriggered().get().at().isBefore(maxAge)) {
+                triggerFailing(application, "Job " + job.get().getKey().id() +
+                        " has been running for more than " + jobTimeout);
+            }
+        }
+    }
+
+    private Optional<Map.Entry<JobType, JobStatus>> jobFailingFor(Application application) {
+        return application.deploymentJobs().jobStatus().entrySet().stream()
+                .filter(e -> !e.getValue().isSuccess() && e.getValue().lastCompletedFor(application.deploying().get()))
+                .findFirst();
+    }
 
-        // Applications with jobs that have been in progress for more than 12 hours
-        Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12));
-        toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList());
+    private Optional<Map.Entry<JobType, JobStatus>> oldestRunningJob(Application application) {
+        return application.deploymentJobs().jobStatus().entrySet().stream()
+                .filter(kv -> kv.getValue().inProgress())
+                .sorted(Comparator.comparing(kv -> kv.getValue().lastTriggered().get().at()))
+                .findFirst();
+    }
 
-        toTrigger.forEach(application -> controller().applications().deploymentTrigger()
-                .triggerFailing(application.id()));
+    private void triggerFailing(Application application, String cause) {
+        controller().applications().deploymentTrigger().triggerFailing(application.id(), cause);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java
index 62b935842f7..bf21467bc8d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java
@@ -27,9 +27,11 @@ public class ZoneRegistryMock implements ZoneRegistry {
         deploymentTimeToLive.put(zone, duration);
     }
 
+    private SystemName system = SystemName.main;
+
     @Override
     public SystemName system() {
-        return SystemName.main;
+        return system;
     }
 
     @Override
@@ -71,4 +73,8 @@ public class ZoneRegistryMock implements ZoneRegistry {
     public URI getDashboardUri() {
         return URI.create("http://dashboard.test");
     }
+
+    public void setSystem(SystemName system) {
+        this.system = system;
+    }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 32d1714ea52..0e816be864d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -160,7 +160,7 @@ public class DeploymentTester {
     }
 
     public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) {
-        job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion));
+        job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion));
     }
 
     public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index ce06910240b..0f48afc0ca4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -34,14 +34,14 @@ public class DeploymentTriggerTest {
         tester.buildSystem().takeJobsToRun();
         assertEquals("Job removed", 0, tester.buildSystem().jobs().size());        
         tester.clock().advance(Duration.ofHours(2));
-        tester.deploymentTrigger().triggerFailing(app1.id());
+        tester.deploymentTrigger().triggerFailing(app1.id(), "unit test");
         assertEquals("Retried job", 1, tester.buildSystem().jobs().size());
         assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName());
 
         tester.buildSystem().takeJobsToRun();
         assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
         tester.clock().advance(Duration.ofHours(7));
-        tester.deploymentTrigger().triggerFailing(app1.id());
+        tester.deploymentTrigger().triggerFailing(app1.id(), "unit test");
         assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size());
         assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index cde511a9076..052ca87f791 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -3,13 +3,20 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer;
 import org.junit.Test;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.time.Duration;
 
 import static org.junit.Assert.assertEquals;
@@ -166,4 +173,60 @@ public class FailureRedeployerTest {
         assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty());
     }
 
+    @Test
+    public void retryIgnoresStaleJobData() throws Exception {
+        DeploymentTester tester = new DeploymentTester();
+        tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd);
+
+        // Current system version, matches version in test data
+        Version version = Version.fromString("6.141.117");
+        tester.configServerClientMock().setDefaultConfigServerVersion(version);
+        tester.updateVersionStatus(version);
+        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+
+        // Load test data data
+        ApplicationSerializer serializer = new ApplicationSerializer();
+        byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json"));
+        Slime slime = SlimeUtils.jsonToSlime(json);
+        Application application = serializer.fromSlime(slime);
+        try (Lock lock = tester.controller().applications().lock(application.id())) {
+            tester.controller().applications().store(application, lock);
+        }
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .upgradePolicy("canary")
+                .region("cd-us-central-1")
+                .build();
+
+        // New version is released
+        version = Version.fromString("6.142.1");
+        tester.configServerClientMock().setDefaultConfigServerVersion(version);
+        tester.updateVersionStatus(version);
+        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        tester.upgrader().maintain();
+
+        // Test environments pass
+        tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage);
+        tester.buildSystem().takeJobsToRun();
+        tester.clock().advance(Duration.ofMinutes(10));
+        tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true);
+
+        tester.deploy(DeploymentJobs.JobType.stagingTest, application, applicationPackage);
+        tester.buildSystem().takeJobsToRun();
+        tester.clock().advance(Duration.ofMinutes(10));
+        tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, application, true);
+
+        // Production job starts, but does not complete
+        assertEquals(1, tester.buildSystem().jobs().size());
+        assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName());
+        tester.buildSystem().takeJobsToRun();
+
+        // Failure re-deployer runs
+        tester.failureRedeployer().maintain();
+        assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty());
+
+        // Deployment completes
+        tester.notifyJobCompletion(DeploymentJobs.JobType.productionCdUsCentral1, application, true);
+        assertFalse("Change deployed", tester.application(application.id()).deploying().isPresent());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json
new file mode 100644
index 00000000000..323889c7c45
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json
@@ -0,0 +1,295 @@
+{
+  "id": "vespa:canary:default",
+  "deploymentSpecField": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<deployment version=\"1.0\">\n    <upgrade policy='canary'/>\n    <test />\n    <staging />\n    <prod>\n        <region active=\"true\">cd-us-central-1</region>\n    </prod>\n</deployment>\n",
+  "validationOverrides": "<validation-overrides>\n    <allow until=\"2017-04-27\">force-automatic-tenant-upgrade-test</allow>\n</validation-overrides>\n",
+  "deployments": [
+    {
+      "zone": {
+        "environment": "prod",
+        "region": "cd-us-central-1"
+      },
+      "version": "6.141.117",
+      "deployTime": 1503901783487,
+      "applicationPackageRevision": {
+        "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+        "sourceRevision": {
+          "repositoryField": "git@git.test:vespa/canary-application.git",
+          "branchField": "origin/canary-cd",
+          "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+        }
+      }
+    }
+  ],
+  "deploymentJobs": {
+    "projectId": 191186,
+    "jobStatus": [
+      {
+        "jobType": "production-eu-west-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034019032
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493033995026
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493033995026
+        }
+      },
+      {
+        "jobType": "production-cd-us-central-1",
+        "jobError": "unknown",
+        "lastTriggered": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503903384816
+        },
+        "lastCompleted": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503903659364
+        },
+        "firstFailing": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503903659364
+        },
+        "lastSuccess": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503903556127
+        }
+      },
+      {
+        "jobType": "component",
+        "lastTriggered": {
+          "version": "6.141.109",
+          "at": 1503867517105
+        },
+        "lastCompleted": {
+          "version": "6.141.109",
+          "at": 1503867704464
+        },
+        "lastSuccess": {
+          "version": "6.141.109",
+          "at": 1503867704464
+        }
+      },
+      {
+        "jobType": "production-corp-us-east-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034428590
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034114538
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034114538
+        }
+      },
+      {
+        "jobType": "production-ap-southeast-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034265146
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034097617
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034097617
+        }
+      },
+      {
+        "jobType": "production-us-central-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493033800484
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034273753
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034273753
+        }
+      },
+      {
+        "jobType": "staging-test",
+        "lastTriggered": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503900683154
+        },
+        "lastCompleted": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503901635745
+        },
+        "lastSuccess": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503901635745
+        }
+      },
+      {
+        "jobType": "system-test",
+        "lastTriggered": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503899621243
+        },
+        "lastCompleted": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503900025214
+        },
+        "lastSuccess": {
+          "version": "6.141.117",
+          "revision": {
+            "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273",
+            "sourceRevision": {
+              "repositoryField": "git@git.test:vespa/canary-application.git",
+              "branchField": "origin/canary-cd",
+              "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633"
+            }
+          },
+          "at": 1503900025214
+        }
+      },
+      {
+        "jobType": "production-us-west-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034273768
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034019015
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034019015
+        }
+      },
+      {
+        "jobType": "production-ap-northeast-1",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493033995045
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034257206
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034257206
+        }
+      },
+      {
+        "jobType": "production-ap-northeast-2",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034257222
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493034265048
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493034265048
+        }
+      },
+      {
+        "jobType": "production-us-east-3",
+        "lastTriggered": {
+          "version": "6.98.12",
+          "at": 1493034114555
+        },
+        "lastCompleted": {
+          "version": "6.98.12",
+          "at": 1493033800469
+        },
+        "lastSuccess": {
+          "version": "6.98.12",
+          "at": 1493033800469
+        }
+      }
+    ],
+    "selfTriggering": false
+  },
+  "outstandingChangeField": false
+}