Merge pull request #21360 from vespa-engine/jonmv/long-deployment-pipelines-2

Jonmv/long deployment pipelines 2

5 files changed, 39 insertions, 8 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index 34e2f1350f0..8e0c029aeab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -237,6 +237,12 @@ public class ApplicationPackage {
                       .hash().toString();
     }
 
+    public static String calculateHash(byte[] bytes) {
+        return Hashing.sha1().newHasher()
+                      .putBytes(bytes)
+                      .hash().toString();
+    }
+
 
     /** Maps normalized paths to cached content read from a zip archive. */
     private static class ZipArchiveCache {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index 0f9be643cdf..3f12cce9aae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -196,7 +196,7 @@ public class DeploymentStatus {
         jobs.putAll(productionJobs);
         // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions.
         jobSteps.forEach((job, step) -> {
-            if ( ! step.isDeclared() || jobs.containsKey(job))
+            if ( ! step.isDeclared() || step.type() != StepType.test || jobs.containsKey(job))
                 return;
 
             Change change = changes.get(job.application().instance());
@@ -813,6 +813,15 @@ public class DeploymentStatus {
             JobStatus job = status.instanceJobs(instance).get(testType);
             return new JobStepStatus(StepType.test, step, dependencies, job, status) {
                 @Override
+                Optional<Instant> readyAt(Change change, Optional<JobId> dependent) {
+                    JobId prodId = new JobId(status.application().id().instance(instance()), prodType);
+                    Optional<Instant> readyAt = super.readyAt(change, dependent);
+                    Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId));
+                    if (readyAt.isEmpty() || deployedAt.isEmpty()) return Optional.empty();
+                    return readyAt.get().isAfter(deployedAt.get()) ? readyAt : deployedAt;
+                }
+
+                @Override
                 Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
                     Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion);
                     return dependent.equals(job()) ? job.lastSuccess()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 60c41ef520d..60339a3233c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -28,9 +28,11 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff;
+import com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
+import java.io.ByteArrayInputStream;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
@@ -452,13 +454,14 @@ public class JobController {
             Optional<ApplicationPackage> previousPackage = previousVersion.flatMap(previous -> controller.applications().applicationStore().find(id.tenant(), id.application(), previous.buildNumber().getAsLong()))
                                                                           .map(ApplicationPackage::new);
             long previousBuild = previousVersion.map(latestVersion -> latestVersion.buildNumber().getAsLong()).orElse(0L);
+            String packageHash = applicationPackage.bundleHash() + ApplicationPackage.calculateHash(testPackageBytes);
             version.set(ApplicationVersion.from(revision, 1 + previousBuild, authorEmail,
                                                 applicationPackage.compileVersion(),
                                                 applicationPackage.buildTime(),
                                                 sourceUrl,
                                                 revision.map(SourceRevision::commit),
                                                 false,
-                                                Optional.of(applicationPackage.bundleHash())));
+                                                Optional.of(packageHash)));
 
             byte[] diff = previousPackage.map(previous -> ApplicationPackageDiff.diff(previous, applicationPackage))
                                          .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 83397768264..08cd88cf23f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -271,17 +271,13 @@ public class DeploymentContext {
 
     /** Submit given application package for deployment */
     public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision, long salt) {
-        var buffer = new ByteArrayOutputStream();
-        ZipStreamReader.transferAndWrite(buffer,
-                                         new ByteArrayInputStream(applicationPackage.zippedContent()),
-                                         "salt",
-                                         new byte[]{ (byte) (salt >> 56), (byte) (salt >> 48), (byte) (salt >> 40), (byte) (salt >> 32), (byte) (salt >> 24), (byte) (salt >> 16), (byte) (salt >> 8), (byte) salt });
         var projectId = tester.controller().applications()
                               .requireApplication(applicationId)
                               .projectId()
                               .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set.
+        var testerpackage = new byte[]{ (byte) (salt >> 56), (byte) (salt >> 48), (byte) (salt >> 40), (byte) (salt >> 32), (byte) (salt >> 24), (byte) (salt >> 16), (byte) (salt >> 8), (byte) salt };
         lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(),
-                                     projectId, new ApplicationPackage(buffer.toByteArray()), new byte[0]);
+                                     projectId, applicationPackage, testerpackage);
         return this;
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index d1b76dd96be..bfedd325ae7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -1101,6 +1101,23 @@ public class DeploymentTriggerTest {
         tester.clock().advance(Duration.ofMinutes(11)); // Job is cooling down after consecutive failures.
         app.runJob(testUsEast3);
         assertEquals(Change.empty().withPin(), app.instance().change());
+
+        // Same upgrade is attempted, and production tests wait for redeployment.
+        tester.deploymentTrigger().cancelChange(app.instanceId(), ALL);
+        tester.upgrader().overrideConfidence(version1, VespaVersion.Confidence.high);
+        tester.controllerTester().computeVersionStatus();
+        tester.upgrader().maintain();
+
+        app.triggerJobs();
+        app.assertRunning(productionUsEast3);
+        app.assertNotRunning(testUsEast3);
+        app.runJob(productionUsEast3);
+        tester.clock().advance(Duration.ofMinutes(1));
+        app.runJob(testUsEast3).runJob(productionUsWest1).triggerJobs();
+        app.assertRunning(productionUsCentral1);
+        tester.runner().run();
+        app.triggerJobs();
+        app.assertNotRunning(testUsCentral1);
     }
 
     @Test