Stores all deployed app versions. Stores app/bundle hashes

11 files changed, 155 insertions, 33 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
index 3a863f21ca0..984035645ed 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
@@ -23,7 +23,8 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
      */
     public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty(),
                                                                             Optional.empty(), Optional.empty(), Optional.empty(),
-                                                                            Optional.empty(), Optional.empty(), true);
+                                                                            Optional.empty(), Optional.empty(), true,
+                                                                            Optional.empty(), Optional.empty());
 
     // This never changes and is only used to create a valid semantic version number, as required by application bundles
     private static final String majorVersion = "1.0";
@@ -36,11 +37,13 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
     private final Optional<String> sourceUrl;
     private final Optional<String> commit;
     private final boolean deployedDirectly;
+    private final Optional<String> applicationPackageHash;
+    private final Optional<String> bundleHash;
 
     /** Public for serialisation only. */
     public ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail,
                               Optional<Version> compileVersion, Optional<Instant> buildTime, Optional<String> sourceUrl,
-                              Optional<String> commit, boolean deployedDirectly) {
+                              Optional<String> commit, boolean deployedDirectly, Optional<String> applicationPackageHash, Optional<String> bundleHash) {
         if (buildNumber.isEmpty() && (   source.isPresent() || authorEmail.isPresent() || compileVersion.isPresent()
                                       || buildTime.isPresent() || sourceUrl.isPresent() || commit.isPresent()))
             throw new IllegalArgumentException("Build number must be present if any other attribute is");
@@ -65,26 +68,31 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
         this.sourceUrl = Objects.requireNonNull(sourceUrl, "sourceUrl cannot be null");
         this.commit = Objects.requireNonNull(commit, "commit cannot be null");
         this.deployedDirectly = deployedDirectly;
+        this.applicationPackageHash = applicationPackageHash;
+        this.bundleHash = bundleHash;
     }
 
     /** Create an application package version from a completed build, without an author email */
     public static ApplicationVersion from(SourceRevision source, long buildNumber) {
         return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.empty(),
-                                      Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false);
+                                      Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false,
+                                      Optional.empty(), Optional.empty());
     }
 
     /** Creates a version from a completed build, an author email, and build meta data. */
     public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail,
                                           Version compileVersion, Instant buildTime) {
         return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail),
-                                      Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty(), false);
+                                      Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty(), false,
+                                      Optional.empty(), Optional.empty());
     }
 
     /** Creates a version from a completed build, an author email, and build meta data. */
     public static ApplicationVersion from(Optional<SourceRevision> source, long buildNumber, Optional<String> authorEmail,
                                           Optional<Version> compileVersion, Optional<Instant> buildTime,
-                                          Optional<String> sourceUrl, Optional<String> commit, boolean deployedDirectly) {
-        return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
+                                          Optional<String> sourceUrl, Optional<String> commit, boolean deployedDirectly,
+                                          Optional<String> applicationPackageHash, Optional<String> bundleHash ) {
+        return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, applicationPackageHash, bundleHash);
     }
 
     /** Returns a unique identifier for this version or "unknown" if version is not known */
@@ -115,6 +123,16 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
     /** Returns the time this package was built, if known. */
     public Optional<Instant> buildTime() { return buildTime; }
 
+    /** Returns the hash of entire application package */
+    public Optional<String> applicationPackageHash() {
+        return applicationPackageHash;
+    }
+
+    /** Returns the hash of app package except deployment/build-meta data */
+    public Optional<String> bundleHash() {
+        return bundleHash;
+    }
+
     /** Returns the source URL for this application version. */
     public Optional<String> sourceUrl() {
         return sourceUrl.or(() -> source.map(source -> {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index bbf76fcb480..d97b1b6547d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -47,6 +47,7 @@ public class Application {
     private final DeploymentSpec deploymentSpec;
     private final ValidationOverrides validationOverrides;
     private final Optional<ApplicationVersion> latestVersion;
+    private final List<ApplicationVersion> versions;
     private final OptionalLong projectId;
     private final Optional<IssueId> deploymentIssueId;
     private final Optional<IssueId> ownershipIssueId;
@@ -60,14 +61,14 @@ public class Application {
     public Application(TenantAndApplicationId id, Instant now) {
         this(id, now, DeploymentSpec.empty, ValidationOverrides.empty,
              Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
-             new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), List.of());
+             new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), List.of(), List.of());
     }
 
     // DO NOT USE! For serialization purposes, only.
     public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                        OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId,
-                       Optional<ApplicationVersion> latestVersion, Collection<Instance> instances) {
+                       Optional<ApplicationVersion> latestVersion, List<ApplicationVersion> versions, Collection<Instance> instances) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
         this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
@@ -80,6 +81,7 @@ public class Application {
         this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null");
         this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null");
         this.latestVersion = requireNotUnknown(latestVersion);
+        this.versions = versions;
         this.instances = instances.stream().collect(
                 Collectors.collectingAndThen(Collectors.toMap(Instance::name,
                                                               Function.identity(),
@@ -104,9 +106,15 @@ public class Application {
     /** Returns the project id of this application, if it has any. */
     public OptionalLong projectId() { return projectId; }
 
-    /** Returns the last submitted version of this application. */
+    /** Returns the last submitted version of this application.
+     * TODO: Replace with latest in {@link #versions }*/
     public Optional<ApplicationVersion> latestVersion() { return latestVersion; }
 
+    /** Returns the currently deployed versions of the application */
+    public List<ApplicationVersion> versions() {
+        return versions;
+    }
+
     /**
      * Returns the last deployed validation overrides of this application,
      * or the empty validation overrides if it has never been deployed
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 53792440f58..f0c85e7eeb8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -445,6 +445,19 @@ public class ApplicationController {
                 controller.notificationsDb().removeNotifications(notification.source());
         }
 
+        var existingVersions = application.get()
+                .instances()
+                .values()
+                .stream()
+                .flatMap(instance -> instance.deployments().values().stream())
+                .map(Deployment::applicationVersion)
+                .collect(Collectors.toSet());
+
+        // Remove any version not deployed anywhere
+        for (ApplicationVersion version : application.get().versions()) {
+            if (!existingVersions.contains(version)) application = application.withoutVersion(version);
+        }
+
         store(application);
         return application;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 13ef1339e44..f476ae649f6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -13,8 +13,10 @@ import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 
 import java.security.PublicKey;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -43,6 +45,7 @@ public class LockedApplication {
     private final Set<PublicKey> deployKeys;
     private final OptionalLong projectId;
     private final Optional<ApplicationVersion> latestVersion;
+    private final List<ApplicationVersion> versions;
     private final Map<InstanceName, Instance> instances;
 
     /**
@@ -56,14 +59,14 @@ public class LockedApplication {
              application.deploymentSpec(), application.validationOverrides(),
              application.deploymentIssueId(), application.ownershipIssueId(),
              application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(),
-             application.projectId(), application.latestVersion(), application.instances());
+             application.projectId(), application.latestVersion(), application.versions(), application.instances());
     }
 
     private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec,
                               ValidationOverrides validationOverrides,
                               Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                               OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
-                              OptionalLong projectId, Optional<ApplicationVersion> latestVersion,
+                              OptionalLong projectId, Optional<ApplicationVersion> latestVersion, List<ApplicationVersion> versions,
                               Map<InstanceName, Instance> instances) {
         this.lock = lock;
         this.id = id;
@@ -78,6 +81,7 @@ public class LockedApplication {
         this.deployKeys = deployKeys;
         this.projectId = projectId;
         this.latestVersion = latestVersion;
+        this.versions = versions;
         this.instances = Map.copyOf(instances);
     }
 
@@ -85,7 +89,7 @@ public class LockedApplication {
     public Application get() {
         return new Application(id, createdAt, deploymentSpec, validationOverrides,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                               projectId, latestVersion, instances.values());
+                               projectId, latestVersion,  versions, instances.values());
     }
 
     LockedApplication withNewInstance(InstanceName instance) {
@@ -93,7 +97,7 @@ public class LockedApplication {
         instances.put(instance, new Instance(id.instance(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) {
@@ -101,7 +105,7 @@ public class LockedApplication {
         instances.put(instance, modification.apply(instances.get(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication without(InstanceName instance) {
@@ -109,49 +113,51 @@ public class LockedApplication {
         instances.remove(instance);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withNewSubmission(ApplicationVersion latestVersion) {
+        List<ApplicationVersion> applicationVersions = new ArrayList<>(versions);
+        if (!applicationVersions.contains(latestVersion)) applicationVersions.add(latestVersion);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, Optional.of(latestVersion), instances);
+                                     projectId, Optional.of(latestVersion), applicationVersions, instances);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withOwner(User owner) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     /** Set a major version for this, or set to null to remove any major version override */
@@ -159,13 +165,13 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner,
                                      majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion),
-                                     metrics, deployKeys, projectId, latestVersion, instances);
+                                     metrics, deployKeys, projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withDeployKey(PublicKey pemDeployKey) {
@@ -173,7 +179,7 @@ public class LockedApplication {
         keys.add(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withoutDeployKey(PublicKey pemDeployKey) {
@@ -181,7 +187,15 @@ public class LockedApplication {
         keys.remove(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
+    }
+
+    public LockedApplication withoutVersion(ApplicationVersion version) {
+        List<ApplicationVersion> applicationVersions = new ArrayList<>(versions);
+        applicationVersions.remove(version);
+        return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
+                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
+                projectId, latestVersion, applicationVersions, instances);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index b088c2fd0fd..2ffe7e3ddee 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -37,8 +37,11 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.toMap;
@@ -62,6 +65,7 @@ public class ApplicationPackage {
     private static final String servicesFile = "services.xml";
 
     private final String contentHash;
+    private final String bundleHash; // Hash of all files except deployment.xml and build-meta
     private final byte[] zippedContent;
     private final DeploymentSpec deploymentSpec;
     private final ValidationOverrides validationOverrides;
@@ -102,6 +106,8 @@ public class ApplicationPackage {
         this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong())));
 
         this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of());
+
+        this.bundleHash = calculateBundleHash();
     }
 
     /** Returns a copy of this with the given certificate appended. */
@@ -117,6 +123,10 @@ public class ApplicationPackage {
 
     /** Returns a hash of the content of this package */
     public String hash() { return contentHash; }
+
+    public String bundleHash() {
+        return bundleHash;
+    }
     
     /** Returns the content of this package. The content <b>must not</b> be modified. */
     public byte[] zippedContent() { return zippedContent; }
@@ -209,6 +219,13 @@ public class ApplicationPackage {
         return ValidationOverrides.fromXml(validationOverridesContents.toString());
     }
 
+    // Hashes all files that require a deployment to be forwarded to configservers
+    private String calculateBundleHash() {
+        Predicate<String> entryMatcher = name -> !name.contains(deploymentFile) && !name.contains(buildMetaFile);
+        SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher);
+        return Hashing.sha1().hashInt(entryCRCs.hashCode()).toString();
+    }
+
 
     /** Maps normalized paths to cached content read from a zip archive. */
     private static class ZipArchiveCache {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
index f89cc0cc0fc..7ac51a32a93 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
@@ -9,8 +9,12 @@ import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.function.Predicate;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -59,6 +63,24 @@ public class ZipStreamReader {
         }
     }
 
+    public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) {
+        SortedMap<String, Long> entryCRCs = new TreeMap<>();
+        byte[] buffer = new byte[2048];
+        try (ZipInputStream zipIn = new ZipInputStream(in);
+             OutputStream os = new ByteArrayOutputStream()) {
+            for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) {
+                if (!entryNameMatcher.test(entry.getName()))
+                    continue;
+                // CRC is not set until entry is read
+                while ( -1 != zipIn.read(buffer)){}
+                entryCRCs.put(entry.getName(), entry.getCrc());
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        return entryCRCs;
+    }
+
     private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) {
         try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
             byte[] buffer = new byte[2048];
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 1601af2612b..fb722ab7193 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -439,7 +439,9 @@ public class JobController {
                                                 applicationPackage.buildTime(),
                                                 sourceUrl,
                                                 revision.map(SourceRevision::commit),
-                                                false));
+                                                false,
+                                                Optional.of(applicationPackage.hash()),
+                                                Optional.of(applicationPackage.bundleHash())));
             byte[] diff = application.get().latestVersion()
                     .map(v -> v.buildNumber().getAsLong())
                     .flatMap(prevBuild -> controller.applications().applicationStore().find(id.tenant(), id.application(), prevBuild))
@@ -512,7 +514,7 @@ public class JobController {
 
         long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L);
         ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(),
-                Optional.empty(), Optional.empty(), Optional.empty(), true);
+                Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty(), Optional.empty());
 
         byte[] diff = lastRun.map(run -> run.versions().targetApplication())
                 .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index a7413273a38..8e117876b5a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -75,6 +75,7 @@ public class ApplicationSerializer {
     private static final String deployingField = "deployingField";
     private static final String projectIdField = "projectId";
     private static final String latestVersionField = "latestVersion";
+    private static final String versionsField = "versions";
     private static final String pinnedField = "pinned";
     private static final String deploymentIssueField = "deploymentIssueId";
     private static final String ownershipIssueIdField = "ownershipIssueId";
@@ -112,6 +113,8 @@ public class ApplicationSerializer {
     private static final String compileVersionField = "compileVersion";
     private static final String buildTimeField = "buildTime";
     private static final String sourceUrlField = "sourceUrl";
+    private static final String applicationHashField = "applicationHash";
+    private static final String bundleHashField = "bundleHash";
     private static final String lastQueriedField = "lastQueried";
     private static final String lastWrittenField = "lastWritten";
     private static final String lastQueriesPerSecondField = "lastQueriesPerSecond";
@@ -164,6 +167,7 @@ public class ApplicationSerializer {
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
         deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField));
         application.latestVersion().ifPresent(version -> toSlime(version, root.setObject(latestVersionField)));
+        versionsToSlime(application.versions(), root.setArray(versionsField));
         instancesToSlime(application, root.setArray(instancesField));
         return slime;
     }
@@ -223,6 +227,10 @@ public class ApplicationSerializer {
         object.setString(regionField, zone.region().value());
     }
 
+    private void versionsToSlime(List<ApplicationVersion> versions, Cursor object) {
+        versions.forEach(version -> toSlime(version, object.addObject()));
+    }
+
     private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
         applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number));
         applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField)));
@@ -312,10 +320,11 @@ public class ApplicationSerializer {
         List<Instance> instances = instancesFromSlime(id, root.field(instancesField));
         OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField));
         Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField));
+        List<ApplicationVersion> versions = versionsFromSlime(root.field(versionsField));
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics,
-                               deployKeys, projectId, latestVersion, instances);
+                               deployKeys, projectId, latestVersion, versions, instances);
     }
 
     private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) {
@@ -323,6 +332,12 @@ public class ApplicationSerializer {
                        .filter(version -> ! version.isUnknown());
     }
 
+    private List<ApplicationVersion> versionsFromSlime(Inspector versionsObject) {
+        List<ApplicationVersion> versions = new ArrayList<>();
+        versionsObject.traverse((ArrayTraverser) (name, object) -> versions.add(applicationVersionFromSlime(object)));
+        return versions;
+    }
+
     private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) {
         List<Instance> instances = new ArrayList<>();
         field.traverse((ArrayTraverser) (name, object) -> {
@@ -428,8 +443,10 @@ public class ApplicationSerializer {
         Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField));
         Optional<String> commit = SlimeUtils.optionalString(object.field(commitField));
         boolean deployedDirectly = object.field(deployedDirectlyField).asBool();
+        Optional<String> applicationPackageHash = SlimeUtils.optionalString(object.field(applicationHashField));
+        Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField));
 
-        return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
+        return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, applicationPackageHash, bundleHash);
     }
 
     private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index c93b77e9af1..657625c812c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -96,6 +96,8 @@ class RunSerializer {
     private static final String sourceUrlField = "sourceUrl";
     private static final String buildField = "build";
     private static final String sourceField = "source";
+    private static final String applicationHashField = "applicationHash";
+    private static final String bundleHashField = "bundleHash";
     private static final String lastTestRecordField = "lastTestRecord";
     private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp";
     private static final String noNodesDownSinceField = "noNodesDownSince";
@@ -179,9 +181,11 @@ class RunSerializer {
         Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField));
         Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField));
         boolean deployedDirectly = versionObject.field(deployedDirectlyField).asBool();
+        Optional<String> applicationPackageHash = SlimeUtils.optionalString(versionObject.field(applicationHashField));
+        Optional<String> bundleHash = SlimeUtils.optionalString(versionObject.field(bundleHashField));
 
         return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail,
-                                      compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
+                                      compileVersion, buildTime, sourceUrl, commit, deployedDirectly, applicationPackageHash, bundleHash);
     }
 
     // Don't change this — introduce a separate array instead.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 86e5acf07a0..f2b38f57bff 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -89,7 +89,10 @@ public class ApplicationSerializerTest {
                                                                         Optional.of(Instant.ofEpochMilli(666)),
                                                                         Optional.empty(),
                                                                         Optional.of("best commit"),
-                                                                        true);
+                                                                        true,
+                                                                        Optional.of("hash1"),
+                                                                        Optional.of("hash2"));
+        List<ApplicationVersion> versions = List.of(applicationVersion1);
         assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get());
 
         ApplicationVersion applicationVersion2 = ApplicationVersion
@@ -142,6 +145,7 @@ public class ApplicationSerializerTest {
                                                Set.of(publicKey, otherPublicKey),
                                                projectId,
                                                Optional.of(applicationVersion1),
+                                               versions,
                                                instances);
 
         Application serialized = APPLICATION_SERIALIZER.fromSlime(SlimeUtils.toJsonBytes(APPLICATION_SERIALIZER.toSlime(original)));
@@ -153,6 +157,7 @@ public class ApplicationSerializerTest {
         assertEquals(original.latestVersion().get().buildTime(), serialized.latestVersion().get().buildTime());
         assertEquals(original.latestVersion().get().sourceUrl(), serialized.latestVersion().get().sourceUrl());
         assertEquals(original.latestVersion().get().commit(), serialized.latestVersion().get().commit());
+        assertEquals(original.versions(), serialized.versions());
 
         assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
         assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index eca2b17a5f1..061b5e71396 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -91,7 +91,9 @@ public class RunSerializerTest {
                                                                         Optional.of(Instant.ofEpochMilli(100)),
                                                                         Optional.empty(),
                                                                         Optional.empty(),
-                                                                        true);
+                                                                        true,
+                                                                        Optional.empty(),
+                                                                        Optional.empty());
         assertEquals(applicationVersion, run.versions().targetApplication());
         assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail());
         assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime());