Merge pull request #20948 from vespa-engine/olaa/store-application-hashes

Stores all deployed app versions. Stores app/bundle hashes

14 files changed, 176 insertions, 27 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index bbf76fcb480..bb7863a7b69 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -29,7 +29,9 @@ import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -47,6 +49,7 @@ public class Application {
     private final DeploymentSpec deploymentSpec;
     private final ValidationOverrides validationOverrides;
     private final Optional<ApplicationVersion> latestVersion;
+    private final SortedSet<ApplicationVersion> versions;
     private final OptionalLong projectId;
     private final Optional<IssueId> deploymentIssueId;
     private final Optional<IssueId> ownershipIssueId;
@@ -60,14 +63,14 @@ public class Application {
     public Application(TenantAndApplicationId id, Instant now) {
         this(id, now, DeploymentSpec.empty, ValidationOverrides.empty,
              Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
-             new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), List.of());
+             new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), new TreeSet<>(), List.of());
     }
 
     // DO NOT USE! For serialization purposes, only.
     public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                        OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId,
-                       Optional<ApplicationVersion> latestVersion, Collection<Instance> instances) {
+                       Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions, Collection<Instance> instances) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
         this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
@@ -80,6 +83,7 @@ public class Application {
         this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null");
         this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null");
         this.latestVersion = requireNotUnknown(latestVersion);
+        this.versions = versions;
         this.instances = instances.stream().collect(
                 Collectors.collectingAndThen(Collectors.toMap(Instance::name,
                                                               Function.identity(),
@@ -104,9 +108,15 @@ public class Application {
     /** Returns the project id of this application, if it has any. */
     public OptionalLong projectId() { return projectId; }
 
-    /** Returns the last submitted version of this application. */
+    /** Returns the last submitted version of this application.
+     * TODO: Replace with latest in {@link #versions }*/
     public Optional<ApplicationVersion> latestVersion() { return latestVersion; }
 
+    /** Returns the currently deployed versions of the application */
+    public SortedSet<ApplicationVersion> versions() {
+        return versions;
+    }
+
     /**
      * Returns the last deployed validation overrides of this application,
      * or the empty validation overrides if it has never been deployed
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 53792440f58..7156e4ac937 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -445,6 +445,27 @@ public class ApplicationController {
                 controller.notificationsDb().removeNotifications(notification.source());
         }
 
+        var oldestDeployedVersion = application.get()
+                .instances()
+                .values()
+                .stream()
+                .flatMap(instance -> instance.deployments().values().stream())
+                .map(Deployment::applicationVersion)
+                .sorted()
+                .findFirst()
+                .orElse(ApplicationVersion.unknown);
+
+        var olderVersions = application.get().versions()
+                .stream()
+                .filter(version -> version.compareTo(oldestDeployedVersion) < 0)
+                .sorted()
+                .collect(Collectors.toList());
+
+        // Remove any version not deployed anywhere - but keep one
+        for (int i = 0; i < olderVersions.size() - 1; i++) {
+            application = application.withoutVersion(olderVersions.get(i));
+        }
+
         store(application);
         return application;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 13ef1339e44..06ff381e4dc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -21,6 +21,8 @@ import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.function.UnaryOperator;
 
 /**
@@ -43,6 +45,7 @@ public class LockedApplication {
     private final Set<PublicKey> deployKeys;
     private final OptionalLong projectId;
     private final Optional<ApplicationVersion> latestVersion;
+    private final SortedSet<ApplicationVersion> versions;
     private final Map<InstanceName, Instance> instances;
 
     /**
@@ -56,14 +59,14 @@ public class LockedApplication {
              application.deploymentSpec(), application.validationOverrides(),
              application.deploymentIssueId(), application.ownershipIssueId(),
              application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(),
-             application.projectId(), application.latestVersion(), application.instances());
+             application.projectId(), application.latestVersion(), application.versions(), application.instances());
     }
 
     private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec,
                               ValidationOverrides validationOverrides,
                               Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                               OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys,
-                              OptionalLong projectId, Optional<ApplicationVersion> latestVersion,
+                              OptionalLong projectId, Optional<ApplicationVersion> latestVersion, SortedSet<ApplicationVersion> versions,
                               Map<InstanceName, Instance> instances) {
         this.lock = lock;
         this.id = id;
@@ -78,6 +81,7 @@ public class LockedApplication {
         this.deployKeys = deployKeys;
         this.projectId = projectId;
         this.latestVersion = latestVersion;
+        this.versions = versions;
         this.instances = Map.copyOf(instances);
     }
 
@@ -85,7 +89,7 @@ public class LockedApplication {
     public Application get() {
         return new Application(id, createdAt, deploymentSpec, validationOverrides,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                               projectId, latestVersion, instances.values());
+                               projectId, latestVersion,  versions, instances.values());
     }
 
     LockedApplication withNewInstance(InstanceName instance) {
@@ -93,7 +97,7 @@ public class LockedApplication {
         instances.put(instance, new Instance(id.instance(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) {
@@ -101,7 +105,7 @@ public class LockedApplication {
         instances.put(instance, modification.apply(instances.get(instance)));
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication without(InstanceName instance) {
@@ -109,49 +113,51 @@ public class LockedApplication {
         instances.remove(instance);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withNewSubmission(ApplicationVersion latestVersion) {
+        SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions);
+        applicationVersions.add(latestVersion);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, Optional.of(latestVersion), instances);
+                                     projectId, Optional.of(latestVersion), applicationVersions, instances);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withOwner(User owner) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     /** Set a major version for this, or set to null to remove any major version override */
@@ -159,13 +165,13 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner,
                                      majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion),
-                                     metrics, deployKeys, projectId, latestVersion, instances);
+                                     metrics, deployKeys, projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication with(ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withDeployKey(PublicKey pemDeployKey) {
@@ -173,7 +179,7 @@ public class LockedApplication {
         keys.add(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
     }
 
     public LockedApplication withoutDeployKey(PublicKey pemDeployKey) {
@@ -181,7 +187,15 @@ public class LockedApplication {
         keys.remove(pemDeployKey);
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
                                      deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys,
-                                     projectId, latestVersion, instances);
+                                     projectId, latestVersion, versions, instances);
+    }
+
+    public LockedApplication withoutVersion(ApplicationVersion version) {
+        SortedSet<ApplicationVersion> applicationVersions = new TreeSet<>(versions);
+        applicationVersions.remove(version);
+        return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides,
+                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys,
+                projectId, latestVersion, applicationVersions, instances);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index b088c2fd0fd..da62175089d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application.pkg;
 
+import com.google.common.hash.Funnel;
 import com.google.common.hash.Hashing;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.FileSystemWrapper;
@@ -37,8 +38,11 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.toMap;
@@ -62,6 +66,7 @@ public class ApplicationPackage {
     private static final String servicesFile = "services.xml";
 
     private final String contentHash;
+    private final String bundleHash; // Hash of all files except deployment.xml and build-meta
     private final byte[] zippedContent;
     private final DeploymentSpec deploymentSpec;
     private final ValidationOverrides validationOverrides;
@@ -102,6 +107,8 @@ public class ApplicationPackage {
         this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong())));
 
         this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of());
+
+        this.bundleHash = calculateBundleHash();
     }
 
     /** Returns a copy of this with the given certificate appended. */
@@ -117,6 +124,10 @@ public class ApplicationPackage {
 
     /** Returns a hash of the content of this package */
     public String hash() { return contentHash; }
+
+    public String bundleHash() {
+        return bundleHash;
+    }
     
     /** Returns the content of this package. The content <b>must not</b> be modified. */
     public byte[] zippedContent() { return zippedContent; }
@@ -209,6 +220,17 @@ public class ApplicationPackage {
         return ValidationOverrides.fromXml(validationOverridesContents.toString());
     }
 
+    // Hashes all files that require a deployment to be forwarded to configservers
+    private String calculateBundleHash() {
+        Predicate<String> entryMatcher = name -> !name.endsWith(deploymentFile) && !name.endsWith(buildMetaFile);
+        SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher);
+        Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.entrySet().forEach(entry -> {
+            into.putBytes(entry.getKey().getBytes());
+            into.putLong(entry.getValue());
+        });
+        return Hashing.sha1().hashObject(entryCRCs, funnel).toString();
+    }
+
 
     /** Maps normalized paths to cached content read from a zip archive. */
     private static class ZipArchiveCache {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
index f89cc0cc0fc..7ac51a32a93 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java
@@ -9,8 +9,12 @@ import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.function.Predicate;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -59,6 +63,24 @@ public class ZipStreamReader {
         }
     }
 
+    public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) {
+        SortedMap<String, Long> entryCRCs = new TreeMap<>();
+        byte[] buffer = new byte[2048];
+        try (ZipInputStream zipIn = new ZipInputStream(in);
+             OutputStream os = new ByteArrayOutputStream()) {
+            for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) {
+                if (!entryNameMatcher.test(entry.getName()))
+                    continue;
+                // CRC is not set until entry is read
+                while ( -1 != zipIn.read(buffer)){}
+                entryCRCs.put(entry.getName(), entry.getCrc());
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        return entryCRCs;
+    }
+
     private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) {
         try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
             byte[] buffer = new byte[2048];
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 1601af2612b..1051df2cd1c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -439,7 +439,8 @@ public class JobController {
                                                 applicationPackage.buildTime(),
                                                 sourceUrl,
                                                 revision.map(SourceRevision::commit),
-                                                false));
+                                                false,
+                                                Optional.of(applicationPackage.bundleHash())));
             byte[] diff = application.get().latestVersion()
                     .map(v -> v.buildNumber().getAsLong())
                     .flatMap(prevBuild -> controller.applications().applicationStore().find(id.tenant(), id.application(), prevBuild))
@@ -512,7 +513,7 @@ public class JobController {
 
         long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L);
         ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(),
-                Optional.empty(), Optional.empty(), Optional.empty(), true);
+                Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty());
 
         byte[] diff = lastRun.map(run -> run.versions().targetApplication())
                 .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index a7413273a38..30b6920fd79 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -49,6 +49,8 @@ import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 /**
  * Serializes {@link Application}s to/from slime.
@@ -75,6 +77,7 @@ public class ApplicationSerializer {
     private static final String deployingField = "deployingField";
     private static final String projectIdField = "projectId";
     private static final String latestVersionField = "latestVersion";
+    private static final String versionsField = "versions";
     private static final String pinnedField = "pinned";
     private static final String deploymentIssueField = "deploymentIssueId";
     private static final String ownershipIssueIdField = "ownershipIssueId";
@@ -112,6 +115,7 @@ public class ApplicationSerializer {
     private static final String compileVersionField = "compileVersion";
     private static final String buildTimeField = "buildTime";
     private static final String sourceUrlField = "sourceUrl";
+    private static final String bundleHashField = "bundleHash";
     private static final String lastQueriedField = "lastQueried";
     private static final String lastWrittenField = "lastWritten";
     private static final String lastQueriesPerSecondField = "lastQueriesPerSecond";
@@ -164,6 +168,7 @@ public class ApplicationSerializer {
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
         deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField));
         application.latestVersion().ifPresent(version -> toSlime(version, root.setObject(latestVersionField)));
+        versionsToSlime(application, root.setArray(versionsField));
         instancesToSlime(application, root.setArray(instancesField));
         return slime;
     }
@@ -223,6 +228,18 @@ public class ApplicationSerializer {
         object.setString(regionField, zone.region().value());
     }
 
+    private void versionsToSlime(Application application, Cursor object) {
+        // Add all deployed versions as well
+        application.instances()
+                .values()
+                .stream()
+                .flatMap(instance -> instance.deployments().values().stream())
+                .map(Deployment::applicationVersion)
+                .forEach(application.versions()::add);
+
+        application.versions().forEach(version -> toSlime(version, object.addObject()));
+    }
+
     private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
         applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number));
         applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField)));
@@ -312,10 +329,11 @@ public class ApplicationSerializer {
         List<Instance> instances = instancesFromSlime(id, root.field(instancesField));
         OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField));
         Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField));
+        SortedSet<ApplicationVersion> versions = versionsFromSlime(root.field(versionsField));
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides,
                                deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics,
-                               deployKeys, projectId, latestVersion, instances);
+                               deployKeys, projectId, latestVersion, versions, instances);
     }
 
     private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) {
@@ -323,6 +341,12 @@ public class ApplicationSerializer {
                        .filter(version -> ! version.isUnknown());
     }
 
+    private SortedSet<ApplicationVersion> versionsFromSlime(Inspector versionsObject) {
+        SortedSet<ApplicationVersion> versions = new TreeSet<>();
+        versionsObject.traverse((ArrayTraverser) (name, object) -> versions.add(applicationVersionFromSlime(object)));
+        return versions;
+    }
+
     private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) {
         List<Instance> instances = new ArrayList<>();
         field.traverse((ArrayTraverser) (name, object) -> {
@@ -428,8 +452,9 @@ public class ApplicationSerializer {
         Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField));
         Optional<String> commit = SlimeUtils.optionalString(object.field(commitField));
         boolean deployedDirectly = object.field(deployedDirectlyField).asBool();
+        Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField));
 
-        return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
+        return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash);
     }
 
     private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index c93b77e9af1..cd55b689c3c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -96,6 +96,7 @@ class RunSerializer {
     private static final String sourceUrlField = "sourceUrl";
     private static final String buildField = "build";
     private static final String sourceField = "source";
+    private static final String bundleHashField = "bundleHash";
     private static final String lastTestRecordField = "lastTestRecord";
     private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp";
     private static final String noNodesDownSinceField = "noNodesDownSince";
@@ -179,9 +180,10 @@ class RunSerializer {
         Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField));
         Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField));
         boolean deployedDirectly = versionObject.field(deployedDirectlyField).asBool();
+        Optional<String> bundleHash = SlimeUtils.optionalString(versionObject.field(bundleHashField));
 
         return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail,
-                                      compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
+                                      compileVersion, buildTime, sourceUrl, commit, deployedDirectly, bundleHash);
     }
 
     // Don't change this — introduce a separate array instead.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
index 2ee03457046..cf0b46dfba2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
@@ -7,6 +7,8 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
@@ -14,6 +16,7 @@ import java.util.stream.Collectors;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -108,6 +111,21 @@ public class ApplicationPackageTest {
         }
     }
 
+    @Test
+    public void testBundleHashesAreSameWithDifferentDeploymentXml() throws Exception {
+        var originalPackage = getApplicationZip("original.zip");
+        var changedDeploymentXml = getApplicationZip("changed-deployment-xml.zip");
+        var changedServices = getApplicationZip("changed-services-xml.zip");
+
+        // services.xml is changed -> different bundle hash
+        assertNotEquals(originalPackage.bundleHash(), changedServices.bundleHash());
+        assertNotEquals(originalPackage.hash(), changedServices.hash());
+
+        // deployment.xml is changed -> same bundle hash
+        assertEquals(originalPackage.bundleHash(), changedDeploymentXml.bundleHash());
+        assertNotEquals(originalPackage.hash(), changedDeploymentXml.hash());
+    }
+
     private static Map<String, String> unzip(byte[] zip) {
         return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true)
                 .entries().stream()
@@ -115,4 +133,8 @@ public class ApplicationPackageTest {
                                           entry -> new String(entry.contentOrThrow(), UTF_8)));
     }
 
+    private ApplicationPackage getApplicationZip(String path) throws Exception {
+        return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 86e5acf07a0..06c9daa1892 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -42,6 +42,8 @@ import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 import static org.junit.Assert.assertEquals;
 
@@ -89,7 +91,9 @@ public class ApplicationSerializerTest {
                                                                         Optional.of(Instant.ofEpochMilli(666)),
                                                                         Optional.empty(),
                                                                         Optional.of("best commit"),
-                                                                        true);
+                                                                        true,
+                                                                        Optional.of("hash1"));
+        SortedSet<ApplicationVersion> versions = new TreeSet<>(Set.of(applicationVersion1));
         assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get());
 
         ApplicationVersion applicationVersion2 = ApplicationVersion
@@ -142,6 +146,7 @@ public class ApplicationSerializerTest {
                                                Set.of(publicKey, otherPublicKey),
                                                projectId,
                                                Optional.of(applicationVersion1),
+                                               versions,
                                                instances);
 
         Application serialized = APPLICATION_SERIALIZER.fromSlime(SlimeUtils.toJsonBytes(APPLICATION_SERIALIZER.toSlime(original)));
@@ -153,6 +158,10 @@ public class ApplicationSerializerTest {
         assertEquals(original.latestVersion().get().buildTime(), serialized.latestVersion().get().buildTime());
         assertEquals(original.latestVersion().get().sourceUrl(), serialized.latestVersion().get().sourceUrl());
         assertEquals(original.latestVersion().get().commit(), serialized.latestVersion().get().commit());
+        // Check deployed versions are added
+        assertEquals(original.versions(), serialized.versions());
+        assertEquals(serialized.versions(), new TreeSet<>(Set.of(applicationVersion1, applicationVersion2)));
+
 
         assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
         assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index eca2b17a5f1..f4f52b20325 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -91,7 +91,8 @@ public class RunSerializerTest {
                                                                         Optional.of(Instant.ofEpochMilli(100)),
                                                                         Optional.empty(),
                                                                         Optional.empty(),
-                                                                        true);
+                                                                        true,
+                                                                        Optional.empty());
         assertEquals(applicationVersion, run.versions().targetApplication());
         assertEquals(applicationVersion.authorEmail(), run.versions().targetApplication().authorEmail());
         assertEquals(applicationVersion.buildTime(), run.versions().targetApplication().buildTime());
diff --git a/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip
new file mode 100644
index 00000000000..90d52524fb3
--- /dev/null
+++ b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip
diff --git a/controller-server/src/test/resources/application-packages/changed-services-xml.zip b/controller-server/src/test/resources/application-packages/changed-services-xml.zip
new file mode 100644
index 00000000000..3051d27836a
--- /dev/null
+++ b/controller-server/src/test/resources/application-packages/changed-services-xml.zip
diff --git a/controller-server/src/test/resources/application-packages/original.zip b/controller-server/src/test/resources/application-packages/original.zip
new file mode 100644
index 00000000000..4cf2ffa7c46
--- /dev/null
+++ b/controller-server/src/test/resources/application-packages/original.zip