Store application package diffs on submit/deploy

4 files changed, 51 insertions, 12 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
index d072ad595c6..6620cd4d0e3 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
@@ -1,10 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.deployment;
 
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 
 import java.time.Instant;
@@ -23,11 +21,14 @@ public interface ApplicationStore {
     /** Returns the tenant application package of the given version. */
     byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion);
 
+    /** Returns the application package diff, compared to the previous build, for the given tenant, application and build number */
+    Optional<String> getDiff(TenantName tenantName, ApplicationName applicationName, long buildNumber);
+
     /** Find application package by given build number */
     Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber);
 
-    /** Stores the given tenant application package of the given version. */
-    void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage);
+    /** Stores the given tenant application package of the given version and diff since previous version. */
+    void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, String diff);
 
     /** Removes applications older than the given version, for the given application, and returns whether something was removed. */
     boolean prune(TenantName tenant, ApplicationName application, ApplicationVersion olderThanVersion);
@@ -47,8 +48,11 @@ public interface ApplicationStore {
     /** Removes all tester packages for the given tester. */
     void removeAllTesters(TenantName tenant, ApplicationName application);
 
-    /** Stores the given application package as the development package for the given application and zone. */
-    void putDev(DeploymentId deploymentId, byte[] applicationPackage);
+    /** Returns the application package diff, compared to the previous build, for the given deployment and build number */
+    Optional<String> getDevDiff(DeploymentId deploymentId, long buildNumber);
+
+    /** Stores the given application package as the development package for the given deployment and version and diff since previous version. */
+    void putDev(DeploymentId deploymentId, ApplicationVersion version, byte[] applicationPackage, String diff);
 
     /** Stores the given application meta data with the current time as part of the path. */
     void putMeta(TenantName tenant, ApplicationName application, Instant now, byte[] metaZip);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 71c1d860fc1..cf73d28473b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
@@ -424,11 +425,18 @@ public class JobController {
                                                 sourceUrl,
                                                 revision.map(SourceRevision::commit),
                                                 false));
+            String diff = application.get().latestVersion()
+                    .map(ApplicationVersion::buildNumber)
+                    .flatMap(build -> build.isPresent() ? Optional.of(build.getAsLong()) : Optional.empty())
+                    .flatMap(prevBuild -> controller.applications().applicationStore().find(id.tenant(), id.application(), prevBuild))
+                    .map(prevApplication -> ApplicationPackageDiff.diff(new ApplicationPackage(prevApplication), applicationPackage))
+                    .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage));
 
             controller.applications().applicationStore().put(id.tenant(),
                                                              id.application(),
                                                              version.get(),
-                                                             applicationPackage.zippedContent());
+                                                             applicationPackage.zippedContent(),
+                                                             diff);
             controller.applications().applicationStore().putTester(id.tenant(),
                                                                    id.application(),
                                                                    version.get(),
@@ -478,6 +486,7 @@ public class JobController {
             controller.applications().store(application);
         });
 
+        DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system()));
         Optional<Run> lastRun = last(id, type);
         lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
 
@@ -485,8 +494,12 @@ public class JobController {
         ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(),
                 Optional.empty(), Optional.empty(), Optional.empty(), true);
 
+        String diff = lastRun.map(run -> run.versions().targetApplication())
+                .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage))
+                .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage));
+
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
-            controller.applications().applicationStore().putDev(new DeploymentId(id, type.zone(controller.system())), applicationPackage.zippedContent());
+            controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff);
             start(id,
                   type,
                   new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index b98e6825353..8fafd620c0b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -597,6 +597,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage);
     }
 
+    private HttpResponse devApplicationPackageDiff(RunId runId) {
+        DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system()));
+        return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number())
+                .map(StringResponse::new)
+                .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId));
+    }
+
     private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) {
         var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName);
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
index 7db7268e4bf..7f4546dcfde 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java
@@ -31,6 +31,8 @@ public class ApplicationStoreMock implements ApplicationStore {
 
     private final Map<ApplicationId, Map<ApplicationVersion, byte[]>> store = new ConcurrentHashMap<>();
     private final Map<DeploymentId, byte[]> devStore = new ConcurrentHashMap<>();
+    private final Map<ApplicationId, Map<Long, String>> diffs = new ConcurrentHashMap<>();
+    private final Map<DeploymentId, Map<Long, String>> devDiffs = new ConcurrentHashMap<>();
     private final Map<ApplicationId, NavigableMap<Instant, byte[]>> meta = new ConcurrentHashMap<>();
     private final Map<DeploymentId, NavigableMap<Instant, byte[]>> metaManual = new ConcurrentHashMap<>();
 
@@ -56,6 +58,11 @@ public class ApplicationStoreMock implements ApplicationStore {
     }
 
     @Override
+    public Optional<String> getDiff(TenantName tenantName, ApplicationName applicationName, long buildNumber) {
+        return Optional.ofNullable(diffs.get(appId(tenantName, applicationName))).map(map -> map.get(buildNumber));
+    }
+
+    @Override
     public Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber) {
         return store.getOrDefault(appId(tenant, application), Map.of()).entrySet().stream()
                     .filter(kv -> kv.getKey().buildNumber().orElse(Long.MIN_VALUE) == buildNumber)
@@ -64,9 +71,10 @@ public class ApplicationStoreMock implements ApplicationStore {
     }
 
     @Override
-    public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage) {
-        store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>())
-                .put(applicationVersion, applicationPackage);
+    public void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, String diff) {
+        store.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(applicationVersion, applicationPackage);
+        applicationVersion.buildNumber().ifPresent(buildNumber ->
+                diffs.computeIfAbsent(appId(tenant, application), __ -> new ConcurrentHashMap<>()).put(buildNumber, diff));
     }
 
     @Override
@@ -103,8 +111,15 @@ public class ApplicationStoreMock implements ApplicationStore {
     }
 
     @Override
-    public void putDev(DeploymentId deploymentId, byte[] applicationPackage) {
+    public Optional<String> getDevDiff(DeploymentId deploymentId, long buildNumber) {
+        return Optional.ofNullable(devDiffs.get(deploymentId)).map(map -> map.get(buildNumber));
+    }
+
+    @Override
+    public void putDev(DeploymentId deploymentId, ApplicationVersion applicationVersion, byte[] applicationPackage, String diff) {
         devStore.put(deploymentId, applicationPackage);
+        applicationVersion.buildNumber().ifPresent(buildNumber ->
+                devDiffs.computeIfAbsent(deploymentId, __ -> new ConcurrentHashMap<>()).put(buildNumber, diff));
     }
 
     @Override