Merge pull request #19057 from vespa-engine/freva/diff

Application package diff

3 files changed, 43 insertions, 33 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
index dd9f8c38802..71f1821ff9a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java
@@ -1,10 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.deployment;
 
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 
 import java.time.Instant;
@@ -21,13 +19,19 @@ import java.util.Optional;
 public interface ApplicationStore {
 
     /** Returns the tenant application package of the given version. */
-    byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion);
+    byte[] get(DeploymentId deploymentId, ApplicationVersion applicationVersion);
+
+    /** Returns the application package diff, compared to the previous build, for the given tenant, application and build number */
+    Optional<byte[]> getDiff(TenantName tenantName, ApplicationName applicationName, long buildNumber);
+
+    /** Removes diffs for packages before the given build number */
+    void pruneDiffs(TenantName tenantName, ApplicationName applicationName, long beforeBuildNumber);
 
     /** Find application package by given build number */
     Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber);
 
-    /** Stores the given tenant application package of the given version. */
-    void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage);
+    /** Stores the given tenant application package of the given version and diff since previous version. */
+    void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage, byte[] diff);
 
     /** Removes applications older than the given version, for the given application, and returns whether something was removed. */
     boolean prune(TenantName tenant, ApplicationName application, ApplicationVersion olderThanVersion);
@@ -47,11 +51,14 @@ public interface ApplicationStore {
     /** Removes all tester packages for the given tester. */
     void removeAllTesters(TenantName tenant, ApplicationName application);
 
-    /** Stores the given application package as the development package for the given application and zone. */
-    void putDev(ApplicationId application, ZoneId zone, byte[] applicationPackage);
+    /** Returns the application package diff, compared to the previous build, for the given deployment and build number */
+    Optional<byte[]> getDevDiff(DeploymentId deploymentId, long buildNumber);
+
+    /** Removes diffs for dev packages before the given build number */
+    void pruneDevDiffs(DeploymentId deploymentId, long beforeBuildNumber);
 
-    /** Returns the development package for the given application and zone. */
-    byte[] getDev(ApplicationId application, ZoneId zone);
+    /** Stores the given application package as the development package for the given deployment and version and diff since previous version. */
+    void putDev(DeploymentId deploymentId, ApplicationVersion version, byte[] applicationPackage, byte[] diff);
 
     /** Stores the given application meta data with the current time as part of the path. */
     void putMeta(TenantName tenant, ApplicationName application, Instant now, byte[] metaZip);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
index 30fd8fad1bd..f83809e84c2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
@@ -23,7 +23,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
      */
     public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty(),
                                                                             Optional.empty(), Optional.empty(), Optional.empty(),
-                                                                            Optional.empty(), Optional.empty());
+                                                                            Optional.empty(), Optional.empty(), true);
 
     // This never changes and is only used to create a valid semantic version number, as required by application bundles
     private static final String majorVersion = "1.0";
@@ -35,11 +35,12 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
     private final Optional<Instant> buildTime;
     private final Optional<String> sourceUrl;
     private final Optional<String> commit;
+    private final boolean deployedDirectly;
 
     /** Public for serialisation only. */
     public ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail,
-                               Optional<Version> compileVersion, Optional<Instant> buildTime, Optional<String> sourceUrl,
-                               Optional<String> commit) {
+                              Optional<Version> compileVersion, Optional<Instant> buildTime, Optional<String> sourceUrl,
+                              Optional<String> commit, boolean deployedDirectly) {
         if (buildNumber.isEmpty() && (   source.isPresent() || authorEmail.isPresent() || compileVersion.isPresent()
                                       || buildTime.isPresent() || sourceUrl.isPresent() || commit.isPresent()))
             throw new IllegalArgumentException("Build number must be present if any other attribute is");
@@ -63,45 +64,37 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
         this.buildTime = buildTime;
         this.sourceUrl = Objects.requireNonNull(sourceUrl, "sourceUrl cannot be null");
         this.commit = Objects.requireNonNull(commit, "commit cannot be null");
+        this.deployedDirectly = deployedDirectly;
     }
 
     /** Create an application package version from a completed build, without an author email */
     public static ApplicationVersion from(SourceRevision source, long buildNumber) {
         return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.empty(),
-                                      Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
-    }
-
-    /** Creates an version from a completed build and an author email. */
-    public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail) {
-        return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail),
-                                      Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
+                                      Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false);
     }
 
     /** Creates an version from a completed build, an author email, and build meta data. */
     public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail,
                                           Version compileVersion, Instant buildTime) {
         return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail),
-                                      Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty());
+                                      Optional.of(compileVersion), Optional.of(buildTime), Optional.empty(), Optional.empty(), false);
     }
 
     /** Creates an version from a completed build, an author email, and build meta data. */
     public static ApplicationVersion from(Optional<SourceRevision> source, long buildNumber, Optional<String> authorEmail,
                                           Optional<Version> compileVersion, Optional<Instant> buildTime,
-                                          Optional<String> sourceUrl, Optional<String> commit) {
-        return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit);
+                                          Optional<String> sourceUrl, Optional<String> commit, boolean deployedDirectly) {
+        return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit, deployedDirectly);
     }
 
     /** Returns an unique identifier for this version or "unknown" if version is not known */
     public String id() {
-        if (isUnknown()) {
-            return "unknown";
-        }
-        return String.format("%s.%d-%s",
-                             majorVersion,
-                             buildNumber.getAsLong(),
-                             source.map(SourceRevision::commit).map(ApplicationVersion::abbreviateCommit)
-                                   .or(this::commit)
-                                   .orElse("unknown"));
+        if (isUnknown()) return "unknown";
+
+        return source.map(SourceRevision::commit).map(ApplicationVersion::abbreviateCommit)
+                .or(this::commit)
+                .map(commit -> String.format("%s.%d-%s", majorVersion, buildNumber.getAsLong(), commit))
+                .orElseGet(() -> majorVersion + "." + buildNumber.getAsLong());
     }
 
     /**
@@ -142,18 +135,24 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
         return this.equals(unknown);
     }
 
+    /** Returns whether the application package for this version was deployed directly to zone */
+    public boolean isDeployedDirectly() {
+        return deployedDirectly;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if ( ! (o instanceof ApplicationVersion)) return false;
         ApplicationVersion that = (ApplicationVersion) o;
         return    Objects.equals(buildNumber, that.buildNumber)
-               && Objects.equals(commit(), that.commit());
+               && Objects.equals(commit(), that.commit())
+               && deployedDirectly == that.deployedDirectly;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(buildNumber, commit());
+        return Objects.hash(buildNumber, commit(), deployedDirectly);
     }
 
     @Override
@@ -175,6 +174,9 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
         if (buildNumber().isEmpty() || o.buildNumber().isEmpty())
             return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Unknown version sorts first
 
+        if (deployedDirectly || o.deployedDirectly)
+            return Boolean.compare(deployedDirectly, o.deployedDirectly); // Directly deployed versions sort first
+
         return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong());
     }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
index 135429be8f9..1306f4846c2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
@@ -103,6 +103,7 @@ enum PathGroup {
     applicationInfo(Matcher.tenant,
                     Matcher.application,
                     "/application/v4/tenant/{tenant}/application/{application}/package",
+                    "/application/v4/tenant/{tenant}/application/{application}/diff/{number}",
                     "/application/v4/tenant/{tenant}/application/{application}/compile-version",
                     "/application/v4/tenant/{tenant}/application/{application}/deployment",
                     "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}",