Copy deployment spec to test package on submission

7 files changed, 28 insertions, 10 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index 3ec7f120726..46ba22af512 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -63,10 +63,10 @@ import static java.util.stream.Collectors.toMap;
  */
 public class ApplicationPackage {
 
+    public static  final String deploymentFile = "deployment.xml";
     static final String trustedCertificatesDir = "security/";
     static final String trustedCertificatesFile = trustedCertificatesDir + "clients.pem";
     static final String buildMetaFile = "build-meta.json";
-    static final String deploymentFile = "deployment.xml";
     static final String validationOverridesFile = "validation-overrides.xml";
     static final String servicesFile = "services.xml";
     static final Set<String> prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
index d22a41f74a4..e034e9c7a33 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java
@@ -32,7 +32,7 @@ public class ZipEntries {
 
         Options options = Options.standard()
                                  .pathPredicate(entryNameMatcher)
-                                 .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB
+                                 .maxSize(2L << 30) // 2 GB
                                  .maxEntrySize(maxEntrySizeInBytes)
                                  .maxEntries(1024)
                                  .truncateEntry(!throwIfEntryExceedsMaxSize);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index f2357a49952..03489222922 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
 import com.yahoo.component.VersionCompatibility;
 import com.yahoo.concurrent.UncheckedTimeoutException;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.SystemName;
@@ -75,6 +76,7 @@ import java.util.logging.Logger;
 import java.util.stream.Stream;
 
 import static com.yahoo.collections.Iterables.reversed;
+import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset;
@@ -89,6 +91,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialRe
 import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.time.temporal.ChronoUnit.SECONDS;
 import static java.util.Comparator.comparing;
 import static java.util.Comparator.naturalOrder;
@@ -577,7 +580,8 @@ public class JobController {
                                                 id.application(),
                                                 version.get().id(),
                                                 submission.applicationPackage().zippedContent(),
-                                                submission.testPackage(),
+                                                withDeploymentSpec(submission.testPackage(),
+                                                                   submission.applicationPackage().deploymentSpec()),
                                                 diff);
             applications.applicationStore().putMeta(id.tenant(),
                                                     id.application(),
@@ -599,6 +603,15 @@ public class JobController {
         return version.get();
     }
 
+    static byte[] withDeploymentSpec(byte[] testZip, DeploymentSpec spec) {
+        ZipBuilder zip = new ZipBuilder(testZip.length + (1 << 12));
+        try (zip) {
+            zip.add(testZip, name -> !name.equals(deploymentFile));
+            zip.add(deploymentFile, spec.xmlForm().getBytes(UTF_8));
+        }
+        return zip.toByteArray();
+    }
+
     private void validate(TenantAndApplicationId id, Submission submission) {
         controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage);
         controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java
index a05a8cd753f..b5307b07603 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java
@@ -5,6 +5,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.function.Predicate;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
@@ -25,9 +26,10 @@ public class ZipBuilder implements AutoCloseable {
         zipOutputStream = new ZipOutputStream(byteArrayOutputStream);
     }
 
-    public void add(byte[] zippedContent) {
+    public void add(byte[] zippedContent, Predicate<String> filter) {
         try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) {
             for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) {
+                if ( ! filter.test(entry.getName())) continue;
                 zipOutputStream.putNextEntry(new ZipEntry(entry.getName()));
                 zin.transferTo(zipOutputStream);
                 zipOutputStream.closeEntry();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
index f018b566674..910d526a2ed 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java
@@ -166,7 +166,7 @@ public class ApplicationPackageTest {
         }
     }
 
-    static Map<String, String> unzip(byte[] zip) {
+    public static Map<String, String> unzip(byte[] zip) {
         return ZipEntries.from(zip, __ -> true, 1 << 24, true)
                          .asList().stream()
                          .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name,
@@ -181,7 +181,7 @@ public class ApplicationPackageTest {
         return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true, checkCertificateFile);
     }
 
-    static byte[] zip(Map<String, String> content) {
+    public static byte[] zip(Map<String, String> content) {
         return filesZip(content.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(),
                                                                              entry -> entry.getValue().getBytes(UTF_8))));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java
index 8fe48f805df..a6f7f45caa4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java
@@ -40,7 +40,7 @@ public class ZipBuilderTest {
             }
 
             // Add the zipped data from zip1 to zip2
-            zipBuilder2.add(zipBuilder1.toByteArray());
+            zipBuilder2.add(zipBuilder1.toByteArray(), __ -> true);
 
             Map<String, String> actual = unzipToMap(zipBuilder2.toByteArray());
             assertEquals(expected, actual);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 345f164f8da..d3386a3a2ad 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -47,11 +47,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest;
 import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
@@ -102,6 +102,7 @@ import static com.yahoo.application.container.handler.Request.Method.GET;
 import static com.yahoo.application.container.handler.Request.Method.PATCH;
 import static com.yahoo.application.container.handler.Request.Method.POST;
 import static com.yahoo.application.container.handler.Request.Method.PUT;
+import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest.unzip;
 import static java.net.URLEncoder.encode;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.joining;
@@ -157,6 +158,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
     private static final UserId HOSTED_VESPA_OPERATOR = new UserId("johnoperator");
     private static final OAuthCredentials OKTA_CREDENTIALS = OAuthCredentials.createForTesting("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.he0ErCNloe4J7Id0Ry2SEDg09lKkZkfsRiGsdX_vgEg", "okta-it");
 
+    private static final byte[] testZip = ApplicationPackageTest.zip(Map.of("tests", "content"));
 
     private ContainerTester tester;
     private DeploymentTester deploymentTester;
@@ -892,7 +894,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET).userIdentity(HOSTED_VESPA_OPERATOR).properties(Map.of("tests", "true")),
                 (response) -> {
                     assertEquals("attachment; filename=\"tenant1.application1-tests2.zip\"", response.getHeaders().getFirst("Content-Disposition"));
-                    assertArrayEquals("content".getBytes(UTF_8), response.getBody());
+                    assertEquals(Map.of("tests", "content", "deployment.xml", packageWithService.deploymentSpec().xmlForm()),
+                                 unzip(response.getBody()));
                 },
                 200);
 
@@ -1842,7 +1845,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                                                                    + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\","
                                                                                    + "\"description\":\"my best commit yet\",\"risk\":9001}")
                                       .addBytes(EnvironmentResource.APPLICATION_ZIP, applicationPackage.zippedContent())
-                                      .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, "content".getBytes());
+                                      .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, testZip);
     }
 
     private String deployOptions(Optional<ApplicationVersion> applicationVersion) {