From 25bc8d1f050942ed29273b46efe0f7b2b0432045 Mon Sep 17 00:00:00 2001 From: jonmv Date: Fri, 4 Nov 2022 22:46:07 +0100 Subject: Revert "Merge pull request #24725 from vespa-engine/jonmv/application-package-streams" This reverts commit c7a0effde1c205d8790e5d989437aefd724bc70f, reversing changes made to 7bcb7768d75c1eef5bc70a2aa84c363f9b7ec643. --- .../hosted/controller/ApplicationController.java | 66 +++--- .../application/pkg/ApplicationPackage.java | 27 ++- .../application/pkg/ApplicationPackageStream.java | 244 --------------------- .../controller/application/pkg/TestPackage.java | 99 ++++----- .../controller/deployment/InternalStepRunner.java | 9 +- 5 files changed, 95 insertions(+), 350 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java (limited to 'controller-server/src/main/java/com') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 80e6ff4f0ff..34a7ae89dd2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -58,7 +58,6 @@ import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; @@ -80,7 +79,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; -import java.io.ByteArrayInputStream; import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Clock; @@ -100,7 +98,6 @@ import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -492,6 +489,9 @@ public class ApplicationController { DeploymentId deployment = new DeploymentId(job.application(), zone); try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { + Set containerEndpoints; + Optional endpointCertificateMetadata; + Run run = controller.jobController().last(job) .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); @@ -500,32 +500,30 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); - ApplicationPackageStream applicationPackage = new ApplicationPackageStream(() -> applicationStore.stream(deployment, revision), - ApplicationPackageStream.addingCertificate(run.testerCertificate())); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); + AtomicReference lastRevision = new AtomicReference<>(); Instance instance; - Set containerEndpoints; try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); instance = application.get().require(job.application().instance()); + if ( ! applicationPackage.trustedCertificates().isEmpty() + && run.testerCertificate().isPresent()) + applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); + + endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec()); + containerEndpoints = controller.routing().of(deployment).prepare(application); } // Release application lock while doing the deployment, which is a lengthy task. - Supplier> endpointCertificateMetadata = () -> { - try (Mutex lock = lock(applicationId)) { - Optional data = endpointCertificates.getMetadata(instance, zone, applicationPackage.truncatedPackage().deploymentSpec()); - data.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); - return data; - } - }; - // Carry out deployment without holding the application lock. DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, endpointCertificateMetadata, run.isDryRun()); + endpointCertificateMetadata.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); // Record the quota usage for this application var quotaUsage = deploymentQuotaUsage(zone, job.application()); @@ -546,10 +544,8 @@ public class ApplicationController { .distinct() .collect(Collectors.toList())) .orElseGet(List::of); - if (warnings.isEmpty()) - controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else - controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); + else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); } lockApplicationOrThrow(applicationId, application -> @@ -610,23 +606,23 @@ public class ApplicationController { /** Deploy a system application to given zone */ public DeploymentResult deploySystemApplicationPackage(SystemApplication application, ZoneId zone, Version version) { if (application.hasApplicationPackage()) { - ApplicationPackageStream applicationPackage = new ApplicationPackageStream( - () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) + ApplicationPackage applicationPackage = new ApplicationPackage( + artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); - return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), Optional::empty, false); + return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), false); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } } /** Deploys the given tester application to the given zone. */ - public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), Optional::empty, false); + public DeploymentResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { + return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), false); } - private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackageStream applicationPackage, + private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackage applicationPackage, ZoneId zone, Version platform, Set endpoints, - Supplier> endpointCertificateMetadata, + Optional endpointCertificateMetadata, boolean dryRun) { DeploymentId deployment = new DeploymentId(application, zone); try { @@ -642,8 +638,13 @@ public class ApplicationController { .filter(tenant-> tenant instanceof AthenzTenant) .map(tenant -> ((AthenzTenant)tenant).domain()); - Supplier deploymentQuota = () -> DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), - asList(application.tenant()), application, zone, applicationPackage.truncatedPackage().deploymentSpec()); + if (zone.environment().isManuallyDeployed()) + controller.applications().applicationStore().putMeta(deployment, + clock.instant(), + applicationPackage.metaDataZip()); + + Quota deploymentQuota = DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), + asList(application.tenant()), application, zone, applicationPackage.deploymentSpec()); List tenantSecretStores = controller.tenants() .get(application.tenant()) @@ -653,9 +654,9 @@ public class ApplicationController { List operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() .map(SupportAccessGrant::certificate) .collect(toList()); - Supplier> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); + Optional cloudAccount = decideCloudAccountOf(deployment, applicationPackage.deploymentSpec()); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage::zipStream, platform, + configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage.zippedContent(), platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun)); @@ -664,12 +665,7 @@ public class ApplicationController { } finally { // Even if prepare fails, routing configuration may need to be updated if ( ! application.instance().isTester()) { - controller.routing().of(deployment).configure(applicationPackage.truncatedPackage().deploymentSpec()); - if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(deployment, - clock.instant(), - applicationPackage.truncatedPackage().metaDataZip()); - + controller.routing().of(deployment).configure(applicationPackage.deploymentSpec()); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 53c78d7c8ec..b99d825a779 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -59,19 +59,19 @@ import static java.util.stream.Collectors.toMap; * A representation of the content of an application package. * Only meta-data content can be accessed as anything other than compressed data. * A package is identified by a hash of the content. - * + * + * This is immutable. + * * @author bratseth * @author jonmv */ public class ApplicationPackage { - static final String trustedCertificatesFile = "security/clients.pem"; - static final String buildMetaFile = "build-meta.json"; + private static final String trustedCertificatesFile = "security/clients.pem"; + private static final String buildMetaFile = "build-meta.json"; static final String deploymentFile = "deployment.xml"; - static final String validationOverridesFile = "validation-overrides.xml"; + private static final String validationOverridesFile = "validation-overrides.xml"; static final String servicesFile = "services.xml"; - static final Set prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile); - private static Hasher hasher() { return Hashing.murmur3_128().newHasher(); } private final String bundleHash; @@ -101,7 +101,7 @@ public class ApplicationPackage { */ public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.files = new ZipArchiveCache(zippedContent, prePopulated); + this.files = new ZipArchiveCache(zippedContent, Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile)); Optional deploymentSpec = files.get(deploymentFile).map(bytes -> new String(bytes, UTF_8)).map(DeploymentSpec::fromXml); if (requireFiles && deploymentSpec.isEmpty()) @@ -122,6 +122,17 @@ public class ApplicationPackage { preProcessAndPopulateCache(); } + /** Returns a copy of this with the given certificate appended. */ + public ApplicationPackage withTrustedCertificate(X509Certificate certificate) { + List trustedCertificates = new ArrayList<>(this.trustedCertificates); + trustedCertificates.add(certificate); + byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8); + + ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length); + ZipEntries.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); + return new ApplicationPackage(modified.toByteArray()); + } + /** Hash of all files and settings that influence what is deployed to config servers. */ public String bundleHash() { return bundleHash; @@ -284,7 +295,7 @@ public class ApplicationPackage { private Map> read(Collection names) { var entries = ZipEntries.from(zip, - names::contains, + name -> names.contains(name), maxSize, true) .asList().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java deleted file mode 100644 index 3288759b174..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ /dev/null @@ -1,244 +0,0 @@ -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.X509CertificateUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -import static com.yahoo.security.X509CertificateUtils.certificateListFromPem; -import static java.io.OutputStream.nullOutputStream; -import static java.lang.Math.min; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Wraps a zipped application package stream. - * This allows replacing content as the input stream is read. - * This also retains a truncated {@link ApplicationPackage}, containing only the specified set of files, - * which can be accessed when this stream is fully exhausted. - * - * @author jonmv - */ -public class ApplicationPackageStream { - - private final byte[] inBuffer = new byte[1 << 16]; - private final ByteArrayOutputStream out = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream outZip = new ZipOutputStream(out); - private final ByteArrayOutputStream teeOut = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream teeZip = new ZipOutputStream(teeOut); - private final Replacer replacer; - private final Predicate filter; - private final Supplier in; - - private ApplicationPackage ap = null; - private boolean done = false; - - public static Replacer addingCertificate(Optional certificate) { - return certificate.map(cert -> Replacer.of(Map.of(ApplicationPackage.trustedCertificatesFile, - trustBytes -> append(trustBytes, cert)))) - .orElse(Replacer.of(Map.of())); - } - - static InputStream append(InputStream trustIn, X509Certificate cert) { - try { - List trusted = trustIn == null ? new ArrayList<>() - : new ArrayList<>(certificateListFromPem(new String(trustIn.readAllBytes(), UTF_8))); - trusted.add(cert); - return new ByteArrayInputStream(X509CertificateUtils.toPem(trusted).getBytes(UTF_8)); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - /** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier in) { - this(in, __ -> true, Map.of()); - } - - /** Stream that replaces the indicated entries, and copies all metadata files to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier in, Replacer replacer) { - this(in, name -> ApplicationPackage.prePopulated.contains(name) || name.endsWith(".xml"), replacer); - } - - /** Stream that replaces the indicated entries, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier in, Predicate truncation, Map> replacements) { - this(in, truncation, Replacer.of(replacements)); - } - - /** Stream that uses the given replacer to modify content, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier in, Predicate truncation, Replacer replacer) { - this.in = in; - this.filter = truncation; - this.replacer = replacer; - } - - public InputStream zipStream() { - return new Stream(); - } - - /** Returns the application backed by only the files indicated by the truncation filter. Throws if not yet exhausted. */ - public ApplicationPackage truncatedPackage() { - if (ap == null) throw new IllegalStateException("must completely exhaust input before reading package"); - return ap; - } - - private class Stream extends InputStream { - - private final ZipInputStream inZip = new ZipInputStream(in.get()); - private byte[] currentOut = new byte[0]; - private InputStream currentIn = InputStream.nullInputStream(); - private boolean includeCurrent = false; - private int pos = 0; - private boolean closed = false; - - private void fill() throws IOException { - if (done) return; - while (out.size() == 0) { - // Exhaust current entry first. - int i, n = out.size(); - while (out.size() == 0 && (i = currentIn.read(inBuffer)) != -1) { - if (includeCurrent) teeZip.write(inBuffer, 0, i); - outZip.write(inBuffer, 0, i); - n += i; - } - - // Current entry exhausted, look for next. - if (n == 0) { - next(); - if (done) break; - } - } - - currentOut = out.toByteArray(); - out.reset(); - pos = 0; - } - - private void next() throws IOException { - if (includeCurrent) teeZip.closeEntry(); - outZip.closeEntry(); - - ZipEntry next = inZip.getNextEntry(); - String name; - InputStream content = null; - if (next == null) { - // We may still have replacements to fill in, but if we don't, we're done filling, forever! - name = replacer.next(); - if (name == null) { - outZip.close(); // This typically makes new output available, so must check for that after this. - teeZip.close(); - currentIn = nullInputStream(); - ap = new ApplicationPackage(teeOut.toByteArray()); - done = true; - return; - } - } - else { - name = next.getName(); - content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it. - } - - includeCurrent = filter.test(name); - currentIn = replacer.modify(name, content); - if (currentIn == null) { - currentIn = InputStream.nullInputStream(); - } - else { - if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name)); - outZip.putNextEntry(new ZipEntry(name)); - } - } - - @Override - public int read() throws IOException { - if (closed) throw new IOException("stream closed"); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - return 0xff & inBuffer[pos++]; - } - - @Override - public int read(byte[] out, int off, int len) throws IOException { - if (closed) throw new IOException("stream closed"); - if ((off | len | (off + len) | (out.length - (off + len))) < 0) throw new IndexOutOfBoundsException(); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - int n = min(currentOut.length - pos, len); - System.arraycopy(currentOut, pos, out, off, n); - pos += n; - return n; - } - - @Override - public int available() throws IOException { - return pos == currentOut.length && done ? 0 : 1; - } - - @Override - public void close() { - if ( ! closed) try { - transferTo(nullOutputStream()); - inZip.close(); - closed = true; - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } - - /** Replaces entries in a zip stream as they are encountered, then appends remaining entries at the end. */ - public interface Replacer { - - /** Called when the entries of the original zip stream are exhausted. Return remaining names, or {@code null} when none left. */ - String next(); - - /** Modify content for a given name; return {@code null} for removal; in is {@code null} for entries not present in the input. */ - InputStream modify(String name, InputStream in); - - /** - * Wraps a map of fixed replacements, and: - *
    - *
  • Removes entries whose value is {@code null}.
    • - *
  • Modifies entries present in both input and the map.
    • - *
  • Appends entries present exclusively in the map.
    • - *
  • Writes all other entries as they are.
    • - *
- */ - static Replacer of(Map> replacements) { - return new Replacer() { - final Map> remaining = new HashMap<>(replacements); - @Override public String next() { - return remaining.isEmpty() ? null : remaining.keySet().iterator().next(); - } - @Override public InputStream modify(String name, InputStream in) { - UnaryOperator mapper = remaining.remove(name); - return mapper == null ? in : mapper.apply(in); - } - }; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index bb24e70b96d..5b20c57fcca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -18,14 +18,15 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.Replacer; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayInputStream; -import java.io.InputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -42,8 +43,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; import java.util.jar.JarInputStream; import java.util.jar.Manifest; import java.util.regex.Pattern; @@ -54,7 +53,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; -import static java.io.InputStream.nullInputStream; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; @@ -72,14 +71,32 @@ public class TestPackage { static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - private final ApplicationPackageStream applicationPackageStream; + private final ApplicationPackage applicationPackage; private final X509Certificate certificate; - public TestPackage(Supplier inZip, boolean isPublicSystem, RunId id, Testerapp testerApp, + public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp, DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { - KeyPair keyPair; + + // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. + Map entries = new HashMap<>(); + entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); + entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); + + entries.put(servicesFile, + servicesXml(! isPublicSystem, + certificateValidFrom != null, + hasLegacyTests(testPackage), + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), + testerApp)); + + entries.put(deploymentFile, + deploymentXml(id.tester(), + spec.athenzDomain(), + spec.requireInstance(id.application().instance()) + .athenzService(id.type().zone().environment(), id.type().zone().region()))); + if (certificateValidFrom != null) { - keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); this.certificate = X509CertificateBuilder.fromKeypair(keyPair, subject, @@ -88,60 +105,26 @@ public class TestPackage { SignatureAlgorithm.SHA512_WITH_RSA, BigInteger.valueOf(1)) .build(); + entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); + entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); } else { - keyPair = null; this.certificate = null; } - this.applicationPackageStream = new ApplicationPackageStream(inZip, __ -> false, new Replacer() { - - // Initially skips all declared entries, ensuring they're generated and appended after all input entries. - final Map> entries = new HashMap<>(); - final Map> replacements = new HashMap<>(); - boolean hasLegacyTests = false; - - @Override - public String next() { - if (entries.isEmpty()) return null; - String next = entries.keySet().iterator().next(); - replacements.put(next, entries.remove(next)); - return next; - } - @Override - public InputStream modify(String name, InputStream in) { - hasLegacyTests |= name.startsWith("artifacts/") && name.endsWith("-tests.jar"); - return entries.containsKey(name) ? null : replacements.get(name).apply(in); - } + ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000); + transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries); + this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); + } + + static boolean hasLegacyTests(byte[] testPackage) { + return ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() + .anyMatch(file -> file.name().startsWith("artifacts/") && file.name().endsWith("-tests.jar")); - { - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - entries.put("artifacts/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - entries.put("tests/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - - entries.put(servicesFile, - __ -> new ByteArrayInputStream(servicesXml( ! isPublicSystem, - certificateValidFrom != null, - hasLegacyTests, - testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), - testerApp))); - - entries.put(deploymentFile, - __ -> new ByteArrayInputStream(deploymentXml(id.tester(), - spec.athenzDomain(), - spec.requireInstance(id.application().instance()) - .athenzService(id.type().zone().environment(), id.type().zone().region())))); - - if (certificate != null) { - entries.put("artifacts/key", __ -> new ByteArrayInputStream(KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8))); - entries.put("artifacts/cert", __ -> new ByteArrayInputStream(X509CertificateUtils.toPem(certificate).getBytes(UTF_8))); - } - } - }); } - public ApplicationPackageStream asApplicationPackage() { - return applicationPackageStream; + public ApplicationPackage asApplicationPackage() { + return applicationPackage; } public X509Certificate certificate() { @@ -224,7 +207,7 @@ public class TestPackage { return new TestSummary(problems, suites); } - static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { NodeResources nodeResources = spec.steps().stream() .filter(step -> step.concerns(zone.environment())) .findFirst() @@ -236,7 +219,7 @@ public class TestPackage { } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, + public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { int jdiscMemoryGb = 2; // 2Gb memory for tester application which uses Maven. int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); @@ -296,7 +279,7 @@ public class TestPackage { } /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - static byte[] deploymentXml(TesterId id, Optional athenzDomain, Optional athenzService) { + public static byte[] deploymentXml(TesterId id, Optional athenzDomain, Optional athenzService) { String deploymentSpec = "\n" + " controller.applications().applicationStore().streamTester(id.application().tenant(), - id.application().application(), revision), + TestPackage testPackage = new TestPackage(testZip, controller.system().isPublic(), id, controller.controllerConfig().steprunner().testerapp(), -- cgit v1.2.3