diff options
11 files changed, 286 insertions, 176 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index d3331c3cfd4..7c6ba50b953 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -13,11 +13,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import java.io.InputStream; import java.security.cert.X509Certificate; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import static java.util.Objects.requireNonNull; @@ -32,27 +34,28 @@ public class DeploymentData { private final ApplicationId instance; private final Tags tags; private final ZoneId zone; - private final byte[] applicationPackage; + private final InputStream applicationPackage; private final Version platform; private final Set<ContainerEndpoint> containerEndpoints; - private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepo; private final Optional<AthenzDomain> athenzDomain; - private final Quota quota; + private final Supplier<Quota> quota; private final List<TenantSecretStore> tenantSecretStores; private final List<X509Certificate> operatorCertificates; - private final Optional<CloudAccount> cloudAccount; + private final Supplier<Optional<CloudAccount>> cloudAccount; private final boolean dryRun; - public DeploymentData(ApplicationId instance, Tags tags, ZoneId zone, byte[] applicationPackage, Version platform, + public DeploymentData(ApplicationId instance, Tags tags, ZoneId zone, InputStream applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata, + Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepo, Optional<AthenzDomain> athenzDomain, - Quota quota, + Supplier<Quota> quota, List<TenantSecretStore> tenantSecretStores, List<X509Certificate> operatorCertificates, - Optional<CloudAccount> cloudAccount, boolean dryRun) { + Supplier<Optional<CloudAccount>> cloudAccount, + boolean dryRun) { this.instance = requireNonNull(instance); this.tags = requireNonNull(tags); this.zone = requireNonNull(zone); @@ -79,7 +82,7 @@ public class DeploymentData { return zone; } - public byte[] applicationPackage() { + public InputStream applicationPackage() { return applicationPackage; } @@ -92,7 +95,7 @@ public class DeploymentData { } public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { - return endpointCertificateMetadata; + return endpointCertificateMetadata.get(); } public Optional<DockerImage> dockerImageRepo() { @@ -104,7 +107,7 @@ public class DeploymentData { } public Quota quota() { - return quota; + return quota.get(); } public List<TenantSecretStore> tenantSecretStores() { @@ -116,9 +119,11 @@ public class DeploymentData { } public Optional<CloudAccount> cloudAccount() { - return cloudAccount; + return cloudAccount.get(); } - public boolean isDryRun() { return dryRun; } + public boolean isDryRun() { + return dryRun; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 34a7ae89dd2..299d7031195 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -58,6 +58,7 @@ import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; @@ -79,6 +80,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; +import java.io.ByteArrayInputStream; import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Clock; @@ -98,6 +100,7 @@ import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -489,9 +492,6 @@ public class ApplicationController { DeploymentId deployment = new DeploymentId(job.application(), zone); try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { - Set<ContainerEndpoint> containerEndpoints; - Optional<EndpointCertificateMetadata> endpointCertificateMetadata; - Run run = controller.jobController().last(job) .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); @@ -500,60 +500,65 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); - ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); - - AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); - Instance instance; - try (Mutex lock = lock(applicationId)) { - LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); - application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - instance = application.get().require(job.application().instance()); - - if ( ! applicationPackage.trustedCertificates().isEmpty() - && run.testerCertificate().isPresent()) - applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); - - endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec()); - - containerEndpoints = controller.routing().of(deployment).prepare(application); - - } // Release application lock while doing the deployment, which is a lengthy task. - - // Carry out deployment without holding the application lock. - DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, - endpointCertificateMetadata, run.isDryRun()); - - endpointCertificateMetadata.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); - - // Record the quota usage for this application - var quotaUsage = deploymentQuotaUsage(zone, job.application()); - - // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as - // the source since it's the same application, so it should have the same warnings. - // These notifications are only updated when the last submitted revision is deployed here. - NotificationSource source = zone.environment().isManuallyDeployed() - ? NotificationSource.from(deployment) - : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; - if (source != null) { - List<String> warnings = Optional.ofNullable(result.log()) - .map(logs -> logs.stream() - .filter(LogEntry::concernsPackage) - .filter(log -> log.level().intValue() >= Level.WARNING.intValue()) - .map(LogEntry::message) - .sorted() - .distinct() - .collect(Collectors.toList())) - .orElseGet(List::of); - if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); - } + try (ApplicationPackageStream applicationPackage = new ApplicationPackageStream(applicationStore.stream(deployment, revision), + ApplicationPackageStream.addingCertificate(run.testerCertificate()))) { + AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); + Instance instance; + Set<ContainerEndpoint> containerEndpoints; + try (Mutex lock = lock(applicationId)) { + LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); + instance = application.get().require(job.application().instance()); + + containerEndpoints = controller.routing().of(deployment).prepare(application); + + } // Release application lock while doing the deployment, which is a lengthy task. + + Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata = () -> { + try (Mutex lock = lock(applicationId)) { + Optional<EndpointCertificateMetadata> data = endpointCertificates.getMetadata(instance, zone, applicationPackage.truncatedPackage().deploymentSpec()); + data.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); + return data; + } + }; + + // Carry out deployment without holding the application lock. + DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, + endpointCertificateMetadata, run.isDryRun()); + + + // Record the quota usage for this application + var quotaUsage = deploymentQuotaUsage(zone, job.application()); + + // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as + // the source since it's the same application, so it should have the same warnings. + // These notifications are only updated when the last submitted revision is deployed here. + NotificationSource source = zone.environment().isManuallyDeployed() + ? NotificationSource.from(deployment) + : revision.equals(lastRevision.get()) ? NotificationSource.from(applicationId) : null; + if (source != null) { + List<String> warnings = Optional.ofNullable(result.log()) + .map(logs -> logs.stream() + .filter(LogEntry::concernsPackage) + .filter(log -> log.level().intValue() >= Level.WARNING.intValue()) + .map(LogEntry::message) + .sorted() + .distinct() + .collect(Collectors.toList())) + .orElseGet(List::of); + if (warnings.isEmpty()) + controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); + else + controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + } - lockApplicationOrThrow(applicationId, application -> - store(application.with(job.application().instance(), - i -> i.withNewDeployment(zone, revision, platform, - clock.instant(), warningsFrom(result.log()), - quotaUsage)))); - return result; + lockApplicationOrThrow(applicationId, application -> + store(application.with(job.application().instance(), + i -> i.withNewDeployment(zone, revision, platform, + clock.instant(), warningsFrom(result.log()), + quotaUsage)))); + return result; + } } } @@ -606,26 +611,26 @@ public class ApplicationController { /** Deploy a system application to given zone */ public DeploymentResult deploySystemApplicationPackage(SystemApplication application, ZoneId zone, Version version) { if (application.hasApplicationPackage()) { - ApplicationPackage applicationPackage = new ApplicationPackage( - artifactRepository.getSystemApplicationPackage(application.id(), zone, version) + ApplicationPackageStream applicationPackage = new ApplicationPackageStream( + new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), false); + return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), Optional::empty, false); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } } /** Deploys the given tester application to the given zone. */ - public DeploymentResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), false); + public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { + return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), Optional::empty, false); } - private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackage applicationPackage, + private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata, + Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, boolean dryRun) { DeploymentId deployment = new DeploymentId(application, zone); - try { + try (applicationPackage) { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag .with(FetchVector.Dimension.ZONE_ID, zone.value()) @@ -638,13 +643,8 @@ public class ApplicationController { .filter(tenant-> tenant instanceof AthenzTenant) .map(tenant -> ((AthenzTenant)tenant).domain()); - if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(deployment, - clock.instant(), - applicationPackage.metaDataZip()); - - Quota deploymentQuota = DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), - asList(application.tenant()), application, zone, applicationPackage.deploymentSpec()); + Supplier<Quota> deploymentQuota = () -> DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), + asList(application.tenant()), application, zone, applicationPackage.truncatedPackage().deploymentSpec()); List<TenantSecretStore> tenantSecretStores = controller.tenants() .get(application.tenant()) @@ -654,9 +654,9 @@ public class ApplicationController { List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() .map(SupportAccessGrant::certificate) .collect(toList()); - Optional<CloudAccount> cloudAccount = decideCloudAccountOf(deployment, applicationPackage.deploymentSpec()); + Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage.zippedContent(), platform, + configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage, platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun)); @@ -665,7 +665,12 @@ public class ApplicationController { } finally { // Even if prepare fails, routing configuration may need to be updated if ( ! application.instance().isTester()) { - controller.routing().of(deployment).configure(applicationPackage.deploymentSpec()); + controller.routing().of(deployment).configure(applicationPackage.truncatedPackage().deploymentSpec()); + if (zone.environment().isManuallyDeployed()) + controller.applications().applicationStore().putMeta(deployment, + clock.instant(), + applicationPackage.truncatedPackage().metaDataZip()); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 5c8f04e68ef..f0c278c51cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -67,11 +67,13 @@ import static java.util.stream.Collectors.toMap; */ public class ApplicationPackage { - private static final String trustedCertificatesFile = "security/clients.pem"; - private static final String buildMetaFile = "build-meta.json"; + static final String trustedCertificatesFile = "security/clients.pem"; + static final String buildMetaFile = "build-meta.json"; static final String deploymentFile = "deployment.xml"; - private static final String validationOverridesFile = "validation-overrides.xml"; + static final String validationOverridesFile = "validation-overrides.xml"; static final String servicesFile = "services.xml"; + static final Set<String> prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile); + private static Hasher hasher() { return Hashing.murmur3_128().newHasher(); } private final String bundleHash; @@ -101,7 +103,7 @@ public class ApplicationPackage { */ public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.files = new ZipArchiveCache(zippedContent, Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile)); + this.files = new ZipArchiveCache(zippedContent, prePopulated); Optional<DeploymentSpec> deploymentSpec = files.get(deploymentFile).map(bytes -> new String(bytes, UTF_8)).map(DeploymentSpec::fromXml); if (requireFiles && deploymentSpec.isEmpty()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java index d4f1e896818..d89d21bbee1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java @@ -1,11 +1,19 @@ package com.yahoo.vespa.hosted.controller.application.pkg; +import com.yahoo.security.X509CertificateUtils; + +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.security.cert.X509Certificate; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -13,7 +21,9 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; +import static java.io.OutputStream.nullOutputStream; import static java.lang.Math.min; +import static java.nio.charset.StandardCharsets.UTF_8; /** * Wraps a zipped application package stream. @@ -42,11 +52,33 @@ public class ApplicationPackageStream extends InputStream { private boolean done = false; private boolean closed = false; + public static Replacer addingCertificate(Optional<X509Certificate> certificate) { + return certificate.map(cert -> Replacer.of(Map.of(ApplicationPackage.trustedCertificatesFile, + trustBytes -> append(trustBytes, cert)))) + .orElse(Replacer.of(Map.of())); + } + + static InputStream append(InputStream trustBytes, X509Certificate cert) { + try { + List<X509Certificate> trusted = X509CertificateUtils.certificateListFromPem(new String(trustBytes.readAllBytes(), UTF_8)); + trusted.add(cert); + return new ByteArrayInputStream(X509CertificateUtils.toPem(trusted).getBytes(UTF_8)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + /** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */ public ApplicationPackageStream(InputStream in) { this(in, __ -> true, Map.of()); } + /** Stream that replaces the indicated entries, and copies all metadata files to its {@link #truncatedPackage()} when exhausted. */ + public ApplicationPackageStream(InputStream in, Replacer replacer) { + this(in, name -> ApplicationPackage.prePopulated.contains(name) || name.endsWith(".xml"), replacer); + } + /** Stream that replaces the indicated entries, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ public ApplicationPackageStream(InputStream in, Predicate<String> truncation, Map<String, UnaryOperator<InputStream>> replacements) { this(in, truncation, Replacer.of(replacements)); @@ -153,8 +185,15 @@ public class ApplicationPackageStream extends InputStream { } @Override - public void close() throws IOException { - if (closed != (closed = true)) inZip.close(); + public void close() { + if ( ! closed) try { + transferTo(nullOutputStream()); + inZip.close(); + closed = true; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } /** Replaces entries in a zip stream as they are encountered, then appends remaining entries at the end. */ @@ -167,14 +206,17 @@ public class ApplicationPackageStream extends InputStream { InputStream modify(String name, InputStream in); /** - * Removes entries whose name map to {@code in -> null}. - * Modifies entries present in both input and the map. - * Appends entries present exclusively in the map. - * Writes all other entries as they are. + * Wraps a map of fixed replacements, and: + * <ul> + * <li>Removes entries whose value is {@code null}.</li> + * <li>Modifies entries present in both input and the map.</li> + * <li>Appends entries present exclusively in the map.</li> + * <li>Writes all other entries as they are.</li> + * </ul> */ static Replacer of(Map<String, UnaryOperator<InputStream>> replacements) { return new Replacer() { - Map<String, UnaryOperator<InputStream>> remaining = new HashMap<>(replacements); + final Map<String, UnaryOperator<InputStream>> remaining = new HashMap<>(replacements); @Override public String next() { return remaining.isEmpty() ? null : remaining.keySet().iterator().next(); } @@ -187,32 +229,4 @@ public class ApplicationPackageStream extends InputStream { } - public static class LazyInputStream extends InputStream { - - private Supplier<InputStream> source; - private InputStream delegate; - - public LazyInputStream(Supplier<InputStream> source) { - this.source = source; - } - - private InputStream in() { - if (delegate == null) { - delegate = source.get(); - source = null; - } - return delegate; - } - - @Override public int read() throws IOException { return in().read(); } - @Override public int read(byte[] b, int off, int len) throws IOException { return in().read(b, off, len); } - @Override public long skip(long n) throws IOException { return in().skip(n); } - @Override public int available() throws IOException { return in().available(); } - @Override public void close() throws IOException { in().close(); } - @Override public synchronized void mark(int readlimit) { in().mark(readlimit); } - @Override public synchronized void reset() throws IOException { in().reset(); } - @Override public boolean markSupported() { return in().markSupported(); } - - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index 87d0e97fdf7..f85372d2867 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -18,15 +18,14 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.Replacer; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.InputStream; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -43,6 +42,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.UnaryOperator; import java.util.jar.JarInputStream; import java.util.jar.Manifest; import java.util.regex.Pattern; @@ -53,7 +53,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; -import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite; +import static java.io.InputStream.nullInputStream; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; @@ -71,32 +71,14 @@ public class TestPackage { static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - private final ApplicationPackage applicationPackage; + private final ApplicationPackageStream applicationPackageStream; private final X509Certificate certificate; - public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp, + public TestPackage(InputStream inZip, boolean isPublicSystem, RunId id, Testerapp testerApp, DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { - - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - Map<String, byte[]> entries = new HashMap<>(); - entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); - entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); - - entries.put(servicesFile, - servicesXml(! isPublicSystem, - certificateValidFrom != null, - hasLegacyTests(testPackage), - testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), - testerApp)); - - entries.put(deploymentFile, - deploymentXml(id.tester(), - spec.athenzDomain(), - spec.requireInstance(id.application().instance()) - .athenzService(id.type().zone().environment(), id.type().zone().region()))); - + KeyPair keyPair; if (certificateValidFrom != null) { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); this.certificate = X509CertificateBuilder.fromKeypair(keyPair, subject, @@ -105,26 +87,60 @@ public class TestPackage { SignatureAlgorithm.SHA512_WITH_RSA, BigInteger.valueOf(1)) .build(); - entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); - entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); } else { + keyPair = null; this.certificate = null; } + this.applicationPackageStream = new ApplicationPackageStream(inZip, __ -> false, new Replacer() { + + // Initially skips all declared entries, ensuring they're generated and appended after all input entries. + final Map<String, UnaryOperator<InputStream>> entries = new HashMap<>(); + final Map<String, UnaryOperator<InputStream>> replacements = new HashMap<>(); + boolean hasLegacyTests = false; + + @Override + public String next() { + if (entries.isEmpty()) return null; + String next = entries.keySet().iterator().next(); + replacements.put(next, entries.remove(next)); + return next; + } - ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000); - transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries); - this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); - } - - static boolean hasLegacyTests(byte[] testPackage) { - return ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() - .anyMatch(file -> file.name().startsWith("artifacts/") && file.name().endsWith("-tests.jar")); + @Override + public InputStream modify(String name, InputStream in) { + hasLegacyTests |= name.startsWith("artifacts/") && name.endsWith("-tests.jar"); + return entries.containsKey(name) ? null : replacements.get(name).apply(in); + } + { + // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. + entries.put("artifacts/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); + entries.put("tests/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); + + entries.put(servicesFile, + __ -> new ByteArrayInputStream(servicesXml( ! isPublicSystem, + certificateValidFrom != null, + hasLegacyTests, + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), + testerApp))); + + entries.put(deploymentFile, + __ -> new ByteArrayInputStream(deploymentXml(id.tester(), + spec.athenzDomain(), + spec.requireInstance(id.application().instance()) + .athenzService(id.type().zone().environment(), id.type().zone().region())))); + + if (certificate != null) { + entries.put("artifacts/key", __ -> new ByteArrayInputStream(KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8))); + entries.put("artifacts/cert", __ -> new ByteArrayInputStream(X509CertificateUtils.toPem(certificate).getBytes(UTF_8))); + } + } + }); } - public ApplicationPackage asApplicationPackage() { - return applicationPackage; + public ApplicationPackageStream asApplicationPackage() { + return applicationPackageStream; } public X509Certificate certificate() { @@ -207,7 +223,7 @@ public class TestPackage { return new TestSummary(problems, suites); } - public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { NodeResources nodeResources = spec.steps().stream() .filter(step -> step.concerns(zone.environment())) .findFirst() @@ -279,7 +295,7 @@ public class TestPackage { } /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - public static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { + static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<deployment version=\"1.0\" " + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e8c92d3e3f6..bbe31951ede 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -34,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -43,6 +44,7 @@ import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContex import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.io.PrintStream; import java.io.UncheckedIOException; import java.security.cert.CertificateExpiredException; @@ -194,10 +196,14 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deployTester(RunId id, DualLogger logger) { Version platform = testerPlatformVersion(id); logger.log("Deploying the tester container on platform " + platform + " ..."); - return deploy(() -> controller.applications().deployTester(id.tester(), - testerPackage(id), - id.type().zone(), - platform), + return deploy(() -> { + try (ApplicationPackageStream testerPackage = testerPackage(id)) { + return controller.applications().deployTester(id.tester(), + testerPackage, + id.type().zone(), + platform); + } + }, controller.jobController().run(id) .stepInfo(deployTester).get() .startTime().get(), @@ -926,11 +932,11 @@ public class InternalStepRunner implements StepRunner { } /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ - private ApplicationPackage testerPackage(RunId id) { + private ApplicationPackageStream testerPackage(RunId id) { RevisionId revision = controller.jobController().run(id).versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - byte[] testZip = controller.applications().applicationStore().getTester(id.application().tenant(), - id.application().application(), revision); + InputStream testZip = controller.applications().applicationStore().streamTester(id.application().tenant(), + id.application().application(), revision); boolean useTesterCertificate = useTesterCertificate(id); TestPackage testPackage = new TestPackage(testZip, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 8d465650eac..ed6311de0ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.io.LazyInputStream; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.LazyInputStream; import org.junit.jupiter.api.Test; import javax.security.auth.x500.X500Principal; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 07d9efdf8fc..9f258dff8d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -44,7 +44,9 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.time.Duration; import java.time.Instant; @@ -383,8 +385,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer deployment.instance().instance())); DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); - applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); + try { + applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage().readAllBytes()))); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); + deployment.endpointCertificateMetadata(); // Supplier with side effects >_< if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); diff --git a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java index ed3fee101ed..08ac3b54903 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java @@ -18,6 +18,7 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.HttpEntities; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; import org.apache.hc.core5.util.Timeout; +import sun.misc.Unsafe; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java index ea8328ed793..2e60260e4c0 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java @@ -80,7 +80,7 @@ public interface HttpClient extends Closeable { /** Sets the request body. */ default RequestBuilder body(HttpEntity entity) { if (entity.isRepeatable()) return body(() -> entity); - throw new IllegalArgumentException("entitiy must be repeatable, or a supplier must be used"); + throw new IllegalArgumentException("entity must be repeatable, or a supplier must be used"); } /** Sets the request body. */ diff --git a/vespajlib/src/main/java/com/yahoo/io/LazyInputStream.java b/vespajlib/src/main/java/com/yahoo/io/LazyInputStream.java new file mode 100644 index 00000000000..3ff7ada6b59 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/LazyInputStream.java @@ -0,0 +1,53 @@ +package com.yahoo.io; + +import java.io.IOException; +import java.io.InputStream; +import java.util.function.Supplier; + +/** + * Input stream wrapping an input stream supplier, which doesn't have content yet at declaration time. + * + * @author jonmv + */ +public class LazyInputStream extends InputStream { + + private Supplier<InputStream> source; + private InputStream delegate; + + public LazyInputStream(Supplier<InputStream> source) { + this.source = source; + } + + private InputStream in() { + if (delegate == null) { + delegate = source.get(); + source = null; + } + return delegate; + } + + @Override + public int read() throws IOException { return in().read(); } + + @Override + public int read(byte[] b, int off, int len) throws IOException { return in().read(b, off, len); } + + @Override + public long skip(long n) throws IOException { return in().skip(n); } + + @Override + public int available() throws IOException { return in().available(); } + + @Override + public void close() throws IOException { in().close(); } + + @Override + public synchronized void mark(int readlimit) { in().mark(readlimit); } + + @Override + public synchronized void reset() throws IOException { in().reset(); } + + @Override + public boolean markSupported() { return in().markSupported(); } + +} |