diff options
Diffstat (limited to 'controller-server/src/main')
7 files changed, 120 insertions, 378 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e09e1f04b8e..34a7ae89dd2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.log.LogLevel; import com.yahoo.text.Text; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -38,6 +39,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; @@ -56,7 +58,6 @@ import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; @@ -78,7 +79,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; -import java.io.ByteArrayInputStream; import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Clock; @@ -87,6 +87,7 @@ import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -97,7 +98,6 @@ import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -489,6 +489,9 @@ public class ApplicationController { DeploymentId deployment = new DeploymentId(job.application(), zone); try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { + Set<ContainerEndpoint> containerEndpoints; + Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + Run run = controller.jobController().last(job) .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); @@ -497,32 +500,30 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); - ApplicationPackageStream applicationPackage = new ApplicationPackageStream(() -> applicationStore.stream(deployment, revision), - ApplicationPackageStream.addingCertificate(run.testerCertificate())); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); + AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); Instance instance; - Set<ContainerEndpoint> containerEndpoints; try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); instance = application.get().require(job.application().instance()); + if ( ! applicationPackage.trustedCertificates().isEmpty() + && run.testerCertificate().isPresent()) + applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); + + endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec()); + containerEndpoints = controller.routing().of(deployment).prepare(application); } // Release application lock while doing the deployment, which is a lengthy task. - Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata = () -> { - try (Mutex lock = lock(applicationId)) { - Optional<EndpointCertificateMetadata> data = endpointCertificates.getMetadata(instance, zone, applicationPackage.truncatedPackage().deploymentSpec()); - data.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); - return data; - } - }; - // Carry out deployment without holding the application lock. DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, endpointCertificateMetadata, run.isDryRun()); + endpointCertificateMetadata.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); // Record the quota usage for this application var quotaUsage = deploymentQuotaUsage(zone, job.application()); @@ -543,10 +544,8 @@ public class ApplicationController { .distinct() .collect(Collectors.toList())) .orElseGet(List::of); - if (warnings.isEmpty()) - controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); - else - controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); + else controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); } lockApplicationOrThrow(applicationId, application -> @@ -607,23 +606,23 @@ public class ApplicationController { /** Deploy a system application to given zone */ public DeploymentResult deploySystemApplicationPackage(SystemApplication application, ZoneId zone, Version version) { if (application.hasApplicationPackage()) { - ApplicationPackageStream applicationPackage = new ApplicationPackageStream( - () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) + ApplicationPackage applicationPackage = new ApplicationPackage( + artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); - return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), Optional::empty, false); + return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), false); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } } /** Deploys the given tester application to the given zone. */ - public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), Optional::empty, false); + public DeploymentResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { + return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), false); } - private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackageStream applicationPackage, + private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackage applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, - Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, + Optional<EndpointCertificateMetadata> endpointCertificateMetadata, boolean dryRun) { DeploymentId deployment = new DeploymentId(application, zone); try { @@ -639,8 +638,13 @@ public class ApplicationController { .filter(tenant-> tenant instanceof AthenzTenant) .map(tenant -> ((AthenzTenant)tenant).domain()); - Supplier<Quota> deploymentQuota = () -> DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), - asList(application.tenant()), application, zone, applicationPackage.truncatedPackage().deploymentSpec()); + if (zone.environment().isManuallyDeployed()) + controller.applications().applicationStore().putMeta(deployment, + clock.instant(), + applicationPackage.metaDataZip()); + + Quota deploymentQuota = DeploymentQuotaCalculator.calculate(billingController.getQuota(application.tenant()), + asList(application.tenant()), application, zone, applicationPackage.deploymentSpec()); List<TenantSecretStore> tenantSecretStores = controller.tenants() .get(application.tenant()) @@ -650,9 +654,9 @@ public class ApplicationController { List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() .map(SupportAccessGrant::certificate) .collect(toList()); - Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); + Optional<CloudAccount> cloudAccount = decideCloudAccountOf(deployment, applicationPackage.deploymentSpec()); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage::zipStream, platform, + configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage.zippedContent(), platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun)); @@ -661,12 +665,7 @@ public class ApplicationController { } finally { // Even if prepare fails, routing configuration may need to be updated if ( ! application.instance().isTester()) { - controller.routing().of(deployment).configure(applicationPackage.truncatedPackage().deploymentSpec()); - if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(deployment, - clock.instant(), - applicationPackage.truncatedPackage().metaDataZip()); - + controller.routing().of(deployment).configure(applicationPackage.deploymentSpec()); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 53c78d7c8ec..b99d825a779 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -59,19 +59,19 @@ import static java.util.stream.Collectors.toMap; * A representation of the content of an application package. * Only meta-data content can be accessed as anything other than compressed data. * A package is identified by a hash of the content. - * + * + * This is immutable. + * * @author bratseth * @author jonmv */ public class ApplicationPackage { - static final String trustedCertificatesFile = "security/clients.pem"; - static final String buildMetaFile = "build-meta.json"; + private static final String trustedCertificatesFile = "security/clients.pem"; + private static final String buildMetaFile = "build-meta.json"; static final String deploymentFile = "deployment.xml"; - static final String validationOverridesFile = "validation-overrides.xml"; + private static final String validationOverridesFile = "validation-overrides.xml"; static final String servicesFile = "services.xml"; - static final Set<String> prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile); - private static Hasher hasher() { return Hashing.murmur3_128().newHasher(); } private final String bundleHash; @@ -101,7 +101,7 @@ public class ApplicationPackage { */ public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.files = new ZipArchiveCache(zippedContent, prePopulated); + this.files = new ZipArchiveCache(zippedContent, Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile)); Optional<DeploymentSpec> deploymentSpec = files.get(deploymentFile).map(bytes -> new String(bytes, UTF_8)).map(DeploymentSpec::fromXml); if (requireFiles && deploymentSpec.isEmpty()) @@ -122,6 +122,17 @@ public class ApplicationPackage { preProcessAndPopulateCache(); } + /** Returns a copy of this with the given certificate appended. */ + public ApplicationPackage withTrustedCertificate(X509Certificate certificate) { + List<X509Certificate> trustedCertificates = new ArrayList<>(this.trustedCertificates); + trustedCertificates.add(certificate); + byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8); + + ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length); + ZipEntries.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); + return new ApplicationPackage(modified.toByteArray()); + } + /** Hash of all files and settings that influence what is deployed to config servers. */ public String bundleHash() { return bundleHash; @@ -284,7 +295,7 @@ public class ApplicationPackage { private Map<Path, Optional<byte[]>> read(Collection<String> names) { var entries = ZipEntries.from(zip, - names::contains, + name -> names.contains(name), maxSize, true) .asList().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java deleted file mode 100644 index 3288759b174..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ /dev/null @@ -1,244 +0,0 @@ -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.X509CertificateUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -import static com.yahoo.security.X509CertificateUtils.certificateListFromPem; -import static java.io.OutputStream.nullOutputStream; -import static java.lang.Math.min; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Wraps a zipped application package stream. - * This allows replacing content as the input stream is read. - * This also retains a truncated {@link ApplicationPackage}, containing only the specified set of files, - * which can be accessed when this stream is fully exhausted. - * - * @author jonmv - */ -public class ApplicationPackageStream { - - private final byte[] inBuffer = new byte[1 << 16]; - private final ByteArrayOutputStream out = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream outZip = new ZipOutputStream(out); - private final ByteArrayOutputStream teeOut = new ByteArrayOutputStream(1 << 16); - private final ZipOutputStream teeZip = new ZipOutputStream(teeOut); - private final Replacer replacer; - private final Predicate<String> filter; - private final Supplier<InputStream> in; - - private ApplicationPackage ap = null; - private boolean done = false; - - public static Replacer addingCertificate(Optional<X509Certificate> certificate) { - return certificate.map(cert -> Replacer.of(Map.of(ApplicationPackage.trustedCertificatesFile, - trustBytes -> append(trustBytes, cert)))) - .orElse(Replacer.of(Map.of())); - } - - static InputStream append(InputStream trustIn, X509Certificate cert) { - try { - List<X509Certificate> trusted = trustIn == null ? new ArrayList<>() - : new ArrayList<>(certificateListFromPem(new String(trustIn.readAllBytes(), UTF_8))); - trusted.add(cert); - return new ByteArrayInputStream(X509CertificateUtils.toPem(trusted).getBytes(UTF_8)); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - /** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in) { - this(in, __ -> true, Map.of()); - } - - /** Stream that replaces the indicated entries, and copies all metadata files to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Replacer replacer) { - this(in, name -> ApplicationPackage.prePopulated.contains(name) || name.endsWith(".xml"), replacer); - } - - /** Stream that replaces the indicated entries, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Predicate<String> truncation, Map<String, UnaryOperator<InputStream>> replacements) { - this(in, truncation, Replacer.of(replacements)); - } - - /** Stream that uses the given replacer to modify content, and copies the filtered entries to its {@link #truncatedPackage()} when exhausted. */ - public ApplicationPackageStream(Supplier<InputStream> in, Predicate<String> truncation, Replacer replacer) { - this.in = in; - this.filter = truncation; - this.replacer = replacer; - } - - public InputStream zipStream() { - return new Stream(); - } - - /** Returns the application backed by only the files indicated by the truncation filter. Throws if not yet exhausted. */ - public ApplicationPackage truncatedPackage() { - if (ap == null) throw new IllegalStateException("must completely exhaust input before reading package"); - return ap; - } - - private class Stream extends InputStream { - - private final ZipInputStream inZip = new ZipInputStream(in.get()); - private byte[] currentOut = new byte[0]; - private InputStream currentIn = InputStream.nullInputStream(); - private boolean includeCurrent = false; - private int pos = 0; - private boolean closed = false; - - private void fill() throws IOException { - if (done) return; - while (out.size() == 0) { - // Exhaust current entry first. - int i, n = out.size(); - while (out.size() == 0 && (i = currentIn.read(inBuffer)) != -1) { - if (includeCurrent) teeZip.write(inBuffer, 0, i); - outZip.write(inBuffer, 0, i); - n += i; - } - - // Current entry exhausted, look for next. - if (n == 0) { - next(); - if (done) break; - } - } - - currentOut = out.toByteArray(); - out.reset(); - pos = 0; - } - - private void next() throws IOException { - if (includeCurrent) teeZip.closeEntry(); - outZip.closeEntry(); - - ZipEntry next = inZip.getNextEntry(); - String name; - InputStream content = null; - if (next == null) { - // We may still have replacements to fill in, but if we don't, we're done filling, forever! - name = replacer.next(); - if (name == null) { - outZip.close(); // This typically makes new output available, so must check for that after this. - teeZip.close(); - currentIn = nullInputStream(); - ap = new ApplicationPackage(teeOut.toByteArray()); - done = true; - return; - } - } - else { - name = next.getName(); - content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it. - } - - includeCurrent = filter.test(name); - currentIn = replacer.modify(name, content); - if (currentIn == null) { - currentIn = InputStream.nullInputStream(); - } - else { - if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name)); - outZip.putNextEntry(new ZipEntry(name)); - } - } - - @Override - public int read() throws IOException { - if (closed) throw new IOException("stream closed"); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - return 0xff & inBuffer[pos++]; - } - - @Override - public int read(byte[] out, int off, int len) throws IOException { - if (closed) throw new IOException("stream closed"); - if ((off | len | (off + len) | (out.length - (off + len))) < 0) throw new IndexOutOfBoundsException(); - if (pos == currentOut.length) { - fill(); - if (pos == currentOut.length) return -1; - } - int n = min(currentOut.length - pos, len); - System.arraycopy(currentOut, pos, out, off, n); - pos += n; - return n; - } - - @Override - public int available() throws IOException { - return pos == currentOut.length && done ? 0 : 1; - } - - @Override - public void close() { - if ( ! closed) try { - transferTo(nullOutputStream()); - inZip.close(); - closed = true; - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } - - /** Replaces entries in a zip stream as they are encountered, then appends remaining entries at the end. */ - public interface Replacer { - - /** Called when the entries of the original zip stream are exhausted. Return remaining names, or {@code null} when none left. */ - String next(); - - /** Modify content for a given name; return {@code null} for removal; in is {@code null} for entries not present in the input. */ - InputStream modify(String name, InputStream in); - - /** - * Wraps a map of fixed replacements, and: - * <ul> - * <li>Removes entries whose value is {@code null}.</li> - * <li>Modifies entries present in both input and the map.</li> - * <li>Appends entries present exclusively in the map.</li> - * <li>Writes all other entries as they are.</li> - * </ul> - */ - static Replacer of(Map<String, UnaryOperator<InputStream>> replacements) { - return new Replacer() { - final Map<String, UnaryOperator<InputStream>> remaining = new HashMap<>(replacements); - @Override public String next() { - return remaining.isEmpty() ? null : remaining.keySet().iterator().next(); - } - @Override public InputStream modify(String name, InputStream in) { - UnaryOperator<InputStream> mapper = remaining.remove(name); - return mapper == null ? in : mapper.apply(in); - } - }; - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index f838e44bd02..5b20c57fcca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -18,14 +18,15 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.Replacer; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayInputStream; -import java.io.InputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -42,8 +43,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; import java.util.jar.JarInputStream; import java.util.jar.Manifest; import java.util.regex.Pattern; @@ -54,9 +53,8 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; -import static java.io.InputStream.nullInputStream; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.transferAndWrite; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.function.UnaryOperator.identity; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; @@ -73,14 +71,32 @@ public class TestPackage { static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); - private final ApplicationPackageStream applicationPackageStream; + private final ApplicationPackage applicationPackage; private final X509Certificate certificate; - public TestPackage(Supplier<InputStream> inZip, boolean isPublicSystem, RunId id, Testerapp testerApp, + public TestPackage(byte[] testPackage, boolean isPublicSystem, RunId id, Testerapp testerApp, DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { - KeyPair keyPair; + + // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. + Map<String, byte[]> entries = new HashMap<>(); + entries.put("artifacts/.ignore-" + UUID.randomUUID(), new byte[0]); + entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); + + entries.put(servicesFile, + servicesXml(! isPublicSystem, + certificateValidFrom != null, + hasLegacyTests(testPackage), + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), + testerApp)); + + entries.put(deploymentFile, + deploymentXml(id.tester(), + spec.athenzDomain(), + spec.requireInstance(id.application().instance()) + .athenzService(id.type().zone().environment(), id.type().zone().region()))); + if (certificateValidFrom != null) { - keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); X500Principal subject = new X500Principal("CN=" + id.tester().id().toFullString() + "." + id.type() + "." + id.number()); this.certificate = X509CertificateBuilder.fromKeypair(keyPair, subject, @@ -89,60 +105,26 @@ public class TestPackage { SignatureAlgorithm.SHA512_WITH_RSA, BigInteger.valueOf(1)) .build(); + entries.put("artifacts/key", KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8)); + entries.put("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); } else { - keyPair = null; this.certificate = null; } - this.applicationPackageStream = new ApplicationPackageStream(inZip, __ -> false, new Replacer() { - - // Initially skips all declared entries, ensuring they're generated and appended after all input entries. - final Map<String, UnaryOperator<InputStream>> entries = new HashMap<>(); - final Map<String, UnaryOperator<InputStream>> replacements = new HashMap<>(); - boolean hasLegacyTests = false; - - @Override - public String next() { - if (entries.isEmpty()) return null; - String next = entries.keySet().iterator().next(); - replacements.put(next, entries.remove(next)); - return next; - } - @Override - public InputStream modify(String name, InputStream in) { - hasLegacyTests |= name.startsWith("artifacts/") && name.endsWith("-tests.jar"); - return entries.containsKey(name) ? null : replacements.getOrDefault(name, identity()).apply(in); - } + ByteArrayOutputStream buffer = new ByteArrayOutputStream(testPackage.length + 10_000); + transferAndWrite(buffer, new ByteArrayInputStream(testPackage), entries); + this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); + } + + static boolean hasLegacyTests(byte[] testPackage) { + return ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() + .anyMatch(file -> file.name().startsWith("artifacts/") && file.name().endsWith("-tests.jar")); - { - // Copy contents of submitted application-test.zip, and ensure required directories exist within the zip. - entries.put("artifacts/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - entries.put("tests/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); - - entries.put(servicesFile, - __ -> new ByteArrayInputStream(servicesXml( ! isPublicSystem, - certificateValidFrom != null, - hasLegacyTests, - testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), - testerApp))); - - entries.put(deploymentFile, - __ -> new ByteArrayInputStream(deploymentXml(id.tester(), - spec.athenzDomain(), - spec.requireInstance(id.application().instance()) - .athenzService(id.type().zone().environment(), id.type().zone().region())))); - - if (certificate != null) { - entries.put("artifacts/key", __ -> new ByteArrayInputStream(KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8))); - entries.put("artifacts/cert", __ -> new ByteArrayInputStream(X509CertificateUtils.toPem(certificate).getBytes(UTF_8))); - } - } - }); } - public ApplicationPackageStream asApplicationPackage() { - return applicationPackageStream; + public ApplicationPackage asApplicationPackage() { + return applicationPackage; } public X509Certificate certificate() { @@ -225,7 +207,7 @@ public class TestPackage { return new TestSummary(problems, suites); } - static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + public static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { NodeResources nodeResources = spec.steps().stream() .filter(step -> step.concerns(zone.environment())) .findFirst() @@ -237,7 +219,7 @@ public class TestPackage { } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, + public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { int jdiscMemoryGb = 2; // 2Gb memory for tester application which uses Maven. int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); @@ -297,7 +279,7 @@ public class TestPackage { } /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ - static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { + public static byte[] deploymentXml(TesterId id, Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<deployment version=\"1.0\" " + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index 63915c5050f..6bbcd551924 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -15,7 +15,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; @@ -108,7 +107,7 @@ public class ZipEntries { } public String name() { return name; } - public byte[] contentOrThrow() { return content.orElseThrow(() -> new NoSuchElementException("'" + name + "' has no content")); } + public byte[] contentOrThrow() { return content.orElseThrow(); } public Optional<byte[]> content() { return content; } public long size() { return size; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 309cb1343f9..e8c92d3e3f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -34,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -44,7 +43,6 @@ import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContex import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.io.PrintStream; import java.io.UncheckedIOException; import java.security.cert.CertificateExpiredException; @@ -928,13 +926,14 @@ public class InternalStepRunner implements StepRunner { } /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ - private ApplicationPackageStream testerPackage(RunId id) { + private ApplicationPackage testerPackage(RunId id) { RevisionId revision = controller.jobController().run(id).versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); + byte[] testZip = controller.applications().applicationStore().getTester(id.application().tenant(), + id.application().application(), revision); boolean useTesterCertificate = useTesterCertificate(id); - TestPackage testPackage = new TestPackage(() -> controller.applications().applicationStore().streamTester(id.application().tenant(), - id.application().application(), revision), + TestPackage testPackage = new TestPackage(testZip, controller.system().isPublic(), id, controller.controllerConfig().steprunner().testerapp(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index f94bd51fe4c..08cf8d2e1c4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -207,36 +207,32 @@ public class JobController { return run; List<LogEntry> log; - Optional<Instant> deployedAt; + Instant deployedAt; Instant from; if ( ! run.id().type().isProduction()) { - deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime); - if (deployedAt.isPresent()) { - from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.get().minusSeconds(10); - log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() - .getLogs(new DeploymentId(id.application(), zone), - Map.of("from", Long.toString(from.toEpochMilli()))), - from); - } - else log = List.of(); + deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime).orElseThrow(); + from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); + log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.application(), zone), + Map.of("from", Long.toString(from.toEpochMilli()))), + from); } - else log = List.of(); + else + log = List.of(); if (id.type().isTest()) { - deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime); - if (deployedAt.isPresent()) { - from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.get().minusSeconds(10); - List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() - .getLogs(new DeploymentId(id.tester().id(), zone), - Map.of("from", Long.toString(from.toEpochMilli()))), - from); - - Instant justNow = controller.clock().instant().minusSeconds(2); - log = Stream.concat(log.stream(), testerLog.stream()) - .filter(entry -> entry.at().isBefore(justNow)) - .sorted(comparing(LogEntry::at)) - .toList(); - } + deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime).orElseThrow(); + from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); + List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.tester().id(), zone), + Map.of("from", Long.toString(from.toEpochMilli()))), + from); + + Instant justNow = controller.clock().instant().minusSeconds(2); + log = Stream.concat(log.stream(), testerLog.stream()) + .filter(entry -> entry.at().isBefore(justNow)) + .sorted(comparing(LogEntry::at)) + .collect(toUnmodifiableList()); } if (log.isEmpty()) return run; |