diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2023-08-31 13:48:20 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-08-31 13:48:20 +0200 |
commit | d70ed3ee222320cff442c93878e6f83a73c4bd61 (patch) | |
tree | dd0055e2445f5922c4f146027b0b057aafe39b32 | |
parent | 3c419adc3fbf49fd44eaebe0591ebab0c84c1523 (diff) | |
parent | 91f667de1c3036ca4b3fa2835a3f61d2b459f63e (diff) |
Merge pull request #28320 from vespa-engine/jonmv/reapply-newest-cloud-account-for-tests
Jonmv/reapply newest cloud account for tests
9 files changed, 101 insertions, 29 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 3ec7f120726..46ba22af512 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -63,10 +63,10 @@ import static java.util.stream.Collectors.toMap; */ public class ApplicationPackage { + public static final String deploymentFile = "deployment.xml"; static final String trustedCertificatesDir = "security/"; static final String trustedCertificatesFile = trustedCertificatesDir + "clients.pem"; static final String buildMetaFile = "build-meta.json"; - static final String deploymentFile = "deployment.xml"; static final String validationOverridesFile = "validation-overrides.xml"; static final String servicesFile = "services.xml"; static final Set<String> prePopulated = Set.of(deploymentFile, validationOverridesFile, servicesFile, buildMetaFile, trustedCertificatesFile); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index 120c0a89f45..b4309e3aa00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -29,6 +29,7 @@ import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.io.InputStreamReader; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -36,8 +37,8 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -49,6 +50,8 @@ import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.jar.JarInputStream; import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.regex.Pattern; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.production; @@ -59,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPacka import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.servicesFile; import static java.io.InputStream.nullInputStream; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNullElse; import static java.util.function.UnaryOperator.identity; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; @@ -71,6 +75,8 @@ import static java.util.stream.Collectors.toList; */ public class TestPackage { + private static final Logger log = Logger.getLogger(TestPackage.class.getName()); + // Must match exactly the advertised resources of an AWS instance type. Also consider that the container // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). static final NodeResources DEFAULT_TESTER_RESOURCES_CLOUD = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); @@ -80,7 +86,7 @@ public class TestPackage { private final X509Certificate certificate; public TestPackage(Supplier<InputStream> inZip, boolean isPublicSystem, CloudName cloud, RunId id, Testerapp testerApp, - DeploymentSpec spec, Instant certificateValidFrom, Duration certificateValidDuration) { + DeploymentSpec fallbackSpec, Instant certificateValidFrom, Duration certificateValidDuration) { KeyPair keyPair; if (certificateValidFrom != null) { keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); @@ -97,14 +103,13 @@ public class TestPackage { keyPair = null; this.certificate = null; } - boolean isEnclave = isPublicSystem && - !spec.cloudAccount(cloud, id.application().instance(), id.type().zone()).isUnspecified(); this.applicationPackageStream = new ApplicationPackageStream(inZip, () -> name -> name.endsWith(".xml"), () -> new Replacer() { // Initially skips all declared entries, ensuring they're generated and appended after all input entries. - final Map<String, UnaryOperator<InputStream>> entries = new HashMap<>(); - final Map<String, UnaryOperator<InputStream>> replacements = new HashMap<>(); + final Map<String, UnaryOperator<InputStream>> entries = new LinkedHashMap<>(); + final Map<String, UnaryOperator<InputStream>> replacements = new LinkedHashMap<>(); boolean hasLegacyTests = false; + DeploymentSpec containedSpec; @Override public String next() { @@ -117,7 +122,13 @@ public class TestPackage { @Override public InputStream modify(String name, InputStream in) { hasLegacyTests |= name.startsWith("artifacts/") && name.endsWith("-tests.jar"); - return entries.containsKey(name) ? null : replacements.getOrDefault(name, identity()).apply(in); + + // Pick out the deployment.xml stored in the package, if any. + if (entries.containsKey(deploymentFile) && name.equals(deploymentFile)) + containedSpec = DeploymentSpec.fromXml(new InputStreamReader(in)); + + return entries.containsKey(name) ? null // Skip entry for now, as it will be appended later when we get here again after {@link #next()}. + : replacements.getOrDefault(name, identity()).apply(in); // Modify entry, if needed. } { @@ -126,14 +137,22 @@ public class TestPackage { entries.put("tests/.ignore-" + UUID.randomUUID(), __ -> nullInputStream()); entries.put(servicesFile, - __ -> new ByteArrayInputStream(servicesXml( ! isPublicSystem, - certificateValidFrom != null, - hasLegacyTests, - testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance()), isEnclave), - testerApp))); + __ -> { + DeploymentSpec spec = requireNonNullElse(containedSpec, fallbackSpec); + boolean isEnclave = isPublicSystem && ! spec.cloudAccount(cloud, id.application().instance(), id.type().zone()).isUnspecified(); + return new ByteArrayInputStream(servicesXml( ! isPublicSystem, + certificateValidFrom != null, + hasLegacyTests, + testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance()), isEnclave), + testerApp)); + }); entries.put(deploymentFile, - __ -> new ByteArrayInputStream(deploymentXml(id.tester(), id.application().instance(), cloud, id.type().zone(), spec))); + __ -> new ByteArrayInputStream(deploymentXml(id.tester(), + id.application().instance(), + cloud, + id.type().zone(), + requireNonNullElse(containedSpec, fallbackSpec)))); if (certificate != null) { entries.put("artifacts/key", __ -> new ByteArrayInputStream(KeyUtils.toPem(keyPair.getPrivate()).getBytes(UTF_8))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index d22a41f74a4..e034e9c7a33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -32,7 +32,7 @@ public class ZipEntries { Options options = Options.standard() .pathPredicate(entryNameMatcher) - .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB + .maxSize(2L << 30) // 2 GB .maxEntrySize(maxEntrySizeInBytes) .maxEntries(1024) .truncateEntry(!throwIfEntryExceedsMaxSize); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index f2357a49952..03489222922 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; @@ -75,6 +76,7 @@ import java.util.logging.Logger; import java.util.stream.Stream; import static com.yahoo.collections.Iterables.reversed; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.cancelled; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; @@ -89,6 +91,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialRe import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.temporal.ChronoUnit.SECONDS; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; @@ -577,7 +580,8 @@ public class JobController { id.application(), version.get().id(), submission.applicationPackage().zippedContent(), - submission.testPackage(), + withDeploymentSpec(submission.testPackage(), + submission.applicationPackage().deploymentSpec()), diff); applications.applicationStore().putMeta(id.tenant(), id.application(), @@ -599,6 +603,15 @@ public class JobController { return version.get(); } + static byte[] withDeploymentSpec(byte[] testZip, DeploymentSpec spec) { + ZipBuilder zip = new ZipBuilder(testZip.length + (1 << 12)); + try (zip) { + zip.add(testZip, name -> !name.equals(deploymentFile)); + zip.add(deploymentFile, spec.xmlForm().getBytes(UTF_8)); + } + return zip.toByteArray(); + } + private void validate(TenantAndApplicationId id, Submission submission) { controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.testPackage); controller.notificationsDb().removeNotification(NotificationSource.from(id), Type.submission); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java index a05a8cd753f..b5307b07603 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java @@ -5,6 +5,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.function.Predicate; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -25,9 +26,10 @@ public class ZipBuilder implements AutoCloseable { zipOutputStream = new ZipOutputStream(byteArrayOutputStream); } - public void add(byte[] zippedContent) { + public void add(byte[] zippedContent, Predicate<String> filter) { try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) { for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) { + if ( ! filter.test(entry.getName())) continue; zipOutputStream.putNextEntry(new ZipEntry(entry.getName())); zin.transferTo(zipOutputStream); zipOutputStream.closeEntry(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index f018b566674..910d526a2ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -166,7 +166,7 @@ public class ApplicationPackageTest { } } - static Map<String, String> unzip(byte[] zip) { + public static Map<String, String> unzip(byte[] zip) { return ZipEntries.from(zip, __ -> true, 1 << 24, true) .asList().stream() .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, @@ -181,7 +181,7 @@ public class ApplicationPackageTest { return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true, checkCertificateFile); } - static byte[] zip(Map<String, String> content) { + public static byte[] zip(Map<String, String> content) { return filesZip(content.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getBytes(UTF_8)))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java index 4c61fe7c77d..450b7f5e6cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -32,7 +32,9 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.deploymentFile; import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest.unzip; +import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.deploymentXml; import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.validateTests; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -125,19 +127,51 @@ public class TestPackageTest { } @Test - void testTestPackageAssembly() throws IOException { + void testTestPackageAssemblyWithFallbackSpec() throws IOException { + String deploymentSpec = """ + <deployment> + <test /> + </deployment> + """; byte[] bundleZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "ProductionTest"), "artifacts/key", new byte[0])); + TestPackage bundleTests = new TestPackage(() -> new ByteArrayInputStream(bundleZip), false, CloudName.DEFAULT, new RunId(ApplicationId.defaultId(), JobType.dev("abc"), 123), new Testerapp.Builder().tenantCdBundle("foo").runtimeProviderClass("bar").build(), - DeploymentSpec.fromXml(""" - <deployment> - <test /> - </deployment> - """), + DeploymentSpec.fromXml(deploymentSpec), + null, + null); + + Map<String, String> bundlePackage = unzip(bundleTests.asApplicationPackage().zipStream().readAllBytes()); + bundlePackage.keySet().removeIf(name -> name.startsWith("tests/.ignore") || name.startsWith("artifacts/.ignore")); + assertEquals(Set.of("deployment.xml", + "services.xml", + "components/foo-tests.jar", + "artifacts/key"), + bundlePackage.keySet()); + assertEquals(Set.of("deployment.xml", "services.xml"), + unzip(bundleTests.asApplicationPackage().truncatedPackage().zippedContent()).keySet()); + } + @Test + void testTestPackageAssembly() throws IOException { + String deploymentSpec = """ + <deployment> + <test /> + </deployment> + """; + byte[] bundleZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "ProductionTest"), + "artifacts/key", new byte[0], + deploymentFile, deploymentSpec.getBytes(UTF_8))); + + TestPackage bundleTests = new TestPackage(() -> new ByteArrayInputStream(bundleZip), + false, + CloudName.DEFAULT, + new RunId(ApplicationId.defaultId(), JobType.dev("abc"), 123), + new Testerapp.Builder().tenantCdBundle("foo").runtimeProviderClass("bar").build(), + DeploymentSpec.empty, // Will fail, unless contained spec is used. null, null); @@ -168,6 +202,7 @@ public class TestPackageTest { </prod> </deployment> """); + verifyAttributes("", 0, DEFAULT, ZoneId.from("test", "us-east-1"), spec); verifyAttributes("", 0, DEFAULT, ZoneId.from("staging", "us-east-2"), spec); verifyAttributes("", 0, DEFAULT, ZoneId.from("prod", "us-east-3"), spec); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java index 8fe48f805df..a6f7f45caa4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java @@ -40,7 +40,7 @@ public class ZipBuilderTest { } // Add the zipped data from zip1 to zip2 - zipBuilder2.add(zipBuilder1.toByteArray()); + zipBuilder2.add(zipBuilder1.toByteArray(), __ -> true); Map<String, String> actual = unzipToMap(zipBuilder2.toByteArray()); assertEquals(expected, actual); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 345f164f8da..d3386a3a2ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -47,11 +47,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; @@ -102,6 +102,7 @@ import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest.unzip; import static java.net.URLEncoder.encode; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; @@ -157,6 +158,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final UserId HOSTED_VESPA_OPERATOR = new UserId("johnoperator"); private static final OAuthCredentials OKTA_CREDENTIALS = OAuthCredentials.createForTesting("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.he0ErCNloe4J7Id0Ry2SEDg09lKkZkfsRiGsdX_vgEg", "okta-it"); + private static final byte[] testZip = ApplicationPackageTest.zip(Map.of("tests", "content")); private ContainerTester tester; private DeploymentTester deploymentTester; @@ -892,7 +894,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET).userIdentity(HOSTED_VESPA_OPERATOR).properties(Map.of("tests", "true")), (response) -> { assertEquals("attachment; filename=\"tenant1.application1-tests2.zip\"", response.getHeaders().getFirst("Content-Disposition")); - assertArrayEquals("content".getBytes(UTF_8), response.getBody()); + assertEquals(Map.of("tests", "content", "deployment.xml", packageWithService.deploymentSpec().xmlForm()), + unzip(response.getBody())); }, 200); @@ -1842,7 +1845,7 @@ public class ApplicationApiTest extends ControllerContainerTest { + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\"," + "\"description\":\"my best commit yet\",\"risk\":9001}") .addBytes(EnvironmentResource.APPLICATION_ZIP, applicationPackage.zippedContent()) - .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, "content".getBytes()); + .addBytes(EnvironmentResource.APPLICATION_TEST_ZIP, testZip); } private String deployOptions(Optional<ApplicationVersion> applicationVersion) { |