diff options
Diffstat (limited to 'controller-server/src/test')
5 files changed, 226 insertions, 108 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 8cf861ff963..71ce291df36 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -3,18 +3,40 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.io.LazyInputStream; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; import org.junit.jupiter.api.Test; +import javax.security.auth.x500.X500Principal; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.filesZip; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.addingCertificate; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -24,35 +46,41 @@ import static org.junit.jupiter.api.Assertions.fail; */ public class ApplicationPackageTest { - static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<deployment version=\"1.0\">\n" + - " <test />\n" + - " <prod>\n" + - " <parallel>\n" + - " <region active=\"true\">us-central-1</region>\n" + - " </parallel>\n" + - " </prod>\n" + - "</deployment>\n"; - - static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + - " <preprocess:include file='jdisc.xml' />\n" + - " <content version='1.0' if='foo' />\n" + - " <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" + - " <preprocess:include file='content/content.xml' />\n" + - " </content>\n" + - " <preprocess:include file='not_found.xml' required='false' />\n" + - "</services>\n"; + static final String deploymentXml = """ + <?xml version="1.0" encoding="UTF-8"?> + <deployment version="1.0"> + <test /> + <prod> + <parallel> + <region active="true">us-central-1</region> + </parallel> + </prod> + </deployment> + """; + + static final String servicesXml = """ + <services version='1.0' xmlns:deploy="vespa" xmlns:preprocess="properties"> + <preprocess:include file='jdisc.xml' /> + <content version='1.0' if='foo' /> + <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'> + <preprocess:include file='content/content.xml' /> + </content> + <preprocess:include file='not_found.xml' required='false' /> + </services> + """; private static final String jdiscXml = "<container id='stateless' version='1.0' />\n"; - private static final String contentXml = "<documents>\n" + - " <document type=\"music.sd\" mode=\"index\" />\n" + - "</documents>\n" + - "<preprocess:include file=\"nodes.xml\" />"; + private static final String contentXml = """ + <documents> + <document type="music.sd" mode="index" /> + </documents> + <preprocess:include file="nodes.xml" />"""; - private static final String nodesXml = "<nodes>\n" + - " <node hostalias=\"node0\" distribution-key=\"0\" />\n" + - "</nodes>"; + private static final String nodesXml = """ + <nodes> + <node hostalias="node0" distribution-key="0" /> + </nodes>"""; @Test void test_createEmptyForDeploymentRemoval() { @@ -67,22 +95,22 @@ public class ApplicationPackageTest { @Test void testMetaData() { - byte[] zip = ApplicationPackage.filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8), - "jdisc.xml", jdiscXml.getBytes(UTF_8), - "content/content.xml", contentXml.getBytes(UTF_8), - "content/nodes.xml", nodesXml.getBytes(UTF_8), - "gurba", "gurba".getBytes(UTF_8))); + byte[] zip = filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8), + "jdisc.xml", jdiscXml.getBytes(UTF_8), + "content/content.xml", contentXml.getBytes(UTF_8), + "content/nodes.xml", nodesXml.getBytes(UTF_8), + "gurba", "gurba".getBytes(UTF_8))); assertEquals(Map.of("services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml), - unzip(new ApplicationPackage(zip, false).metaDataZip())); + "jdisc.xml", jdiscXml, + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml), + unzip(new ApplicationPackage(zip, false).metaDataZip())); } @Test void testMetaDataWithMissingFiles() { - byte[] zip = ApplicationPackage.filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); + byte[] zip = filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); try { new ApplicationPackage(zip, false).metaDataZip(); @@ -132,15 +160,108 @@ public class ApplicationPackageTest { assertEquals(originalPackage.bundleHash(), similarDeploymentXml.bundleHash()); } - private static Map<String, String> unzip(byte[] zip) { - return ZipEntries.from(zip, __ -> true, 1 << 10, true) + static Map<String, String> unzip(byte[] zip) { + return ZipEntries.from(zip, __ -> true, 1 << 24, true) .asList().stream() .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, - entry -> new String(entry.contentOrThrow(), UTF_8))); + entry -> new String(entry.content().orElse(new byte[0]), UTF_8))); } - private ApplicationPackage getApplicationZip(String path) throws Exception { + private ApplicationPackage getApplicationZip(String path) throws IOException { return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true); } + @Test + void test_replacement() throws IOException { + byte[] zip = zip(Map.of()); + List<X509Certificate> certificates = IntStream.range(0, 3) + .mapToObj(i -> { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=subject" + i); + return X509CertificateBuilder.fromKeypair(keyPair, + subject, + Instant.now(), + Instant.now().plusSeconds(1), + SignatureAlgorithm.SHA512_WITH_ECDSA, + BigInteger.valueOf(1)) + .build(); + }).toList(); + + assertEquals(List.of(), new ApplicationPackage(zip).trustedCertificates()); + for (int i = 0; i < certificates.size(); i++) { + InputStream in = new ByteArrayInputStream(zip); + zip = new ApplicationPackageStream(() -> in, __ -> false, addingCertificate(Optional.of(certificates.get(i)))).zipStream().readAllBytes(); + assertEquals(certificates.subList(0, i + 1), new ApplicationPackage(zip).trustedCertificates()); + } + } + + static byte[] zip(Map<String, String> content) { + return filesZip(content.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(), + entry -> entry.getValue().getBytes(UTF_8)))); + } + + @Test + void testApplicationPackageStream() throws Exception { + Map<String, String> content = Map.of("deployment.xml", deploymentXml, + "services.xml", servicesXml, + "jdisc.xml", jdiscXml, + "unused1.xml", jdiscXml, + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml, + "gurba", "gurba"); + byte[] zip = zip(content); + assertEquals(content, unzip(zip)); + + ApplicationPackageStream identity = new ApplicationPackageStream(() -> new ByteArrayInputStream(zip)); + InputStream lazy = new LazyInputStream(() -> new ByteArrayInputStream(identity.truncatedPackage().zippedContent())); + assertEquals("must completely exhaust input before reading package", + assertThrows(IllegalStateException.class, identity::truncatedPackage).getMessage()); + + // Verify no content has changed when passing through the stream. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + identity.zipStream().transferTo(out); + assertEquals(content, unzip(out.toByteArray())); + assertEquals(content, unzip(identity.truncatedPackage().zippedContent())); + assertEquals(content, unzip(lazy.readAllBytes())); + ApplicationPackage original = new ApplicationPackage(zip); + assertEquals(unzip(original.metaDataZip()), unzip(identity.truncatedPackage().metaDataZip())); + assertEquals(original.bundleHash(), identity.truncatedPackage().bundleHash()); + + // Change deployment.xml, remove unused1.xml and add unused2.xml + Map<String, UnaryOperator<InputStream>> replacements = Map.of("deployment.xml", in -> new SequenceInputStream(in, new ByteArrayInputStream("\n\n".getBytes(UTF_8))), + "unused1.xml", in -> null, + "unused2.xml", __ -> new ByteArrayInputStream(jdiscXml.getBytes(UTF_8))); + Predicate<String> truncation = name -> name.endsWith(".xml"); + ApplicationPackageStream modifier = new ApplicationPackageStream(() -> new ByteArrayInputStream(zip), truncation, replacements); + out.reset(); + modifier.zipStream().transferTo(out); + + assertEquals(Map.of("deployment.xml", deploymentXml + "\n\n", + "services.xml", servicesXml, + "jdisc.xml", jdiscXml, + "unused2.xml", jdiscXml, + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml, + "gurba", "gurba"), + unzip(out.toByteArray())); + + assertEquals(Map.of("deployment.xml", deploymentXml + "\n\n", + "services.xml", servicesXml, + "jdisc.xml", jdiscXml, + "unused2.xml", jdiscXml, + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml), + unzip(modifier.truncatedPackage().zippedContent())); + + // Compare retained metadata for an updated original package, and the truncated package of the modifier. + assertEquals(unzip(new ApplicationPackage(zip(Map.of("deployment.xml", deploymentXml + "\n\n", // Expected to change. + "services.xml", servicesXml, + "jdisc.xml", jdiscXml, + "unused1.xml", jdiscXml, // Irrelevant. + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml, + "gurba", "gurba"))).metaDataZip()), + unzip(modifier.truncatedPackage().metaDataZip())); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java index bff0ccc8ae1..6da8db1c259 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -1,18 +1,24 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; +import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; @@ -20,9 +26,11 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; +import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest.unzip; import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.validateTests; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author jonmv @@ -77,15 +85,15 @@ public class TestPackageTest { @Test void testBundleValidation() throws IOException { byte[] testZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "StagingSetup", "ProductionTest"), - "artifacts/key", new byte[0])); + "artifacts/key", new byte[0])); TestSummary summary = validateTests(List.of(system), testZip); assertEquals(List.of(system, staging_setup, production), summary.suites()); assertEquals(List.of("test package contains 'artifacts/key'; this conflicts with credentials used to run tests in Vespa Cloud", - "test package has staging setup, so it should also include staging tests", - "test package has production tests, but no production tests are declared in deployment.xml", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); + "test package has staging setup, so it should also include staging tests", + "test package has production tests, but no production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); } @Test @@ -95,20 +103,47 @@ public class TestPackageTest { assertEquals(List.of(staging, production), summary.suites()); assertEquals(List.of("test package has staging tests, so it should also include staging setup", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); } @Test void testBasicTestsValidation() { byte[] testZip = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0], - "tests/staging-setup/foo.json", new byte[0])); + "tests/staging-setup/foo.json", new byte[0])); TestSummary summary = validateTests(List.of(system, production), testZip); assertEquals(List.of(staging_setup, staging), summary.suites()); assertEquals(List.of("test package has no system tests, but <test /> is declared in deployment.xml", - "test package has no production tests, but production tests are declared in deployment.xml", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); + "test package has no production tests, but production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); + } + + @Test + void testTestPacakgeAssembly() throws IOException { + byte[] bundleZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "ProductionTest"), + "artifacts/key", new byte[0])); + TestPackage bundleTests = new TestPackage(() -> new ByteArrayInputStream(bundleZip), + false, + new RunId(ApplicationId.defaultId(), JobType.dev("abc"), 123), + new Testerapp.Builder().tenantCdBundle("foo").runtimeProviderClass("bar").build(), + DeploymentSpec.fromXml(""" + <deployment> + <test /> + </deployment> + """), + null, + null); + + Map<String, String> bundlePackage = unzip(bundleTests.asApplicationPackage().zipStream().readAllBytes()); + bundlePackage.keySet().removeIf(name -> name.startsWith("tests/.ignore") || name.startsWith("artifacts/.ignore")); + assertEquals(Set.of("deployment.xml", + "services.xml", + "components/foo-tests.jar", + "artifacts/key"), + bundlePackage.keySet()); + assertEquals(Map.of(), + unzip(bundleTests.asApplicationPackage().truncatedPackage().zippedContent())); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java deleted file mode 100644 index 37062e1002b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import org.junit.jupiter.api.Test; - -import javax.security.auth.x500.X500Principal; -import java.math.BigInteger; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author mpolden - */ -public class ZipEntriesTest { - - @Test - void test_replacement() { - ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); - List<X509Certificate> certificates = IntStream.range(0, 3) - .mapToObj(i -> { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - X500Principal subject = new X500Principal("CN=subject" + i); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.now(), - Instant.now().plusSeconds(1), - SignatureAlgorithm.SHA512_WITH_ECDSA, - BigInteger.valueOf(1)) - .build(); - }) - .collect(Collectors.toUnmodifiableList()); - - assertEquals(List.of(), applicationPackage.trustedCertificates()); - for (int i = 0; i < certificates.size(); i++) { - applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); - assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); - } - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java index 8ed38761c95..e025a3bea4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationStoreMock.java @@ -12,6 +12,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.time.Instant; import java.util.Map; import java.util.NavigableMap; @@ -46,15 +48,14 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] get(DeploymentId deploymentId, RevisionId revisionId) { + public InputStream stream(DeploymentId deploymentId, RevisionId revisionId) { if ( ! revisionId.isProduction()) - return requireNonNull(devStore.get(deploymentId)); + return new ByteArrayInputStream(devStore.get(deploymentId)); TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(deploymentId.applicationId()); byte[] bytes = store.get(appId(tenantAndApplicationId.tenant(), tenantAndApplicationId.application())).get(revisionId); - if (bytes == null) - throw new NotExistsException("No " + revisionId + " found for " + tenantAndApplicationId); - return bytes; + if (bytes == null) throw new NotExistsException("No " + revisionId + " found for " + tenantAndApplicationId); + return new ByteArrayInputStream(bytes); } @Override @@ -96,8 +97,8 @@ public class ApplicationStoreMock implements ApplicationStore { } @Override - public byte[] getTester(TenantName tenant, ApplicationName application, RevisionId revision) { - return requireNonNull(store.get(testerId(tenant, application)).get(revision)); + public InputStream streamTester(TenantName tenant, ApplicationName application, RevisionId revision) { + return new ByteArrayInputStream(store.get(testerId(tenant, application)).get(revision)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 07d9efdf8fc..eaa178c9727 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -42,9 +42,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartF import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import wiremock.org.checkerframework.checker.units.qual.A; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.time.Duration; import java.time.Instant; @@ -376,6 +379,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentData deployment) { + ApplicationPackage appPackage; + try (InputStream in = deployment.applicationPackage()) { + appPackage = new ApplicationPackage(in.readAllBytes()); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } lastPrepareVersion = deployment.platform(); if (prepareException != null) prepareException.accept(ApplicationId.from(deployment.instance().tenant(), @@ -383,8 +393,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer deployment.instance().instance())); DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); - applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); + applications.put(id, new Application(id.applicationId(), lastPrepareVersion, appPackage)); ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); + deployment.endpointCertificateMetadata(); // Supplier with side effects >_< if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); |