diff options
Diffstat (limited to 'controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg')
3 files changed, 100 insertions, 224 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index ab8f696492a..8cf861ff963 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -3,41 +3,18 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.io.LazyInputStream; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; import org.junit.jupiter.api.Test; -import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; -import java.security.KeyPair; -import java.security.cert.X509Certificate; import java.time.Instant; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import java.util.stream.IntStream; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage.filesZip; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream.addingCertificate; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -47,41 +24,35 @@ import static org.junit.jupiter.api.Assertions.fail; */ public class ApplicationPackageTest { - static final String deploymentXml = """ - <?xml version="1.0" encoding="UTF-8"?> - <deployment version="1.0"> - <test /> - <prod> - <parallel> - <region active="true">us-central-1</region> - </parallel> - </prod> - </deployment> - """; - - static final String servicesXml = """ - <services version='1.0' xmlns:deploy="vespa" xmlns:preprocess="properties"> - <preprocess:include file='jdisc.xml' /> - <content version='1.0' if='foo' /> - <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'> - <preprocess:include file='content/content.xml' /> - </content> - <preprocess:include file='not_found.xml' required='false' /> - </services> - """; + static final String deploymentXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<deployment version=\"1.0\">\n" + + " <test />\n" + + " <prod>\n" + + " <parallel>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>\n"; + + static final String servicesXml = "<services version='1.0' xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + + " <preprocess:include file='jdisc.xml' />\n" + + " <content version='1.0' if='foo' />\n" + + " <content version='1.0' id='foo' deploy:environment='staging prod' deploy:region='us-east-3 us-central-1'>\n" + + " <preprocess:include file='content/content.xml' />\n" + + " </content>\n" + + " <preprocess:include file='not_found.xml' required='false' />\n" + + "</services>\n"; private static final String jdiscXml = "<container id='stateless' version='1.0' />\n"; - private static final String contentXml = """ - <documents> - <document type="music.sd" mode="index" /> - </documents> - <preprocess:include file="nodes.xml" />"""; + private static final String contentXml = "<documents>\n" + + " <document type=\"music.sd\" mode=\"index\" />\n" + + "</documents>\n" + + "<preprocess:include file=\"nodes.xml\" />"; - private static final String nodesXml = """ - <nodes> - <node hostalias="node0" distribution-key="0" /> - </nodes>"""; + private static final String nodesXml = "<nodes>\n" + + " <node hostalias=\"node0\" distribution-key=\"0\" />\n" + + "</nodes>"; @Test void test_createEmptyForDeploymentRemoval() { @@ -96,22 +67,22 @@ public class ApplicationPackageTest { @Test void testMetaData() { - byte[] zip = filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8), - "jdisc.xml", jdiscXml.getBytes(UTF_8), - "content/content.xml", contentXml.getBytes(UTF_8), - "content/nodes.xml", nodesXml.getBytes(UTF_8), - "gurba", "gurba".getBytes(UTF_8))); + byte[] zip = ApplicationPackage.filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8), + "jdisc.xml", jdiscXml.getBytes(UTF_8), + "content/content.xml", contentXml.getBytes(UTF_8), + "content/nodes.xml", nodesXml.getBytes(UTF_8), + "gurba", "gurba".getBytes(UTF_8))); assertEquals(Map.of("services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml), - unzip(new ApplicationPackage(zip, false).metaDataZip())); + "jdisc.xml", jdiscXml, + "content/content.xml", contentXml, + "content/nodes.xml", nodesXml), + unzip(new ApplicationPackage(zip, false).metaDataZip())); } @Test void testMetaDataWithMissingFiles() { - byte[] zip = filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); + byte[] zip = ApplicationPackage.filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); try { new ApplicationPackage(zip, false).metaDataZip(); @@ -161,125 +132,15 @@ public class ApplicationPackageTest { assertEquals(originalPackage.bundleHash(), similarDeploymentXml.bundleHash()); } - static Map<String, String> unzip(byte[] zip) { - return ZipEntries.from(zip, __ -> true, 1 << 24, true) + private static Map<String, String> unzip(byte[] zip) { + return ZipEntries.from(zip, __ -> true, 1 << 10, true) .asList().stream() .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, - entry -> new String(entry.content().orElse(new byte[0]), UTF_8))); + entry -> new String(entry.contentOrThrow(), UTF_8))); } - private ApplicationPackage getApplicationZip(String path) throws IOException { + private ApplicationPackage getApplicationZip(String path) throws Exception { return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true); } - @Test - void test_replacement() throws IOException { - byte[] zip = zip(Map.of()); - List<X509Certificate> certificates = IntStream.range(0, 3) - .mapToObj(i -> { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - X500Principal subject = new X500Principal("CN=subject" + i); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.now(), - Instant.now().plusSeconds(1), - SignatureAlgorithm.SHA512_WITH_ECDSA, - BigInteger.valueOf(1)) - .build(); - }).toList(); - - assertEquals(List.of(), new ApplicationPackage(zip).trustedCertificates()); - for (int i = 0; i < certificates.size(); i++) { - InputStream in = new ByteArrayInputStream(zip); - zip = new ApplicationPackageStream(() -> in, () -> __ -> false, addingCertificate(Optional.of(certificates.get(i)))).zipStream().readAllBytes(); - assertEquals(certificates.subList(0, i + 1), new ApplicationPackage(zip).trustedCertificates()); - } - } - - static byte[] zip(Map<String, String> content) { - return filesZip(content.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(), - entry -> entry.getValue().getBytes(UTF_8)))); - } - - @Test - void testApplicationPackageStream() throws Exception { - Map<String, String> content = Map.of("deployment.xml", deploymentXml, - "services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "unused1.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml, - "gurba", "gurba"); - byte[] zip = zip(content); - assertEquals(content, unzip(zip)); - - ApplicationPackageStream identity = new ApplicationPackageStream(() -> new ByteArrayInputStream(zip)); - InputStream lazy = new LazyInputStream(() -> new ByteArrayInputStream(identity.truncatedPackage().zippedContent())); - assertEquals("must completely exhaust input before reading package", - assertThrows(IllegalStateException.class, identity::truncatedPackage).getMessage()); - - // Verify no content has changed when passing through the stream. - ByteArrayOutputStream out = new ByteArrayOutputStream(); - identity.zipStream().transferTo(out); - assertEquals(content, unzip(out.toByteArray())); - assertEquals(content, unzip(identity.truncatedPackage().zippedContent())); - assertEquals(content, unzip(lazy.readAllBytes())); - ApplicationPackage original = new ApplicationPackage(zip); - assertEquals(unzip(original.metaDataZip()), unzip(identity.truncatedPackage().metaDataZip())); - assertEquals(original.bundleHash(), identity.truncatedPackage().bundleHash()); - - // Change deployment.xml, remove unused1.xml and add unused2.xml - Map<String, UnaryOperator<InputStream>> replacements = Map.of("deployment.xml", in -> new SequenceInputStream(in, new ByteArrayInputStream("\n\n".getBytes(UTF_8))), - "unused1.xml", in -> null, - "unused2.xml", __ -> new ByteArrayInputStream(jdiscXml.getBytes(UTF_8))); - Predicate<String> truncation = name -> name.endsWith(".xml"); - ApplicationPackageStream modifier = new ApplicationPackageStream(() -> new ByteArrayInputStream(Arrays.copyOf(zip, zip.length)), () -> truncation, replacements); - out.reset(); - - InputStream partiallyRead = modifier.zipStream(); - assertEquals(15, partiallyRead.readNBytes(15).length); - - modifier.zipStream().transferTo(out); - - assertEquals(Map.of("deployment.xml", deploymentXml + "\n\n", - "services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "unused2.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml, - "gurba", "gurba"), - unzip(out.toByteArray())); - - assertEquals(Map.of("deployment.xml", deploymentXml + "\n\n", - "services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "unused2.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml), - unzip(modifier.truncatedPackage().zippedContent())); - - // Compare retained metadata for an updated original package, and the truncated package of the modifier. - assertEquals(unzip(new ApplicationPackage(zip(Map.of("deployment.xml", deploymentXml + "\n\n", // Expected to change. - "services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "unused1.xml", jdiscXml, // Irrelevant. - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml, - "gurba", "gurba"))).metaDataZip()), - unzip(modifier.truncatedPackage().metaDataZip())); - - assertArrayEquals(modifier.zipStream().readAllBytes(), - modifier.zipStream().readAllBytes()); - - ByteArrayOutputStream byteAtATime = new ByteArrayOutputStream(); - try (InputStream stream = modifier.zipStream()) { - for (int b; (b = stream.read()) != -1; ) byteAtATime.write(b); - assertArrayEquals(modifier.zipStream().readAllBytes(), - byteAtATime.toByteArray()); - } - - assertEquals(modifier.zipStream().readAllBytes().length, - 15 + partiallyRead.readAllBytes().length); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java index 6da8db1c259..bff0ccc8ae1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -1,24 +1,18 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.TestSummary; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig.Steprunner.Testerapp; import org.junit.jupiter.api.Test; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; @@ -26,11 +20,9 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.staging_setup; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Suite.system; -import static com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageTest.unzip; import static com.yahoo.vespa.hosted.controller.application.pkg.TestPackage.validateTests; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author jonmv @@ -85,15 +77,15 @@ public class TestPackageTest { @Test void testBundleValidation() throws IOException { byte[] testZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "StagingSetup", "ProductionTest"), - "artifacts/key", new byte[0])); + "artifacts/key", new byte[0])); TestSummary summary = validateTests(List.of(system), testZip); assertEquals(List.of(system, staging_setup, production), summary.suites()); assertEquals(List.of("test package contains 'artifacts/key'; this conflicts with credentials used to run tests in Vespa Cloud", - "test package has staging setup, so it should also include staging tests", - "test package has production tests, but no production tests are declared in deployment.xml", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); + "test package has staging setup, so it should also include staging tests", + "test package has production tests, but no production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); } @Test @@ -103,47 +95,20 @@ public class TestPackageTest { assertEquals(List.of(staging, production), summary.suites()); assertEquals(List.of("test package has staging tests, so it should also include staging setup", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); } @Test void testBasicTestsValidation() { byte[] testZip = ApplicationPackage.filesZip(Map.of("tests/staging-test/foo.json", new byte[0], - "tests/staging-setup/foo.json", new byte[0])); + "tests/staging-setup/foo.json", new byte[0])); TestSummary summary = validateTests(List.of(system, production), testZip); assertEquals(List.of(staging_setup, staging), summary.suites()); assertEquals(List.of("test package has no system tests, but <test /> is declared in deployment.xml", - "test package has no production tests, but production tests are declared in deployment.xml", - "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), - summary.problems()); - } - - @Test - void testTestPacakgeAssembly() throws IOException { - byte[] bundleZip = ApplicationPackage.filesZip(Map.of("components/foo-tests.jar", testsJar("SystemTest", "ProductionTest"), - "artifacts/key", new byte[0])); - TestPackage bundleTests = new TestPackage(() -> new ByteArrayInputStream(bundleZip), - false, - new RunId(ApplicationId.defaultId(), JobType.dev("abc"), 123), - new Testerapp.Builder().tenantCdBundle("foo").runtimeProviderClass("bar").build(), - DeploymentSpec.fromXml(""" - <deployment> - <test /> - </deployment> - """), - null, - null); - - Map<String, String> bundlePackage = unzip(bundleTests.asApplicationPackage().zipStream().readAllBytes()); - bundlePackage.keySet().removeIf(name -> name.startsWith("tests/.ignore") || name.startsWith("artifacts/.ignore")); - assertEquals(Set.of("deployment.xml", - "services.xml", - "components/foo-tests.jar", - "artifacts/key"), - bundlePackage.keySet()); - assertEquals(Map.of(), - unzip(bundleTests.asApplicationPackage().truncatedPackage().zippedContent())); + "test package has no production tests, but production tests are declared in deployment.xml", + "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"), + summary.problems()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java new file mode 100644 index 00000000000..37062e1002b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.jupiter.api.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author mpolden + */ +public class ZipEntriesTest { + + @Test + void test_replacement() { + ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); + List<X509Certificate> certificates = IntStream.range(0, 3) + .mapToObj(i -> { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=subject" + i); + return X509CertificateBuilder.fromKeypair(keyPair, + subject, + Instant.now(), + Instant.now().plusSeconds(1), + SignatureAlgorithm.SHA512_WITH_ECDSA, + BigInteger.valueOf(1)) + .build(); + }) + .collect(Collectors.toUnmodifiableList()); + + assertEquals(List.of(), applicationPackage.trustedCertificates()); + for (int i = 0; i < certificates.size(); i++) { + applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); + assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); + } + } + +} |