From 9665a3d5e1f171ea18054d25a957ca188449aa0e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 25 Nov 2022 21:35:12 +0100 Subject: Exclude onnxruntime --- config-model-fat/pom.xml | 1 - 1 file changed, 1 deletion(-) (limited to 'config-model-fat/pom.xml') diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 9fdca334876..87f82fd7bab 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -201,7 +201,6 @@ com.google.inject:guice:jar:no_aop:*:* com.google.j2objc:j2objc-annotations:*:* com.google.protobuf:protobuf-java:*:* - com.microsoft.onnxruntime:onnxruntime:*:* com.sun.activation:javax.activation:*:* com.sun.xml.bind:jaxb-core:*:* com.sun.xml.bind:jaxb-impl:*:* -- cgit v1.2.3 From af5dab0c2363156ffcc782140b22870597fab41b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 25 Nov 2022 23:13:58 +0100 Subject: Exclude commons-math3 in config-model-fat --- config-model-fat/pom.xml | 1 - fat-model-dependencies/pom.xml | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'config-model-fat/pom.xml') diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 87f82fd7bab..cff981c5d6e 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -216,7 +216,6 @@ org.antlr:antlr4-runtime:*:* org.apache.commons:commons-compress:*:* org.apache.commons:commons-exec:*:* - org.apache.commons:commons-math3:*:* org.apache.felix:org.apache.felix.framework:*:* org.apache.opennlp:opennlp-tools:*:* org.bouncycastle:bcpkix-jdk18on:*:* diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 446840e6a80..078bf852356 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -75,7 +75,7 @@ ${project.version} - + com.microsoft.onnxruntime onnxruntime @@ -161,6 +161,11 @@ com.ibm.icu icu4j + + + org.apache.commons + commons-math3 + -- cgit v1.2.3 From 9b11ce7f3b9080c6c3e640e694b253a1122e8bfa Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Nov 2022 16:04:20 +0100 Subject: Move ArchiveStreamReader and its large dependency from vespajlib to application-model. This avoids it being pulled in to the config-model-fat that make the install larger than necessary. --- application-model/pom.xml | 10 + .../yahoo/vespa/archive/ArchiveStreamReader.java | 216 +++++++++++++++++++++ .../vespa/archive/ArchiveStreamReaderTest.java | 131 +++++++++++++ cloud-tenant-base-dependencies-enforcer/pom.xml | 1 - config-model-fat/pom.xml | 1 - .../CompressedApplicationInputStream.java | 4 +- container-dev/pom.xml | 4 - container-test/pom.xml | 5 - .../application/pkg/ApplicationPackage.java | 8 +- .../controller/application/pkg/ZipEntries.java | 15 +- filedistribution/pom.xml | 7 +- vespajlib/pom.xml | 4 - .../com/yahoo/compress/ArchiveStreamReader.java | 216 --------------------- .../yahoo/compress/ArchiveStreamReaderTest.java | 131 ------------- 14 files changed, 371 insertions(+), 382 deletions(-) create mode 100644 application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java create mode 100644 application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java delete mode 100644 vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java delete mode 100644 vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java (limited to 'config-model-fat/pom.xml') diff --git a/application-model/pom.xml b/application-model/pom.xml index 2143f3a5ffd..f81c4ea4b62 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -21,6 +21,11 @@ ${jackson2.version} provided + + org.apache.commons + commons-compress + provided + com.yahoo.vespa vespajlib @@ -39,6 +44,11 @@ ${project.version} provided + + org.junit.jupiter + junit-jupiter + test + diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java new file mode 100644 index 00000000000..87665efc1ef --- /dev/null +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -0,0 +1,216 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.archive; + +import com.yahoo.path.Path; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.function.Predicate; +import java.util.zip.GZIPInputStream; + +/** + * Helper class for safely reading files from a compressed archive. + * + * @author mpolden + */ +public class ArchiveStreamReader implements AutoCloseable { + + private final ArchiveInputStream archiveInputStream; + private final Options options; + + private long totalRead = 0; + private long entriesRead = 0; + + private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { + this.archiveInputStream = Objects.requireNonNull(archiveInputStream); + this.options = Objects.requireNonNull(options); + } + + /** Create reader for an inputStream containing a tar.gz file */ + public static ArchiveStreamReader ofTarGzip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new TarArchiveInputStream(Exceptions.uncheck(() -> new GZIPInputStream(inputStream))), options); + } + + /** Create reader for an inputStream containing a ZIP file */ + public static ArchiveStreamReader ofZip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new ZipArchiveInputStream(inputStream), options); + } + + /** + * Read the next file in this archive and write it to given outputStream. Returns information about the read archive + * file, or null if there are no more files to read. + */ + public ArchiveFile readNextTo(OutputStream outputStream) { + ArchiveEntry entry; + try { + while ((entry = archiveInputStream.getNextEntry()) != null) { + Path path = Path.fromString(requireNormalized(entry.getName(), options.allowDotSegment)); + if (isSymlink(entry)) throw new IllegalArgumentException("Archive entry " + path + " is a symbolic link, which is unsupported"); + if (entry.isDirectory()) continue; + if (!options.pathPredicate.test(path.toString())) continue; + if (++entriesRead > options.maxEntries) throw new IllegalArgumentException("Attempted to read more entries than entry limit of " + options.maxEntries); + + long size = 0; + byte[] buffer = new byte[2048]; + int read; + while ((read = archiveInputStream.read(buffer)) != -1) { + totalRead += read; + size += read; + if (totalRead > options.maxSize) throw new IllegalArgumentException("Total size of archive exceeds size limit of " + options.maxSize + " bytes"); + if (read > options.maxEntrySize) { + if (!options.truncateEntry) throw new IllegalArgumentException("Size of entry " + path + " exceeded entry size limit of " + options.maxEntrySize + " bytes"); + } else { + outputStream.write(buffer, 0, read); + } + } + return new ArchiveFile(path, crc32(entry), size); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return null; + } + + @Override + public void close() { + Exceptions.uncheck(archiveInputStream::close); + } + + /** Information about a file extracted from a compressed archive */ + public static class ArchiveFile { + + private final Path path; + private final OptionalLong crc32; + private final long size; + + public ArchiveFile(Path name, OptionalLong crc32, long size) { + this.path = Objects.requireNonNull(name); + this.crc32 = Objects.requireNonNull(crc32); + if (crc32.isPresent()) { + requireNonNegative("crc32", crc32.getAsLong()); + } + this.size = requireNonNegative("size", size); + } + + /** The path of this file inside its containing archive */ + public Path path() { + return path; + } + + /** The CRC-32 checksum of this file, if any */ + public OptionalLong crc32() { + return crc32; + } + + /** The decompressed size of this file */ + public long size() { + return size; + } + + } + + /** Get the CRC-32 checksum of given archive entry, if any */ + private static OptionalLong crc32(ArchiveEntry entry) { + long crc32 = -1; + if (entry instanceof ZipArchiveEntry) { + crc32 = ((ZipArchiveEntry) entry).getCrc(); + } + return crc32 > -1 ? OptionalLong.of(crc32) : OptionalLong.empty(); + } + + private static boolean isSymlink(ArchiveEntry entry) { + // Symlinks inside ZIP files are not part of the ZIP spec and are only supported by some implementations, such + // as Info-ZIP. + // + // Commons Compress only has limited support for symlinks as they are only detected when the ZIP file is read + // through org.apache.commons.compress.archivers.zip.ZipFile. This is not the case in this class, because it must + // support reading ZIP files from generic input streams. The check below thus always returns false. + if (entry instanceof ZipArchiveEntry zipEntry) return zipEntry.isUnixSymlink(); + if (entry instanceof TarArchiveEntry tarEntry) return tarEntry.isSymbolicLink(); + throw new IllegalArgumentException("Unsupported archive entry " + entry.getClass().getSimpleName() + ", cannot check for symbolic link"); + } + + private static String requireNormalized(String name, boolean allowDotSegment) { + for (var part : name.split("/")) { + if (part.isEmpty() || (!allowDotSegment && part.equals(".")) || part.equals("..")) { + throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); + } + } + return name; + } + + private static long requireNonNegative(String field, long n) { + if (n < 0) throw new IllegalArgumentException(field + " cannot be negative, got " + n); + return n; + } + + /** Options for reading entries of an archive */ + public static class Options { + + private long maxSize = 8 * (long) Math.pow(1024, 3); // 8 GB + private long maxEntrySize = Long.MAX_VALUE; + private long maxEntries = Long.MAX_VALUE; + private boolean truncateEntry = false; + private boolean allowDotSegment = false; + private Predicate pathPredicate = (path) -> true; + + private Options() {} + + /** Returns the standard set of read options */ + public static Options standard() { + return new Options(); + } + + /** Set the maximum total size of decompressed entries. Default is 8 GB */ + public Options maxSize(long size) { + this.maxSize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum size a decompressed entry. Default is no limit */ + public Options maxEntrySize(long size) { + this.maxEntrySize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum number of entries to decompress. Default is no limit */ + public Options maxEntries(long count) { + this.maxEntries = requireNonNegative("count", count); + return this; + } + + /** + * Set whether to truncate the content of an entry exceeding the configured size limit, instead of throwing. + * Default is to throw. + */ + public Options truncateEntry(boolean truncate) { + this.truncateEntry = truncate; + return this; + } + + /** Set a predicate that an entry path must match in order to be extracted. Default is to extract all entries */ + public Options pathPredicate(Predicate predicate) { + this.pathPredicate = predicate; + return this; + } + + /** Set whether to allow single-dot segments in entry paths. Default is false */ + public Options allowDotSegment(boolean allow) { + this.allowDotSegment = allow; + return this; + } + + } + +} diff --git a/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java new file mode 100644 index 00000000000..78ff2a805e5 --- /dev/null +++ b/application-model/src/test/java/com/yahoo/vespa/archive/ArchiveStreamReaderTest.java @@ -0,0 +1,131 @@ +package com.yahoo.vespa.archive; + +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author mpolden + */ +class ArchiveStreamReaderTest { + + @Test + void reading() { + Map zipFiles = Map.of("foo", "contents of foo", + "bar", "contents of bar", + "baz", "0".repeat(2049)); + Map zipContents = new HashMap<>(zipFiles); + zipContents.put("dir/", ""); // Directories are always ignored + Map extracted = readAll(zip(zipContents), Options.standard()); + assertEquals(zipFiles, extracted); + } + + @Test + void entry_size_limit() { + Map entries = Map.of("foo.xml", "foobar"); + Options options = Options.standard().pathPredicate("foo.xml"::equals).maxEntrySize(1); + try { + readAll(zip(entries), options); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + entries = Map.of("foo.xml", "foobar", + "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit + ); + Map extracted = readAll(zip(entries), options.maxEntrySize(10)); + assertEquals(Map.of("foo.xml", "foobar"), extracted); + } + + @Test + void size_limit() { + Map entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxSize(4)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void entry_limit() { + Map entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxEntries(1)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void paths() { + Map tests = Map.of( + "../../services.xml", true, + "/../.././services.xml", true, + "./application/././services.xml", true, + "application//services.xml", true, + "artifacts/", false, // empty dir + "services..xml", false, + "application/services.xml", false, + "components/foo-bar-deploy.jar", false, + "services.xml", false + ); + + Options options = Options.standard().maxEntrySize(1024); + tests.forEach((name, expectException) -> { + try { + readAll(zip(Map.of(name, "foo")), options.pathPredicate(name::equals)); + assertFalse(expectException, "Expected exception for '" + name + "'"); + } catch (IllegalArgumentException ignored) { + assertTrue(expectException, "Unexpected exception for '" + name + "'"); + } + }); + } + + private static Map readAll(InputStream inputStream, Options options) { + ArchiveStreamReader reader = ArchiveStreamReader.ofZip(inputStream, options); + ArchiveStreamReader.ArchiveFile file; + Map entries = new HashMap<>(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.put(file.path().toString(), baos.toString(StandardCharsets.UTF_8)); + baos.reset(); + } + return entries; + } + + private static InputStream zip(Map entries) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipArchiveOutputStream archiveOutputStream = null; + try { + archiveOutputStream = new ZipArchiveOutputStream(baos); + for (var kv : entries.entrySet()) { + String entryName = kv.getKey(); + String contents = kv.getValue(); + ZipArchiveEntry entry = new ZipArchiveEntry(entryName); + archiveOutputStream.putArchiveEntry(entry); + archiveOutputStream.write(contents.getBytes(StandardCharsets.UTF_8)); + archiveOutputStream.closeArchiveEntry(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + if (archiveOutputStream != null) Exceptions.uncheck(archiveOutputStream::close); + } + return new ByteArrayInputStream(baos.toByteArray()); + } + +} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 5e439a870f6..ec422ba3c4c 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -186,7 +186,6 @@ org.antlr:antlr-runtime:3.5.2:test org.antlr:antlr4-runtime:4.9.3:test org.apache.commons:commons-exec:1.3:test - org.apache.commons:commons-compress:1.21:test org.apache.commons:commons-math3:3.6.1:test org.apache.felix:org.apache.felix.framework:${felix.version}:test org.apache.felix:org.apache.felix.log:1.0.1:test diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 9fdca334876..d58bf0fc39f 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -215,7 +215,6 @@ net.openhft:zero-allocation-hashing:*:* org.antlr:antlr-runtime:*:* org.antlr:antlr4-runtime:*:* - org.apache.commons:commons-compress:*:* org.apache.commons:commons-exec:*:* org.apache.commons:commons-math3:*:* org.apache.felix:org.apache.felix.framework:*:* diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 01dd47765d2..3bad813ef4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 85c6371d59c..13ff9de57aa 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -163,10 +163,6 @@ net.openhft zero-allocation-hashing - - org.apache.commons - commons-compress - org.lz4 lz4-java diff --git a/container-test/pom.xml b/container-test/pom.xml index 7dcbc794b77..103c118b083 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -99,11 +99,6 @@ aircompressor compile - - org.apache.commons - commons-compress - compile - org.json diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 53c78d7c8ec..8dbec87828a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -5,9 +5,9 @@ import com.google.common.hash.Funnel; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.yahoo.component.Version; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.file.NoSuchFileException; @@ -38,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index 185c97f866e..d22a41f74a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -1,27 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; -import com.yahoo.compress.ArchiveStreamReader; -import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; -import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; /** * A list of entries read from a ZIP archive, and their contents. diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 7916698b62c..97b17cc00d8 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -22,7 +22,7 @@ com.yahoo.vespa container-apache-http-client-bundle ${project.version} - compile + provided com.yahoo.vespa @@ -34,11 +34,13 @@ com.yahoo.vespa vespajlib ${project.version} + provided com.yahoo.vespa jrt ${project.version} + provided com.yahoo.vespa @@ -55,6 +57,7 @@ com.google.guava guava + provided junit @@ -64,6 +67,7 @@ org.apache.commons commons-compress + provided io.airlift @@ -72,6 +76,7 @@ com.fasterxml.jackson.core jackson-databind + provided org.slf4j diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 4c57b615c16..d903fb5ec0d 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -35,10 +35,6 @@ io.airlift aircompressor - - org.apache.commons - commons-compress - net.openhft zero-allocation-hashing diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java deleted file mode 100644 index f8faf655415..00000000000 --- a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.compress; - -import com.yahoo.path.Path; -import com.yahoo.yolean.Exceptions; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.util.Objects; -import java.util.OptionalLong; -import java.util.function.Predicate; -import java.util.zip.GZIPInputStream; - -/** - * Helper class for safely reading files from a compressed archive. - * - * @author mpolden - */ -public class ArchiveStreamReader implements AutoCloseable { - - private final ArchiveInputStream archiveInputStream; - private final Options options; - - private long totalRead = 0; - private long entriesRead = 0; - - private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { - this.archiveInputStream = Objects.requireNonNull(archiveInputStream); - this.options = Objects.requireNonNull(options); - } - - /** Create reader for an inputStream containing a tar.gz file */ - public static ArchiveStreamReader ofTarGzip(InputStream inputStream, Options options) { - return new ArchiveStreamReader(new TarArchiveInputStream(Exceptions.uncheck(() -> new GZIPInputStream(inputStream))), options); - } - - /** Create reader for an inputStream containing a ZIP file */ - public static ArchiveStreamReader ofZip(InputStream inputStream, Options options) { - return new ArchiveStreamReader(new ZipArchiveInputStream(inputStream), options); - } - - /** - * Read the next file in this archive and write it to given outputStream. Returns information about the read archive - * file, or null if there are no more files to read. - */ - public ArchiveFile readNextTo(OutputStream outputStream) { - ArchiveEntry entry; - try { - while ((entry = archiveInputStream.getNextEntry()) != null) { - Path path = Path.fromString(requireNormalized(entry.getName(), options.allowDotSegment)); - if (isSymlink(entry)) throw new IllegalArgumentException("Archive entry " + path + " is a symbolic link, which is unsupported"); - if (entry.isDirectory()) continue; - if (!options.pathPredicate.test(path.toString())) continue; - if (++entriesRead > options.maxEntries) throw new IllegalArgumentException("Attempted to read more entries than entry limit of " + options.maxEntries); - - long size = 0; - byte[] buffer = new byte[2048]; - int read; - while ((read = archiveInputStream.read(buffer)) != -1) { - totalRead += read; - size += read; - if (totalRead > options.maxSize) throw new IllegalArgumentException("Total size of archive exceeds size limit of " + options.maxSize + " bytes"); - if (read > options.maxEntrySize) { - if (!options.truncateEntry) throw new IllegalArgumentException("Size of entry " + path + " exceeded entry size limit of " + options.maxEntrySize + " bytes"); - } else { - outputStream.write(buffer, 0, read); - } - } - return new ArchiveFile(path, crc32(entry), size); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return null; - } - - @Override - public void close() { - Exceptions.uncheck(archiveInputStream::close); - } - - /** Information about a file extracted from a compressed archive */ - public static class ArchiveFile { - - private final Path path; - private final OptionalLong crc32; - private final long size; - - public ArchiveFile(Path name, OptionalLong crc32, long size) { - this.path = Objects.requireNonNull(name); - this.crc32 = Objects.requireNonNull(crc32); - if (crc32.isPresent()) { - requireNonNegative("crc32", crc32.getAsLong()); - } - this.size = requireNonNegative("size", size); - } - - /** The path of this file inside its containing archive */ - public Path path() { - return path; - } - - /** The CRC-32 checksum of this file, if any */ - public OptionalLong crc32() { - return crc32; - } - - /** The decompressed size of this file */ - public long size() { - return size; - } - - } - - /** Get the CRC-32 checksum of given archive entry, if any */ - private static OptionalLong crc32(ArchiveEntry entry) { - long crc32 = -1; - if (entry instanceof ZipArchiveEntry) { - crc32 = ((ZipArchiveEntry) entry).getCrc(); - } - return crc32 > -1 ? OptionalLong.of(crc32) : OptionalLong.empty(); - } - - private static boolean isSymlink(ArchiveEntry entry) { - // Symlinks inside ZIP files are not part of the ZIP spec and are only supported by some implementations, such - // as Info-ZIP. - // - // Commons Compress only has limited support for symlinks as they are only detected when the ZIP file is read - // through org.apache.commons.compress.archivers.zip.ZipFile. This is not the case in this class, because it must - // support reading ZIP files from generic input streams. The check below thus always returns false. - if (entry instanceof ZipArchiveEntry zipEntry) return zipEntry.isUnixSymlink(); - if (entry instanceof TarArchiveEntry tarEntry) return tarEntry.isSymbolicLink(); - throw new IllegalArgumentException("Unsupported archive entry " + entry.getClass().getSimpleName() + ", cannot check for symbolic link"); - } - - private static String requireNormalized(String name, boolean allowDotSegment) { - for (var part : name.split("/")) { - if (part.isEmpty() || (!allowDotSegment && part.equals(".")) || part.equals("..")) { - throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); - } - } - return name; - } - - private static long requireNonNegative(String field, long n) { - if (n < 0) throw new IllegalArgumentException(field + " cannot be negative, got " + n); - return n; - } - - /** Options for reading entries of an archive */ - public static class Options { - - private long maxSize = 8 * (long) Math.pow(1024, 3); // 8 GB - private long maxEntrySize = Long.MAX_VALUE; - private long maxEntries = Long.MAX_VALUE; - private boolean truncateEntry = false; - private boolean allowDotSegment = false; - private Predicate pathPredicate = (path) -> true; - - private Options() {} - - /** Returns the standard set of read options */ - public static Options standard() { - return new Options(); - } - - /** Set the maximum total size of decompressed entries. Default is 8 GB */ - public Options maxSize(long size) { - this.maxSize = requireNonNegative("size", size); - return this; - } - - /** Set the maximum size a decompressed entry. Default is no limit */ - public Options maxEntrySize(long size) { - this.maxEntrySize = requireNonNegative("size", size); - return this; - } - - /** Set the maximum number of entries to decompress. Default is no limit */ - public Options maxEntries(long count) { - this.maxEntries = requireNonNegative("count", count); - return this; - } - - /** - * Set whether to truncate the content of an entry exceeding the configured size limit, instead of throwing. - * Default is to throw. - */ - public Options truncateEntry(boolean truncate) { - this.truncateEntry = truncate; - return this; - } - - /** Set a predicate that an entry path must match in order to be extracted. Default is to extract all entries */ - public Options pathPredicate(Predicate predicate) { - this.pathPredicate = predicate; - return this; - } - - /** Set whether to allow single-dot segments in entry paths. Default is false */ - public Options allowDotSegment(boolean allow) { - this.allowDotSegment = allow; - return this; - } - - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java deleted file mode 100644 index b7f019282b7..00000000000 --- a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java +++ /dev/null @@ -1,131 +0,0 @@ -package com.yahoo.compress; - -import com.yahoo.compress.ArchiveStreamReader.Options; -import com.yahoo.yolean.Exceptions; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author mpolden - */ -class ArchiveStreamReaderTest { - - @Test - void reading() { - Map zipFiles = Map.of("foo", "contents of foo", - "bar", "contents of bar", - "baz", "0".repeat(2049)); - Map zipContents = new HashMap<>(zipFiles); - zipContents.put("dir/", ""); // Directories are always ignored - Map extracted = readAll(zip(zipContents), Options.standard()); - assertEquals(zipFiles, extracted); - } - - @Test - void entry_size_limit() { - Map entries = Map.of("foo.xml", "foobar"); - Options options = Options.standard().pathPredicate("foo.xml"::equals).maxEntrySize(1); - try { - readAll(zip(entries), options); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - - entries = Map.of("foo.xml", "foobar", - "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit - ); - Map extracted = readAll(zip(entries), options.maxEntrySize(10)); - assertEquals(Map.of("foo.xml", "foobar"), extracted); - } - - @Test - void size_limit() { - Map entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); - try { - readAll(zip(entries), Options.standard().maxSize(4)); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - } - - @Test - void entry_limit() { - Map entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); - try { - readAll(zip(entries), Options.standard().maxEntries(1)); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - } - - @Test - void paths() { - Map tests = Map.of( - "../../services.xml", true, - "/../.././services.xml", true, - "./application/././services.xml", true, - "application//services.xml", true, - "artifacts/", false, // empty dir - "services..xml", false, - "application/services.xml", false, - "components/foo-bar-deploy.jar", false, - "services.xml", false - ); - - Options options = Options.standard().maxEntrySize(1024); - tests.forEach((name, expectException) -> { - try { - readAll(zip(Map.of(name, "foo")), options.pathPredicate(name::equals)); - assertFalse(expectException, "Expected exception for '" + name + "'"); - } catch (IllegalArgumentException ignored) { - assertTrue(expectException, "Unexpected exception for '" + name + "'"); - } - }); - } - - private static Map readAll(InputStream inputStream, Options options) { - ArchiveStreamReader reader = ArchiveStreamReader.ofZip(inputStream, options); - ArchiveStreamReader.ArchiveFile file; - Map entries = new HashMap<>(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - while ((file = reader.readNextTo(baos)) != null) { - entries.put(file.path().toString(), baos.toString(StandardCharsets.UTF_8)); - baos.reset(); - } - return entries; - } - - private static InputStream zip(Map entries) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ZipArchiveOutputStream archiveOutputStream = null; - try { - archiveOutputStream = new ZipArchiveOutputStream(baos); - for (var kv : entries.entrySet()) { - String entryName = kv.getKey(); - String contents = kv.getValue(); - ZipArchiveEntry entry = new ZipArchiveEntry(entryName); - archiveOutputStream.putArchiveEntry(entry); - archiveOutputStream.write(contents.getBytes(StandardCharsets.UTF_8)); - archiveOutputStream.closeArchiveEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } finally { - if (archiveOutputStream != null) Exceptions.uncheck(archiveOutputStream::close); - } - return new ByteArrayInputStream(baos.toByteArray()); - } - -} -- cgit v1.2.3 From 604bc68ae38c4784383d9e62ceb11fb3f0554262 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Nov 2022 18:01:38 +0100 Subject: commons-exec not needed in config-model-fat --- config-model-fat/pom.xml | 1 - fat-model-dependencies/pom.xml | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'config-model-fat/pom.xml') diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index cff981c5d6e..842fcb37c31 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -215,7 +215,6 @@ org.antlr:antlr-runtime:*:* org.antlr:antlr4-runtime:*:* org.apache.commons:commons-compress:*:* - org.apache.commons:commons-exec:*:* org.apache.felix:org.apache.felix.framework:*:* org.apache.opennlp:opennlp-tools:*:* org.bouncycastle:bcpkix-jdk18on:*:* diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 078bf852356..9f279e2731d 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -100,6 +100,12 @@ com.yahoo.vespa vespajlib ${project.version} + + + org.apache.commons + commons-exec + + com.yahoo.vespa -- cgit v1.2.3 From ad0445fd54145f555482595709c870fac9b26d45 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Nov 2022 18:56:24 +0100 Subject: Exclude jna from config-model-fat --- config-model-fat/pom.xml | 1 - fat-model-dependencies/pom.xml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'config-model-fat/pom.xml') diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 842fcb37c31..adf9fa94b5d 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -210,7 +210,6 @@ io.prometheus:simpleclient_common:*:* javax.inject:javax.inject:*:* javax.servlet:javax.servlet-api:*:* - net.java.dev.jna:jna:*:* net.openhft:zero-allocation-hashing:*:* org.antlr:antlr-runtime:*:* org.antlr:antlr4-runtime:*:* diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 9f279e2731d..cd5471e5088 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -105,6 +105,10 @@ org.apache.commons commons-exec + + net.java.dev.jna + jna + -- cgit v1.2.3