diff options
8 files changed, 32 insertions, 24 deletions
diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 05143bfef9f..64ca1e522f5 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -20,6 +20,10 @@ configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" fileReferencesDir string default="var/db/vespa/filedistribution/" +# Application package +# The maximum decompressed size of an application package, in bytes. Defaults to 8 GB +maxApplicationPackageSize long default=8589934592 + # Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 7a754dd84cd..cb1c1a461e3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1053,7 +1053,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = - CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { + CompressedApplicationInputStream.createFromCompressedStream(in, contentType, configserverConfig.maxApplicationPackageSize())) { return decompressApplication(application, tempDir); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 6f141e3e6ad..443ab47e786 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -39,11 +39,12 @@ public class CompressedApplicationInputStream implements AutoCloseable { * * @param is the input stream containing the compressed files. * @param contentType the content type for determining what kind of compressed stream should be used. + * @param maxSizeInBytes the maximum allowed size of the decompressed content * @return An instance of an unpacked application. */ - public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType) { + public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType, long maxSizeInBytes) { try { - Options options = Options.standard().allowDotSegment(true); + Options options = Options.standard().maxSize(maxSizeInBytes).allowDotSegment(true); switch (contentType) { case ApplicationApiHandler.APPLICATION_X_GZIP: return new CompressedApplicationInputStream(ArchiveStreamReader.ofTarGzip(is, options)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index 1a8b36ee19f..c8953d5996c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -49,9 +49,11 @@ public class ApplicationApiHandler extends SessionHandler { public final static String MULTIPART_PARAMS = "prepareParams"; public final static String MULTIPART_APPLICATION_PACKAGE = "applicationPackage"; public final static String contentTypeHeader = "Content-Type"; + private final TenantRepository tenantRepository; private final Duration zookeeperBarrierTimeout; private final Zone zone; + private final long maxApplicationPackageSize; @Inject public ApplicationApiHandler(Context ctx, @@ -61,6 +63,7 @@ public class ApplicationApiHandler extends SessionHandler { super(ctx, applicationRepository); this.tenantRepository = applicationRepository.tenantRepository(); this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); + this.maxApplicationPackageSize = configserverConfig.maxApplicationPackageSize(); this.zone = zone; } @@ -85,14 +88,14 @@ public class ApplicationApiHandler extends SessionHandler { log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8)); prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout); Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); - compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType()); + compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType(), maxApplicationPackageSize); } catch (IOException e) { log.log(Level.WARNING, "Unable to parse multipart in deploy", e); throw new BadRequestException("Request contains invalid data"); } } else { prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); + compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader), maxApplicationPackageSize); } PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 61642d33e62..5ee6c7d7507 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -108,7 +108,7 @@ public class CompressedApplicationInputStreamTest { public void require_that_gnu_tared_file_can_be_unpacked() throws IOException, InterruptedException { File gzFile = createTarGz("src/test/resources/deploy/validapp"); assertTrue(gzFile.exists()); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(gzFile), "application/x-gzip"); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(gzFile), "application/x-gzip", Long.MAX_VALUE); File outApp = unpacked.decompress(); assertTestApp(outApp); } @@ -168,11 +168,11 @@ public class CompressedApplicationInputStreamTest { } private static CompressedApplicationInputStream streamFromZip(File zipFile) { - return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(zipFile), "application/zip")); + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(zipFile), "application/zip", Long.MAX_VALUE)); } private static CompressedApplicationInputStream streamFromTarGz(File tarFile) { - return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(tarFile), "application/x-gzip")); + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(tarFile), "application/x-gzip", Long.MAX_VALUE)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index e35cb55b87a..d10c5cb3e1a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -58,8 +58,8 @@ public class ZipEntries { public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { Options options = Options.standard() .pathPredicate(entryNameMatcher) - .sizeLimit(2 * (long) Math.pow(1024, 3)) // 2 GB - .entrySizeLimit(maxEntrySizeInBytes) + .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB + .maxEntrySize(maxEntrySizeInBytes) .truncateEntry(!throwIfEntryExceedsMaxSize); List<ZipEntryWithContent> entries = new ArrayList<>(); try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zip), options)) { diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java index 748db06c70d..30dbd946155 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java @@ -65,8 +65,8 @@ public class ArchiveStreamReader implements AutoCloseable { while ((read = archiveInputStream.read(buffer)) != -1) { totalRead += read; size += read; - if (totalRead > options.sizeLimit) throw new IllegalArgumentException("Total size of archive exceeds size limit"); - if (read > options.entrySizeLimit) { + if (totalRead > options.maxSize) throw new IllegalArgumentException("Total size of archive exceeds size limit"); + if (read > options.maxEntrySize) { if (!options.truncateEntry) throw new IllegalArgumentException("Size of entry " + path + " exceeded entry size limit"); } else { outputStream.write(buffer, 0, read); @@ -156,8 +156,8 @@ public class ArchiveStreamReader implements AutoCloseable { /** Options for reading entries of an archive */ public static class Options { - private long sizeLimit = 8 * (long) Math.pow(1024, 3); // 8 GB - private long entrySizeLimit = Long.MAX_VALUE; + private long maxSize = 8 * (long) Math.pow(1024, 3); // 8 GB + private long maxEntrySize = Long.MAX_VALUE; private boolean truncateEntry = false; private boolean allowDotSegment = false; private Predicate<String> pathPredicate = (path) -> true; @@ -169,15 +169,15 @@ public class ArchiveStreamReader implements AutoCloseable { return new Options(); } - /** Set the total size limit when decompressing entries. Default is 8 GB */ - public Options sizeLimit(long limit) { - this.sizeLimit = limit; + /** Set the maximum total size of decompressed entries. Default is 8 GB */ + public Options maxSize(long size) { + this.maxSize = size; return this; } - /** Set the size limit of a decompressed entry. Default is no limit */ - public Options entrySizeLimit(long limit) { - this.entrySizeLimit = limit; + /** Set the maximum size a decompressed entry. Default is no limit */ + public Options maxEntrySize(long size) { + this.maxEntrySize = size; return this; } diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java index 847ab4124d3..a288b9f0c55 100644 --- a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java +++ b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java @@ -39,7 +39,7 @@ class ArchiveStreamReaderTest { @Test void entry_size_limit() { Map<String, String> entries = Map.of("foo.xml", "foobar"); - Options options = Options.standard().pathPredicate("foo.xml"::equals).entrySizeLimit(1); + Options options = Options.standard().pathPredicate("foo.xml"::equals).maxEntrySize(1); try { readAll(zip(entries), options); fail("Expected exception"); @@ -48,7 +48,7 @@ class ArchiveStreamReaderTest { entries = Map.of("foo.xml", "foobar", "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit ); - Map<String, String> extracted = readAll(zip(entries), options.entrySizeLimit(10)); + Map<String, String> extracted = readAll(zip(entries), options.maxEntrySize(10)); assertEquals(Map.of("foo.xml", "foobar"), extracted); } @@ -56,7 +56,7 @@ class ArchiveStreamReaderTest { void size_limit() { Map<String, String> entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); try { - readAll(zip(entries), Options.standard().sizeLimit(4)); + readAll(zip(entries), Options.standard().maxSize(4)); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} } @@ -75,7 +75,7 @@ class ArchiveStreamReaderTest { "services.xml", false ); - Options options = Options.standard().entrySizeLimit(1024); + Options options = Options.standard().maxEntrySize(1024); tests.forEach((name, expectException) -> { try { readAll(zip(Map.of(name, "foo")), options.pathPredicate(name::equals)); |