diff options
76 files changed, 1744 insertions, 480 deletions
diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index b1b03b60ce6..d53c2c94d5c 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -21,12 +21,12 @@ <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> - <version>3.5.0</version> + <version>3.8.5</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-archiver</artifactId> - <version>3.5.0</version> + <version>3.5.2</version> </dependency> <dependency> <groupId>org.apache.maven.plugin-tools</groupId> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java index d922b63c24c..bd6151aea9f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java @@ -35,7 +35,10 @@ public class TestBundleDependencyScopeTranslator implements Artifacts.ScopeTrans this.dependencyScopes = dependencyScopes; } - @Override public String scopeOf(Artifact artifact) { return Objects.requireNonNull(dependencyScopes.get(artifact)); } + @Override + public String scopeOf(Artifact artifact) { + return Objects.requireNonNull(dependencyScopes.get(artifact), () -> "Could not lookup scope for " + artifact); + } public static TestBundleDependencyScopeTranslator from(Map<String, Artifact> dependencies, String rawConfig) { List<DependencyOverride> dependencyOverrides = toDependencyOverrides(rawConfig); diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 21e5de31534..b5281050459 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -236,6 +236,7 @@ <include>org.antlr:antlr-runtime:3.5.2:jar:test</include> <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include> <include>org.apache.commons:commons-exec:1.3:jar:test</include> + <include>org.apache.commons:commons-compress:1.21:jar:test</include> <include>org.apache.commons:commons-math3:3.6.1:jar:test</include> <include>org.apache.httpcomponents.client5:httpclient5:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents.core5:httpcore5:${httpclient5.version}:jar:test</include> diff --git a/config-model/.gitignore b/config-model/.gitignore index 4cf50da0853..6edd041cbe8 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -5,3 +5,4 @@ /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ *.cfg.actual +/var/ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java index 47bcc64f663..8b8b9e5f40d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java @@ -13,6 +13,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; * * @author Gunnar Gauslaa Bergem */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class Clients extends ConfigModel { private static final long serialVersionUID = 1L; diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 05143bfef9f..64ca1e522f5 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -20,6 +20,10 @@ configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" fileReferencesDir string default="var/db/vespa/filedistribution/" +# Application package +# The maximum decompressed size of an application package, in bytes. Defaults to 8 GB +maxApplicationPackageSize long default=8589934592 + # Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 7a754dd84cd..cb1c1a461e3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1053,7 +1053,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = - CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { + CompressedApplicationInputStream.createFromCompressedStream(in, contentType, configserverConfig.maxApplicationPackageSize())) { return decompressApplication(application, tempDir); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 0672f13fd6a..443ab47e786 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,23 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.google.common.io.ByteStreams; +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.logging.Level; -import com.yahoo.vespa.config.server.http.BadRequestException; -import com.yahoo.vespa.config.server.http.InternalServerException; -import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; - import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; import static com.yahoo.yolean.Exceptions.uncheck; @@ -29,53 +27,43 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CompressedApplicationInputStream implements AutoCloseable { private static final Logger log = Logger.getLogger(CompressedApplicationInputStream.class.getPackage().getName()); - private final ArchiveInputStream ais; + + private final ArchiveStreamReader reader; + + private CompressedApplicationInputStream(ArchiveStreamReader reader) { + this.reader = reader; + } /** * Create an instance of a compressed application from an input stream. * * @param is the input stream containing the compressed files. * @param contentType the content type for determining what kind of compressed stream should be used. + * @param maxSizeInBytes the maximum allowed size of the decompressed content * @return An instance of an unpacked application. */ - public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType) { + public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType, long maxSizeInBytes) { try { - ArchiveInputStream ais = getArchiveInputStream(is, contentType); - return createFromCompressedStream(ais); - } catch (IOException e) { + Options options = Options.standard().maxSize(maxSizeInBytes).allowDotSegment(true); + switch (contentType) { + case ApplicationApiHandler.APPLICATION_X_GZIP: + return new CompressedApplicationInputStream(ArchiveStreamReader.ofTarGzip(is, options)); + case ApplicationApiHandler.APPLICATION_ZIP: + return new CompressedApplicationInputStream(ArchiveStreamReader.ofZip(is, options)); + default: + throw new BadRequestException("Unable to decompress"); + } + } catch (UncheckedIOException e) { throw new InternalServerException("Unable to create compressed application stream", e); } } - static CompressedApplicationInputStream createFromCompressedStream(ArchiveInputStream ais) { - return new CompressedApplicationInputStream(ais); - } - - private static ArchiveInputStream getArchiveInputStream(InputStream is, String contentTypeHeader) throws IOException { - ArchiveInputStream ais; - switch (contentTypeHeader) { - case ApplicationApiHandler.APPLICATION_X_GZIP: - ais = new TarArchiveInputStream(new GZIPInputStream(is)); - break; - case ApplicationApiHandler.APPLICATION_ZIP: - ais = new ZipArchiveInputStream(is); - break; - default: - throw new BadRequestException("Unable to decompress"); - } - return ais; - } - - private CompressedApplicationInputStream(ArchiveInputStream ais) { - this.ais = ais; - } - /** * Close this stream. * @throws IOException if the stream could not be closed */ public void close() throws IOException { - ais.close(); + reader.close(); } File decompress() throws IOException { @@ -83,45 +71,44 @@ public class CompressedApplicationInputStream implements AutoCloseable { } public File decompress(File dir) throws IOException { - decompressInto(dir); + decompressInto(dir.toPath()); dir = findActualApplicationDir(dir); return dir; } - private void decompressInto(File application) throws IOException { - log.log(Level.FINE, () -> "Application is in " + application.getAbsolutePath()); + private void decompressInto(Path dir) throws IOException { + if (!Files.isDirectory(dir)) throw new IllegalArgumentException("Not a directory: " + dir.toAbsolutePath()); + log.log(Level.FINE, () -> "Application is in " + dir.toAbsolutePath()); int entries = 0; - ArchiveEntry entry; - while ((entry = ais.getNextEntry()) != null) { - log.log(Level.FINE, "Unpacking %s", entry.getName()); - File outFile = new File(application, entry.getName()); - // FIXME/TODO: write more tests that break this logic. I have a feeling it is not very robust. - if (entry.isDirectory()) { - if (!(outFile.exists() && outFile.isDirectory())) { - log.log(Level.FINE, () -> "Creating dir: " + outFile.getAbsolutePath()); - boolean res = outFile.mkdirs(); - if (!res) { - log.log(Level.WARNING, "Could not create dir " + entry.getName()); - } - } - } else { - log.log(Level.FINE, () -> "Creating output file: " + outFile.getAbsolutePath()); - - // Create parent dir if necessary - String parent = outFile.getParent(); - new File(parent).mkdirs(); - - FileOutputStream fos = new FileOutputStream(outFile); - ByteStreams.copy(ais, fos); - fos.close(); + Path tmpFile = null; + OutputStream tmpStream = null; + try { + tmpFile = createTempFile(dir); + tmpStream = Files.newOutputStream(tmpFile); + ArchiveStreamReader.ArchiveFile file; + while ((file = reader.readNextTo(tmpStream)) != null) { + tmpStream.close(); + log.log(Level.FINE, "Creating output file: " + file.path()); + Path dstFile = dir.resolve(file.path().toString()).normalize(); + Files.createDirectories(dstFile.getParent()); + Files.move(tmpFile, dstFile); + tmpFile = createTempFile(dir); + tmpStream = Files.newOutputStream(tmpFile); + entries++; } - entries++; + } finally { + if (tmpStream != null) tmpStream.close(); + if (tmpFile != null) Files.deleteIfExists(tmpFile); } if (entries == 0) { - log.log(Level.WARNING, "Not able to read any entries from " + application.getName()); + log.log(Level.WARNING, "Not able to decompress any entries to " + dir); } } + private static Path createTempFile(Path applicationDir) throws IOException { + return Files.createTempFile(applicationDir, "application", null); + } + private File findActualApplicationDir(File application) { // If application is in e.g. application/, use that as root for UnpackedApplication // TODO: Vespa 8: Remove application/ directory support @@ -131,4 +118,5 @@ public class CompressedApplicationInputStream implements AutoCloseable { } return application; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index 1a8b36ee19f..c8953d5996c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -49,9 +49,11 @@ public class ApplicationApiHandler extends SessionHandler { public final static String MULTIPART_PARAMS = "prepareParams"; public final static String MULTIPART_APPLICATION_PACKAGE = "applicationPackage"; public final static String contentTypeHeader = "Content-Type"; + private final TenantRepository tenantRepository; private final Duration zookeeperBarrierTimeout; private final Zone zone; + private final long maxApplicationPackageSize; @Inject public ApplicationApiHandler(Context ctx, @@ -61,6 +63,7 @@ public class ApplicationApiHandler extends SessionHandler { super(ctx, applicationRepository); this.tenantRepository = applicationRepository.tenantRepository(); this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); + this.maxApplicationPackageSize = configserverConfig.maxApplicationPackageSize(); this.zone = zone; } @@ -85,14 +88,14 @@ public class ApplicationApiHandler extends SessionHandler { log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8)); prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout); Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); - compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType()); + compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType(), maxApplicationPackageSize); } catch (IOException e) { log.log(Level.WARNING, "Unable to parse multipart in deploy", e); throw new BadRequestException("Request contains invalid data"); } } else { prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); + compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader), maxApplicationPackageSize); } PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 23444ac53d6..1e8005c8af6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.yolean.Exceptions; import org.apache.commons.compress.archivers.ArchiveOutputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.junit.Test; @@ -15,11 +15,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -64,9 +63,10 @@ public class CompressedApplicationInputStreamTest { @Test public void require_that_valid_tar_application_can_be_unpacked() throws IOException { File outFile = createTarFile(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); - File outApp = unpacked.decompress(); - assertTestApp(outApp); + try (CompressedApplicationInputStream unpacked = streamFromTarGz(outFile)) { + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } } @Test @@ -91,48 +91,39 @@ public class CompressedApplicationInputStreamTest { archiveOutputStream.close(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); - File outApp = unpacked.decompress(); - assertEquals("application", outApp.getName()); // gets the name of the subdir - assertTestApp(outApp); + try (CompressedApplicationInputStream unpacked = streamFromTarGz(outFile)) { + File outApp = unpacked.decompress(); + assertEquals("application", outApp.getName()); // gets the name of the subdir + assertTestApp(outApp); + } } @Test public void require_that_valid_zip_application_can_be_unpacked() throws IOException { File outFile = createZipFile(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new ZipArchiveInputStream(new FileInputStream(outFile))); - File outApp = unpacked.decompress(); - assertTestApp(outApp); + try (CompressedApplicationInputStream unpacked = streamFromZip(outFile)) { + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } } @Test public void require_that_gnu_tared_file_can_be_unpacked() throws IOException, InterruptedException { - File tmpTar = File.createTempFile("myapp", ".tar"); - Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/validapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); - p.waitFor(); - p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); - p.waitFor(); - File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + File gzFile = createTarGz("src/test/resources/deploy/validapp"); assertTrue(gzFile.exists()); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(gzFile), "application/x-gzip", Long.MAX_VALUE); File outApp = unpacked.decompress(); assertTestApp(outApp); } @Test public void require_that_nested_app_can_be_unpacked() throws IOException, InterruptedException { - File tmpTar = File.createTempFile("myapp", ".tar"); - Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/advancedapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); - p.waitFor(); - p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); - p.waitFor(); - File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + File gzFile = createTarGz("src/test/resources/deploy/advancedapp"); assertTrue(gzFile.exists()); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); - File outApp = unpacked.decompress(); + File outApp; + try (CompressedApplicationInputStream unpacked = streamFromTarGz(gzFile)) { + outApp = unpacked.decompress(); + } List<File> files = Arrays.asList(outApp.listFiles()); assertEquals(5, files.size()); assertTrue(files.contains(new File(outApp, "services.xml"))); @@ -164,11 +155,29 @@ public class CompressedApplicationInputStreamTest { assertEquals(new File(bar, "lol").getAbsolutePath(), bar.listFiles()[0].getAbsolutePath()); } - - @Test(expected = IOException.class) - public void require_that_invalid_application_returns_error_when_unpacked() throws IOException { + @Test(expected = InternalServerException.class) + public void require_that_invalid_application_returns_error_when_unpacked() throws Exception { File app = new File("src/test/resources/deploy/validapp/services.xml"); - CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(app)))); + streamFromTarGz(app).close(); + } + + private static File createTarGz(String appDir) throws IOException, InterruptedException { + File tmpTar = File.createTempFile("myapp", ".tar"); + Process p = new ProcessBuilder("tar", "-C", appDir, "-cvf", tmpTar.getAbsolutePath(), ".").start(); + p.waitFor(); + p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); + p.waitFor(); + File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + assertTrue(gzFile.exists()); + return gzFile; + } + + private static CompressedApplicationInputStream streamFromZip(File zipFile) { + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(zipFile), "application/zip", Long.MAX_VALUE)); + } + + private static CompressedApplicationInputStream streamFromTarGz(File tarFile) { + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(tarFile), "application/x-gzip", Long.MAX_VALUE)); } + } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 44a70ea2f3b..be8ba669ec0 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -19,10 +19,10 @@ public class DocumentAccessProvider implements Provider<VespaDocumentAccess> { private final VespaDocumentAccess access; @Inject - public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, + public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { - this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, System.getProperty("config.id"), + this.access = new VespaDocumentAccess(documentmanagerConfig, System.getProperty("config.id"), messagebusConfig, policiesConfig, distributionConfig); } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index 6976299cc7d..1775dbe53c1 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -43,13 +43,12 @@ public class VespaDocumentAccess extends DocumentAccess { private boolean shutDown = false; VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig, String slobroksConfigId, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)) + this.parameters = new MessageBusParams() .setDocumentProtocolPoliciesConfig(policiesConfig, distributionConfig); this.parameters.setDocumentmanagerConfig(documentmanagerConfig); this.parameters.getRPCNetworkParams().setSlobrokConfigId(slobroksConfigId); diff --git a/container-core/src/main/java/com/yahoo/restapi/HttpURL.java b/container-core/src/main/java/com/yahoo/restapi/HttpURL.java new file mode 100644 index 00000000000..a43c5998c79 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/HttpURL.java @@ -0,0 +1,442 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import ai.vespa.validation.StringWrapper; +import com.yahoo.net.DomainName; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.OptionalInt; +import java.util.StringJoiner; +import java.util.function.Function; + +import static ai.vespa.validation.Validation.require; +import static ai.vespa.validation.Validation.requireInRange; +import static java.net.URLDecoder.decode; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +/** + * This is the best class for creating, manipulating and inspecting HTTP URLs, because: + * <ul> + * <li>It is more restrictive than {@link URI}, but with a richer construction API, reducing risk of blunder. + * <ul> + * <li>Scheme must be HTTP or HTTPS.</li> + * <li>Authority must be a {@link DomainName}, with an optional port.</li> + * <li>{@link Path} must be normalized at all times.</li> + * <li>Only {@link Query} is allowed, in addition to the above.</li> + * </ul> + * </li> + * <li> + * It contains all those helpful builder methods that {@link URI} has none of. + * <ul> + * <li>{@link Path} can be parsed, have segments or other paths appended, and cut.</li> + * <li>{@link Query} can be parsed, and keys and key-value pairs can be inserted or removed.</li> + * </ul> + * All these (except the parse methods) operate on <em>decoded</em> values. + * </li> + * <li>It makes it super-easy to use a {@link StringWrapper} for validation of path and query segments.</li> + * </ul> + * + * @author jonmv + */ +public class HttpURL<T> { + + private final Scheme scheme; + private final DomainName domain; + private final int port; + private final Path<T> path; + private final Query<T> query; + + private HttpURL(Scheme scheme, DomainName domain, int port, Path<T> path, Query<T> query) { + this.scheme = requireNonNull(scheme); + this.domain = requireNonNull(domain); + this.port = requireInRange(port, "port number", -1, (1 << 16) - 1); + this.path = requireNonNull(path); + this.query = requireNonNull(query); + } + + public static <T> HttpURL<T> create(Scheme scheme, DomainName domain, int port, Path<T> path, Query<T> query) { + return new HttpURL<>(scheme, domain, port, path, query); + } + + public static HttpURL<String> create(Scheme scheme, DomainName domain, int port, Path<String> path) { + return create(scheme, domain, port, path, Query.empty()); + } + + public static <T extends StringWrapper<T>> HttpURL<T> create(Scheme scheme, DomainName domain, int port, Path<T> path, Function<String, T> validator) { + return create(scheme, domain, port, path, Query.empty(validator)); + } + + public static <T extends StringWrapper<T>> HttpURL<T> create(Scheme scheme, DomainName domain, int port, Function<String, T> validator) { + return create(scheme, domain, port, Path.empty(validator), validator); + } + + public static HttpURL<String> create(Scheme scheme, DomainName domain, int port) { + return create(scheme, domain, port, Path.empty()); + } + + public static <T extends StringWrapper<T>> HttpURL<T> create(Scheme scheme, DomainName domain, Function<String, T> validator) { + return create(scheme, domain, -1, validator); + } + + public static HttpURL<String> create(Scheme scheme, DomainName domain) { + return create(scheme, domain, -1); + } + + public static HttpURL<String> from(URI uri) { + return from(uri, identity(), identity()); + } + + public static <T extends StringWrapper<T>> HttpURL<T> from(URI uri, Function<String, T> validator) { + return from(uri, validator, T::value); + } + + private static <T> HttpURL<T> from(URI uri, Function<String, T> validator, Function<T, String> inverse) { + if ( ! uri.normalize().equals(uri)) + throw new IllegalArgumentException("uri should be normalized, but got: " + uri); + + return create(Scheme.of(uri.getScheme()), + DomainName.of(requireNonNull(uri.getHost(), "URI must specify a host")), + uri.getPort(), + Path.parse(uri.getRawPath(), validator, inverse), + Query.parse(uri.getRawQuery(), validator, inverse)); + } + + public HttpURL<T> withScheme(Scheme scheme) { + return create(scheme, domain, port, path, query); + } + + public HttpURL<T> withDomain(DomainName domain) { + return create(scheme, domain, port, path, query); + } + + public HttpURL<T> withPort(int port) { + return create(scheme, domain, port, path, query); + } + + public HttpURL<T> withoutPort() { + return create(scheme, domain, -1, path, query); + } + + public HttpURL<T> withPath(Path<T> path) { + return create(scheme, domain, port, path, query); + } + + public HttpURL<T> withQuery(Query<T> query) { + return create(scheme, domain, port, path, query); + } + + public Scheme scheme() { + return scheme; + } + + public DomainName domain() { + return domain; + } + + public OptionalInt port() { + return port == -1 ? OptionalInt.empty() : OptionalInt.of(port); + } + + public Path<T> path() { + return path; + } + + public Query<T> query() { + return query; + } + + /** Returns an absolute, hierarchical URI representing this HTTP URL. */ + public URI asURI() { + try { + return new URI(scheme.name() + "://" + domain.value() + (port == -1 ? "" : ":" + port) + path.raw() + query.raw()); + } + catch (URISyntaxException e) { + throw new IllegalStateException("invalid URI, this should not happen", e); + } + } + + + public static class Path<T> { + + private final List<T> segments; + private final boolean trailingSlash; + private final Function<String, T> validator; + private final Function<T, String> inverse; + + private Path(List<T> segments, boolean trailingSlash, Function<String, T> validator, Function<T, String> inverse) { + this.segments = requireNonNull(segments); + this.trailingSlash = trailingSlash; + this.validator = requireNonNull(validator); + this.inverse = requireNonNull(inverse); + } + + /** Creates a new, empty path, with a trailing slash. */ + public static Path<String> empty() { + return new Path<>(List.of(), true, identity(), identity()); + } + + /** Creates a new, empty path, with a trailing slash, using the indicated string wrapper for segments. */ + public static <T extends StringWrapper<T>> Path<T> empty(Function<String, T> validator) { + return new Path<>(List.of(), true, validator, T::value); + } + /** Creates a new path with the given <em>decoded</em> segments. */ + public static Path<String> from(List<String> segments) { + return empty().append(segments); + } + + /** Creates a new path with the given <em>decoded</em> segments, and the validator applied to each segment. */ + public static <T extends StringWrapper<T>> Path<T> from(List<String> segments, Function<String, T> validator) { + return empty(validator).append(segments, identity(), true); + } + + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative.) */ + public static <T extends StringWrapper<T>> Path<T> parse(String raw, Function<String, T> validator) { + return parse(raw, validator, T::value); + } + + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative. */ + public static Path<String> parse(String raw) { + return parse(raw, identity(), identity()); + } + + private static <T> Path<T> parse(String raw, Function<String, T> validator, Function<T, String> inverse) { + boolean trailingSlash = raw.endsWith("/"); + if (raw.startsWith("/")) raw = raw.substring(1); + if (raw.isEmpty()) return new Path<>(List.of(), trailingSlash, validator, inverse); + List<T> segments = new ArrayList<>(); + for (String segment : raw.split("/")) + segments.add(validator.apply(requireNonNormalizable(decode(segment, UTF_8)))); + if (segments.size() == 0) requireNonNormalizable(""); // Raw path was only slashes. + return new Path<>(segments, trailingSlash, validator, inverse); + } + + private static String requireNonNormalizable(String segment) { + return require( ! (segment.isEmpty() || segment.equals(".") || segment.equals("..")), + segment, "path segments cannot be \"\", \".\", or \"..\""); + } + + /** Returns a copy of this where the first segments are skipped. */ + public Path<T> skip(int count) { + return new Path<>(segments.subList(count, segments.size()), trailingSlash, validator, inverse); + } + + /** Returns a copy of this where the last segments are cut off. */ + public Path<T> cut(int count) { + return new Path<>(segments.subList(0, segments.size() - count), trailingSlash, validator, inverse); + } + + /** Returns a copy of this with the <em>decoded</em> segment appended at the end; it may not be either of {@code ""}, {@code "."} or {@code ".."}. */ + public Path<T> append(String segment) { + return append(List.of(segment), identity(), trailingSlash); + } + + /** Returns a copy of this all segments of the other path appended, with a trailing slash as per the appendage. */ + public <U> Path<T> append(Path<U> other) { + return append(other.segments, other.inverse, other.trailingSlash); + } + + /** Returns a copy of this all given segments appended, with a trailing slash as per this path. */ + public Path<T> append(List<T> segments) { + return append(segments, inverse, trailingSlash); + } + + private <U> Path<T> append(List<U> segments, Function<U, String> inverse, boolean trailingSlash) { + List<T> copy = new ArrayList<>(this.segments); + for (U segment : segments) copy.add(validator.apply(requireNonNormalizable(inverse.apply(segment)))); + return new Path<>(copy, trailingSlash, validator, this.inverse); + } + + /** Returns a copy of this which encodes a trailing slash. */ + public Path<T> withTrailingSlash() { + return new Path<>(segments, true, validator, inverse); + } + + /** Returns a copy of this which does not encode a trailing slash. */ + public Path<T> withoutTrailingSlash() { + return new Path<>(segments, false, validator, inverse); + } + + /** The <em>URL decoded</em> segments that make up this path; never {@code null}, {@code ""}, {@code "."} or {@code ".."}. */ + public List<T> segments() { + return Collections.unmodifiableList(segments); + } + + /** A raw path string which parses to this, by splitting on {@code "/"}, and then URL decoding. */ + private String raw() { + StringJoiner joiner = new StringJoiner("/", "/", trailingSlash ? "/" : "").setEmptyValue(trailingSlash ? "/" : ""); + for (T segment : segments) + joiner.add(encode(inverse.apply(segment), UTF_8)); + return joiner.toString(); + } + + /** Intentionally not usable for constructing new URIs. Use {@link HttpURL} for that instead. */ + @Override + public String toString() { + return "path '" + raw() + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Path<?> path = (Path<?>) o; + return trailingSlash == path.trailingSlash && segments.equals(path.segments); + } + + @Override + public int hashCode() { + return Objects.hash(segments, trailingSlash); + } + + } + + + public static class Query<T> { + + private final Map<T, T> values; + private final Function<String, T> validator; + private final Function<T, String> inverse; + + private Query(Map<T, T> values, Function<String, T> validator, Function<T, String> inverse) { + this.values = requireNonNull(values); + this.validator = requireNonNull(validator); + this.inverse = requireNonNull(inverse); + } + + /** Creates a new, empty query part. */ + public static Query<String> empty() { + return new Query<>(Map.of(), identity(), identity()); + } + + /** Creates a new, empty query part, using the indicated string wrapper for keys and non-null values. */ + public static <T extends StringWrapper<T>> Query<T> empty(Function<String, T> validator) { + return new Query<>(Map.of(), validator, T::value); + } + /** Creates a new query part with the given <em>decoded</em> values. */ + public static Query<String> from(Map<String, String> values) { + return empty().merge(values); + } + + /** Creates a new query part with the given <em>decoded</em> values, and the validator applied to each pair. */ + public static <T extends StringWrapper<T>> Query<T> from(Map<String, String> values, Function<String, T> validator) { + return empty(validator).merge(values, identity()); + } + + /** Parses the given raw query string, using the indicated string wrapper to hold keys and non-null values. */ + public static <T extends StringWrapper<T>> Query<T> parse(String raw, Function<String, T> validator) { + return parse(raw, validator, T::value); + } + + /** Parses the given raw query string. */ + public static Query<String> parse(String raw) { + return parse(raw, identity(), identity()); + } + + private static <T> Query<T> parse(String raw, Function<String, T> validator, Function<T, String> inverse) { + if (raw == null) return new Query<>(Map.of(), validator, inverse); + Map<T, T> values = new LinkedHashMap<>(); + for (String pair : raw.split("&")) { + int split = pair.indexOf("="); + String key, value; + if (split == -1) { key = pair; value = null; } + else { key = pair.substring(0, split); value = pair.substring(split + 1); } + values.put(validator.apply(decode(key, UTF_8)), value == null ? null : validator.apply(decode(value, UTF_8))); + } + return new Query<>(values, validator, inverse); + } + + /** Returns a copy of this with the <em>decoded</em> non-null key pointing to the <em>decoded</em> non-null value. */ + public Query<T> put(String key, String value) { + Map<T, T> copy = new LinkedHashMap<>(values); + copy.put(requireNonNull(validator.apply(key)), requireNonNull(validator.apply(value))); + return new Query<>(copy, validator, inverse); + } + + /** Returns a copy of this with the <em>decoded</em> non-null key pointing to "nothing". */ + public Query<T> add(String key) { + Map<T, T> copy = new LinkedHashMap<>(values); + copy.put(requireNonNull(validator.apply(key)), null); + return new Query<>(copy, validator, inverse); + } + + /** Returns a copy of this without any key-value pair with the <em>decoded</em> key. */ + public Query<T> remove(String key) { + Map<T, T> copy = new LinkedHashMap<>(values); + copy.remove(requireNonNull(validator.apply(key))); + return new Query<>(copy, validator, inverse); + } + + /** Returns a copy of this with all mappings from the other query added to this, possibly overwriting existing mappings. */ + public <U> Query<T> merge(Query<U> other) { + return merge(other.values, other.inverse); + } + + /** Returns a copy of this with all given mappings added to this, possibly overwriting existing mappings. */ + public Query<T> merge(Map<T, T> values) { + return merge(values, inverse); + } + + private <U> Query<T> merge(Map<U, U> values, Function<U, String> inverse) { + Map<T, T> copy = new LinkedHashMap<>(this.values); + values.forEach((key, value) -> copy.put(validator.apply(inverse.apply(requireNonNull(key, "keys cannot be null"))), + value == null ? null : validator.apply(inverse.apply(value)))); + return new Query<>(copy, validator, this.inverse); + } + + /** The <em>URL decoded</em> key-value pairs that make up this query; keys and values may be {@code ""}, and values are {@code null} when only key was specified. */ + public Map<T, T> entries() { + return unmodifiableMap(values); + } + + /** A raw query string, with {@code '?'} prepended, that parses to this, by splitting on {@code "&"}, then on {@code "="}, and then URL decoding; or the empty string if this is empty. */ + private String raw() { + StringJoiner joiner = new StringJoiner("&", "?", "").setEmptyValue(""); + values.forEach((key, value) -> joiner.add(encode(inverse.apply(key), UTF_8) + + (value == null ? "" : "=" + encode(inverse.apply(value), UTF_8)))); + return joiner.toString(); + } + + /** Intentionally not usable for constructing new URIs. Use {@link HttpURL} for that instead. */ + @Override + public String toString() { + return "query '" + raw() + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Query<?> query = (Query<?>) o; + return values.equals(query.values); + } + + @Override + public int hashCode() { + return Objects.hash(values); + } + + } + + + public enum Scheme { + http, + https; + public static Scheme of(String scheme) { + if (scheme.equalsIgnoreCase(http.name())) return http; + if (scheme.equalsIgnoreCase(https.name())) return https; + throw new IllegalArgumentException("scheme must be HTTP or HTTPS"); + } + } + +} diff --git a/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java b/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java new file mode 100644 index 00000000000..de20fcb3193 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java @@ -0,0 +1,178 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import ai.vespa.validation.Name; +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL.Query; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +import static com.yahoo.net.DomainName.localhost; +import static com.yahoo.restapi.HttpURL.Scheme.http; +import static com.yahoo.restapi.HttpURL.Scheme.https; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author jonmv + */ +class HttpURLTest { + + @Test + void testConversionBackAndForth() { + for (String uri : List.of("http://minimal", + "http://empty.query?", + "http://zero-port:0?no=path", + "http://only-path/", + "https://strange/queries?=&foo", + "https://weirdness?=foo", + "https://encoded/%3F%3D%26%2F?%3F%3D%26%2F=%3F%3D%26%2F", + "https://host.at.domain:123/one/two/?three=four&five")) + assertEquals(uri, HttpURL.from(URI.create(uri)).asURI().toString(), + "uri '" + uri + "' should be returned unchanged"); + } + + @Test + void testModification() { + HttpURL<Name> url = HttpURL.create(http, localhost, Name::of); + assertEquals(http, url.scheme()); + assertEquals(localhost, url.domain()); + assertEquals(OptionalInt.empty(), url.port()); + assertEquals(HttpURL.Path.empty(Name::of), url.path()); + assertEquals(HttpURL.Query.empty(Name::of), url.query()); + + url = url.withScheme(https) + .withDomain(DomainName.of("domain")) + .withPort(0) + .withPath(url.path().append("foo").withoutTrailingSlash()) + .withQuery(url.query().put("boo", "bar").add("baz")); + assertEquals(https, url.scheme()); + assertEquals(DomainName.of("domain"), url.domain()); + assertEquals(OptionalInt.of(0), url.port()); + assertEquals(HttpURL.Path.parse("/foo", Name::of), url.path()); + assertEquals(HttpURL.Query.parse("boo=bar&baz", Name::of), url.query()); + } + + @Test + void testInvalidURIs() { + assertEquals("scheme must be HTTP or HTTPS", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("file:/txt"))).getMessage()); + + assertEquals("URI must specify a host", + assertThrows(NullPointerException.class, + () -> HttpURL.from(URI.create("http:///foo"))).getMessage()); + + assertEquals("port number must be at least '-1' and at most '65535', but got: '65536'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo:65536/bar"))).getMessage()); + + assertEquals("uri should be normalized, but got: http://foo//", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo//"))).getMessage()); + + assertEquals("uri should be normalized, but got: http://foo/./", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/./"))).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/.."))).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/.%2E"))).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '/'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/%2F"), Name::of)).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '/'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo?%2F"), Name::of)).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: ''", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo?"), Name::of)).getMessage()); + } + + @Test + void testPath() { + HttpURL.Path<Name> path = HttpURL.Path.parse("foo/bar/baz", Name::of); + List<Name> expected = List.of(Name.of("foo"), Name.of("bar"), Name.of("baz")); + assertEquals(expected, path.segments()); + + assertEquals(expected.subList(1, 3), path.skip(1).segments()); + assertEquals(expected.subList(0, 2), path.cut(1).segments()); + assertEquals(expected.subList(1, 2), path.skip(1).cut(1).segments()); + + assertEquals("path '/foo/bar/baz/'", path.withTrailingSlash().toString()); + assertEquals(path, path.withoutTrailingSlash().withoutTrailingSlash()); + + assertEquals(List.of("one", "foo", "bar", "baz", "two"), + HttpURL.Path.from(List.of("one")).append(path).append("two").segments()); + + assertEquals(List.of(expected.get(2), expected.get(0)), + path.append(path).cut(2).skip(2).segments()); + + assertThrows(NullPointerException.class, + () -> path.append((String) null)); + + List<Name> names = new ArrayList<>(); + names.add(null); + assertThrows(NullPointerException.class, + () -> path.append(names)); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '???'", + assertThrows(IllegalArgumentException.class, + () -> path.append("???")).getMessage()); + + assertEquals("fromIndex(2) > toIndex(1)", + assertThrows(IllegalArgumentException.class, + () -> path.cut(2).skip(2)).getMessage()); + } + + @Test + void testQuery() { + Query<Name> query = Query.parse("foo=bar&baz", Name::of); + Map<Name, Name> expected = new LinkedHashMap<>(); + expected.put(Name.of("foo"), Name.of("bar")); + expected.put(Name.of("baz"), null); + assertEquals(expected, query.entries()); + + expected.remove(Name.of("baz")); + assertEquals(expected, query.remove("baz").entries()); + + expected.put(Name.of("baz"), null); + expected.remove(Name.of("foo")); + assertEquals(expected, query.remove("foo").entries()); + assertEquals(expected, Query.empty(Name::of).add("baz").entries()); + + assertEquals("query '?foo=bar&baz=bax&quu=fez&moo'", + query.put("baz", "bax").merge(Query.from(Map.of("quu", "fez"))).add("moo").toString()); + + assertThrows(NullPointerException.class, + () -> query.remove(null)); + + assertThrows(NullPointerException.class, + () -> query.add(null)); + + assertThrows(NullPointerException.class, + () -> query.put(null, "hax")); + + assertThrows(NullPointerException.class, + () -> query.put("hax", null)); + + Map<Name, Name> names = new LinkedHashMap<>(); + names.put(null, Name.of("hax")); + assertThrows(NullPointerException.class, + () -> query.merge(names)); + } + +} diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 034081f4620..6268e1e6fb4 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -121,6 +121,10 @@ <groupId>io.airlift</groupId> <artifactId>aircompressor</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 46dcaf17abc..91a3181cb68 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -69,14 +69,18 @@ public final class SessionCache extends AbstractComponent { @Inject public SessionCache(NetworkMultiplexerProvider nets, ContainerMbusConfig containerMbusConfig, DocumentTypeManager documentTypeManager, - LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, + MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(nets::net, containerMbusConfig, documentTypeManager, - loadTypeConfig, messagebusConfig, policiesConfig, distributionConfig); + null/*TODO: Remove on Vespa 8*/, messagebusConfig, policiesConfig, distributionConfig); } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public SessionCache(Supplier<NetworkMultiplexer> net, ContainerMbusConfig containerMbusConfig, DocumentTypeManager documentTypeManager, LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, @@ -86,7 +90,6 @@ public final class SessionCache extends AbstractComponent { containerMbusConfig, messagebusConfig, new DocumentProtocol(documentTypeManager, - new LoadTypeSet(loadTypeConfig), policiesConfig, distributionConfig)); } diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index c509fb917fa..b9f33506894 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -36,6 +36,7 @@ public class MbusClientProviderTest { testClient(new SessionConfig(builder)); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void testClient(SessionConfig config) { SessionCache cache = new SessionCache(() -> NetworkMultiplexer.dedicated(new NullNetwork()), new ContainerMbusConfig.Builder().build(), diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 489214bebf8..71a93607b4b 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -90,6 +90,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { } @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypeSet() { return ((MessageBusDocumentAccess) access.delegate()).getParams().getLoadTypes(); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index b2e4821f164..9330e43eaf7 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -76,6 +76,12 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public interface VisitorSessionFactory { VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; + + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 LoadTypeSet getLoadTypeSet(); } @@ -119,6 +125,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { return query.properties().getString(streamingSelection); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void setVisitorParameters(String searchCluster, Route route, String documentType) { params.setDocumentSelection(createSelectionString(documentType, createQuerySelectionString())); params.setTimeoutMs(query.getTimeout()); // Per bucket visitor timeout @@ -134,6 +141,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.visitInconsistentBuckets(true); params.setPriority(DocumentProtocol.Priority.VERY_HIGH); + // TODO remove on Vespa 8 if (query.properties().getString(streamingLoadtype) != null) { LoadType loadType = visitorSessionFactory.getLoadTypeSet().getNameMap().get(query.properties().getString(streamingLoadtype)); if (loadType != null) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index 1d07cafeda9..b1bc926daed 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -31,8 +31,9 @@ import static org.junit.Assert.*; /** * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VdsVisitorTestCase { - private LoadTypeSet loadTypeSet = new LoadTypeSet(); + private LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 public VdsVisitorTestCase() { loadTypeSet.addLoadType(1, "low", DocumentProtocol.Priority.LOW_1); @@ -489,7 +490,7 @@ public class VdsVisitorTestCase { private static class MockVisitorSessionFactory implements VdsVisitor.VisitorSessionFactory { private VisitorParameters params; - private LoadTypeSet loadTypeSet; + private LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private boolean timeoutQuery = false; private boolean failQuery = false; @@ -504,6 +505,7 @@ public class VdsVisitorTestCase { } @Override + // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypeSet() { return loadTypeSet; } diff --git a/container-test/pom.xml b/container-test/pom.xml index 7c739faad26..d4ea39fa3c9 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -92,5 +92,10 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <scope>compile</scope> + </dependency> </dependencies> </project> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java new file mode 100644 index 00000000000..0febc296fc8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +/** + * MailerException wrap all possible Mailer implementation exceptions + * + * @author enygaard + */ +public class MailerException extends RuntimeException { + + public MailerException(Throwable ex) { + super(ex); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java index 6050f238da7..cb2b76d845c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java @@ -12,9 +12,25 @@ import java.util.Map; public class MockMailer implements Mailer { public final Map<String, List<Mail>> mails = new HashMap<>(); + public final boolean blackhole; + + public MockMailer() { + this(false); + } + + MockMailer(boolean blackhole) { + this.blackhole = blackhole; + } + + public static MockMailer blackhole() { + return new MockMailer(true); + } @Override public void send(Mail mail) { + if (blackhole) { + return; + } for (String recipient : mail.recipients()) { mails.putIfAbsent(recipient, new ArrayList<>()); mails.get(recipient).add(mail); @@ -33,7 +49,10 @@ public class MockMailer implements Mailer { /** Returns the list of mails sent to the given recipient. Modifications affect the set of mails stored in this. */ public List<Mail> inbox(String recipient) { - return mails.get(recipient); + return mails.getOrDefault(recipient, List.of()); } + public void reset() { + mails.clear(); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6afeab9b4e6..c35e8c5a7ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.notification.NotificationsDb; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.security.AccessControl; @@ -88,6 +89,7 @@ public class Controller extends AbstractComponent { private final CuratorArchiveBucketDb archiveBucketDb; private final NotificationsDb notificationsDb; private final SupportAccessControl supportAccessControl; + private final Notifier notifier; /** * Creates a controller @@ -126,6 +128,7 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); + notifier = new Notifier(curator, serviceRegistry.mailer()); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); @@ -330,4 +333,7 @@ public class Controller extends AbstractComponent { return supportAccessControl; } + public Notifier notifier() { + return notifier; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 5c7ae5041fe..258884a4d11 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.google.common.hash.Funnel; import com.google.common.hash.Hashing; import com.yahoo.component.Version; +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -24,6 +27,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,6 +44,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import java.util.function.Predicate; @@ -108,7 +113,7 @@ public class ApplicationPackage { this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); - this.bundleHash = calculateBundleHash(); + this.bundleHash = calculateBundleHash(zippedContent); preProcessAndPopulateCache(); } @@ -120,7 +125,7 @@ public class ApplicationPackage { byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8); ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length); - ZipStreamReader.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); + ZipEntries.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); return new ApplicationPackage(modified.toByteArray()); } @@ -227,15 +232,23 @@ public class ApplicationPackage { } // Hashes all files and settings that require a deployment to be forwarded to configservers - private String calculateBundleHash() { + private String calculateBundleHash(byte[] zippedContent) { Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); - SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher); - Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.entrySet().forEach(entry -> { - into.putBytes(entry.getKey().getBytes()); - into.putLong(entry.getValue()); + SortedMap<String, Long> crcByEntry = new TreeMap<>(); + Options options = Options.standard().pathPredicate(entryMatcher); + ArchiveFile file; + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zippedContent), options)) { + OutputStream discard = OutputStream.nullOutputStream(); + while ((file = reader.readNextTo(discard)) != null) { + crcByEntry.put(file.path().toString(), file.crc32().orElse(-1)); + } + } + Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.forEach((key, value) -> { + into.putBytes(key.getBytes()); + into.putLong(value); }); return Hashing.sha1().newHasher() - .putObject(entryCRCs, funnel) + .putObject(crcByEntry, funnel) .putInt(deploymentSpec.deployableHashCode()) .hash().toString(); } @@ -285,13 +298,13 @@ public class ApplicationPackage { } private Map<Path, Optional<byte[]>> read(Collection<String> names) { - var entries = new ZipStreamReader(new ByteArrayInputStream(zip), - name -> names.contains(withoutLegacyDir(name)), - maxSize, - true) - .entries().stream() - .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.zipEntry().getName())).normalize(), - ZipStreamReader.ZipEntryWithContent::content)); + var entries = ZipEntries.from(zip, + name -> names.contains(withoutLegacyDir(name)), + maxSize, + true) + .asList().stream() + .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.name())).normalize(), + ZipEntries.ZipEntryWithContent::content)); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java index 97810b9de80..e18f6247cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java @@ -15,7 +15,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader.ZipEntryWithContent; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.ZipEntryWithContent; /** * @author freva @@ -75,8 +75,8 @@ public class ApplicationPackageDiff { } private static Map<String, ZipEntryWithContent> readContents(ApplicationPackage app, int maxFileSizeToDiff) { - return new ZipStreamReader(new ByteArrayInputStream(app.zippedContent()), entry -> true, maxFileSizeToDiff, false).entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), e -> e)); + return ZipEntries.from(app.zippedContent(), entry -> true, maxFileSizeToDiff, false).asList().stream() + .collect(Collectors.toMap(ZipEntryWithContent::name, e -> e)); } private static List<String> lines(byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java new file mode 100644 index 00000000000..a6cb7f23fc3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -0,0 +1,99 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** + * A list of entries read from a ZIP archive, and their contents. + * + * @author bratseth + */ +public class ZipEntries { + + private final List<ZipEntryWithContent> entries; + + private ZipEntries(List<ZipEntryWithContent> entries) { + this.entries = List.copyOf(Objects.requireNonNull(entries)); + } + + /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ + public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { + try (ZipOutputStream zipOut = new ZipOutputStream(out); + ZipInputStream zipIn = new ZipInputStream(in)) { + for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { + if (entry.getName().equals(name)) + continue; + + zipOut.putNextEntry(new ZipEntry(entry.getName())); + zipIn.transferTo(zipOut); + zipOut.closeEntry(); + } + zipOut.putNextEntry(new ZipEntry(name)); + zipOut.write(content); + zipOut.closeEntry(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Read ZIP entries from inputStream */ + public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { + Options options = Options.standard() + .pathPredicate(entryNameMatcher) + .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB + .maxEntrySize(maxEntrySizeInBytes) + .maxEntries(1024) + .truncateEntry(!throwIfEntryExceedsMaxSize); + List<ZipEntryWithContent> entries = new ArrayList<>(); + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zip), options)) { + ArchiveFile file; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.add(new ZipEntryWithContent(file.path().toString(), + Optional.of(baos.toByteArray()).filter(b -> b.length > 0), + file.size())); + baos.reset(); + } + } + return new ZipEntries(entries); + } + + public List<ZipEntryWithContent> asList() { return entries; } + + public static class ZipEntryWithContent { + + private final String name; + private final Optional<byte[]> content; + private final long size; + + public ZipEntryWithContent(String name, Optional<byte[]> content, long size) { + this.name = name; + this.content = content; + this.size = size; + } + + public String name() { return name; } + public byte[] contentOrThrow() { return content.orElseThrow(); } + public Optional<byte[]> content() { return content; } + public long size() { return size; } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java deleted file mode 100644 index 174ac4cb8b0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -/** - * @author bratseth - */ -public class ZipStreamReader { - - private final List<ZipEntryWithContent> entries = new ArrayList<>(); - private final int maxEntrySizeInBytes; - - public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { - this.maxEntrySizeInBytes = maxEntrySizeInBytes; - try (ZipInputStream zipInput = new ZipInputStream(input)) { - ZipEntry zipEntry; - - while (null != (zipEntry = zipInput.getNextEntry())) { - if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue; - entries.add(readContent(zipEntry, zipInput, throwIfEntryExceedsMaxSize)); - } - } catch (IOException e) { - throw new UncheckedIOException("IO error reading zip content", e); - } - } - - /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ - public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { - try (ZipOutputStream zipOut = new ZipOutputStream(out); - ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if (entry.getName().equals(name)) - continue; - - zipOut.putNextEntry(new ZipEntry(entry.getName())); - zipIn.transferTo(zipOut); - zipOut.closeEntry(); - } - zipOut.putNextEntry(new ZipEntry(name)); - zipOut.write(content); - zipOut.closeEntry(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) { - SortedMap<String, Long> entryCRCs = new TreeMap<>(); - byte[] buffer = new byte[2048]; - try (ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if ( ! entryNameMatcher.test(entry.getName())) - continue; - // CRC is not set until entry is read - while ( -1 != zipIn.read(buffer)){} - entryCRCs.put(entry.getName(), entry.getCrc()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return entryCRCs; - } - - private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) { - try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { - byte[] buffer = new byte[2048]; - int read; - long size = 0; - while ( -1 != (read = zipInput.read(buffer))) { - size += read; - if (size > maxEntrySizeInBytes) { - if (throwIfEntryExceedsMaxSize) throw new IllegalArgumentException( - "Entry in zip content exceeded size limit of " + maxEntrySizeInBytes + " bytes"); - } else bis.write(buffer, 0, read); - } - - boolean hasContent = size <= maxEntrySizeInBytes; - return new ZipEntryWithContent(zipEntry, - Optional.of(bis).filter(__ -> hasContent).map(ByteArrayOutputStream::toByteArray), - size); - } catch (IOException e) { - throw new UncheckedIOException("Failed reading from zipped content", e); - } - } - - public List<ZipEntryWithContent> entries() { return Collections.unmodifiableList(entries); } - - private static String requireName(String name) { - if (List.of(name.split("/")).contains("..") || - !trimTrailingSlash(name).equals(Path.of(name).normalize().toString())) { - throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); - } - return name; - } - - private static String trimTrailingSlash(String name) { - if (name.endsWith("/")) return name.substring(0, name.length() - 1); - return name; - } - - public static class ZipEntryWithContent { - - private final ZipEntry zipEntry; - private final Optional<byte[]> content; - private final long size; - - public ZipEntryWithContent(ZipEntry zipEntry, Optional<byte[]> content, long size) { - this.zipEntry = zipEntry; - this.content = content; - this.size = size; - } - - public ZipEntry zipEntry() { return zipEntry; } - public byte[] contentOrThrow() { return content.orElseThrow(); } - public Optional<byte[]> content() { return content; } - public long size() { return size; } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index c0bd1ac03ff..5244d46d0a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -32,14 +33,16 @@ public class NotificationsDb { private final Clock clock; private final CuratorDb curatorDb; + private final Notifier notifier; public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator()); + this(controller.clock(), controller.curator(), controller.notifier()); } - NotificationsDb(Clock clock, CuratorDb curatorDb) { + NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) { this.clock = clock; this.curatorDb = curatorDb; + this.notifier = notifier; } public List<TenantName> listTenantsWithNotifications() { @@ -61,13 +64,24 @@ public class NotificationsDb { * already exists, it'll be replaced by this one instead */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + Optional<Notification> changed = Optional.empty(); try (Lock lock = curatorDb.lockNotifications(source.tenant())) { - List<Notification> notifications = curatorDb.readNotifications(source.tenant()).stream() + var existingNotifications = curatorDb.readNotifications(source.tenant()); + List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - notifications.add(new Notification(clock.instant(), type, level, source, messages)); + var notification = new Notification(clock.instant(), type, level, source, messages); + // Be conservative for now, only dispatch notifications if they are from new source or with new type. + // the message content and level is ignored for now + if (!existingNotifications.stream().anyMatch(n -> n.source().equals(source) && n.type().equals(type))) { + changed = Optional.of(notification); + } + notifications.add(notification); curatorDb.writeNotifications(source.tenant(), notifications); } + if (changed.isPresent()) { + notifier.dispatch(changed.get()); + } } /** Remove the notification with the given source and type */ @@ -131,8 +145,9 @@ public class NotificationsDb { newNotifications.stream()) .collect(Collectors.toUnmodifiableList()); - if (!initial.equals(updated)) + if (!initial.equals(updated)) { curatorDb.writeNotifications(deploymentSource.tenant(), updated); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java new file mode 100644 index 00000000000..46e1fd904ed --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.notify; + +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; +import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; + +import java.util.Collection; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Notifier is responsible for dispatching user notifications to their chosen Contact points. + * + * @author enygaard + */ +public class Notifier { + private final CuratorDb curatorDb; + private final Mailer mailer; + + private static final Logger log = Logger.getLogger(Notifier.class.getName()); + + public Notifier(CuratorDb curatorDb, Mailer mailer) { + this.curatorDb = Objects.requireNonNull(curatorDb); + this.mailer = Objects.requireNonNull(mailer); + } + + public void dispatch(Notification notification) { + var tenant = curatorDb.readTenant(notification.source().tenant()); + tenant.stream().forEach(t -> { + if (t instanceof CloudTenant) { + var ct = (CloudTenant) t; + ct.info().contacts().all().stream() + .filter(c -> c.audiences().contains(TenantContacts.Audience.NOTIFICATIONS)) + .collect(Collectors.groupingBy(TenantContacts.Contact::type, Collectors.toList())) + .entrySet() + .forEach(e -> dispatch(notification, e.getKey(), e.getValue())); + } + }); + } + + private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) { + switch (type) { + case EMAIL: + dispatch(notification, contacts.stream().map(c -> (TenantContacts.EmailContact) c).collect(Collectors.toList())); + break; + default: + throw new IllegalArgumentException("Unknown TenantContacts type " + type.name()); + } + } + + private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { + try { + mailer.send(mailOf(notification, contacts.stream().map(c -> c.email()).collect(Collectors.toList()))); + } catch (MailerException e) { + log.log(Level.SEVERE, "Failed sending email", e); + } + } + + private Mail mailOf(Notification n, Collection<String> recipients) { + var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name()); + var body = new StringBuilder(); + body.append("Source: ").append(n.source().toString()).append("\n") + .append("\n") + .append(String.join("\n", n.messages())); + return new Mail(recipients, subject.toString(), body.toString()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 766a51e3e8d..21e803800f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -104,7 +104,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from); final Optional<ZoneId> zone; - if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 709a7967a5e..4e155e937b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -154,9 +153,9 @@ public class ApplicationPackageTest { } private static Map<String, String> unzip(byte[] zip) { - return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) - .entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), + return ZipEntries.from(zip, __ -> true, 1 << 10, true) + .asList().stream() + .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, entry -> new String(entry.contentOrThrow(), UTF_8))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java new file mode 100644 index 00000000000..6908464640b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZipEntriesTest { + + @Test + public void test_replacement() { + ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); + List<X509Certificate> certificates = IntStream.range(0, 3) + .mapToObj(i -> { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=subject" + i); + return X509CertificateBuilder.fromKeypair(keyPair, + subject, + Instant.now(), + Instant.now().plusSeconds(1), + SignatureAlgorithm.SHA512_WITH_ECDSA, + BigInteger.valueOf(1)) + .build(); + }) + .collect(Collectors.toUnmodifiableList()); + + assertEquals(List.of(), applicationPackage.trustedCertificates()); + for (int i = 0; i < certificates.size(); i++) { + applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); + assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java deleted file mode 100644 index 33c18d123d2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author mpolden - */ -public class ZipStreamReaderTest { - - @Test - public void test_size_limit() { - Map<String, String> entries = Map.of("foo.xml", "foobar"); - try { - new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1, true); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - - entries = Map.of("foo.xml", "foobar", - "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit - ); - ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 10, true); - byte[] extracted = reader.entries().get(0).contentOrThrow(); - assertEquals("foobar", new String(extracted, StandardCharsets.UTF_8)); - } - - @Test - public void test_paths() { - Map<String, Boolean> tests = Map.of( - "../../services.xml", true, - "/../.././services.xml", true, - "./application/././services.xml", true, - "application//services.xml", true, - "artifacts/", false, // empty dir - "services..xml", false, - "application/services.xml", false, - "components/foo-bar-deploy.jar", false, - "services.xml", false - ); - tests.forEach((name, expectException) -> { - try { - new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024, true); - assertFalse("Expected exception for '" + name + "'", expectException); - } catch (IllegalArgumentException ignored) { - assertTrue("Unexpected exception for '" + name + "'", expectException); - } - }); - } - - @Test - public void test_replacement() { - ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); - List<X509Certificate> certificates = IntStream.range(0, 3) - .mapToObj(i -> { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - X500Principal subject = new X500Principal("CN=subject" + i); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.now(), - Instant.now().plusSeconds(1), - SignatureAlgorithm.SHA512_WITH_ECDSA, - BigInteger.valueOf(1)) - .build(); - }) - .collect(Collectors.toUnmodifiableList()); - - assertEquals(List.of(), applicationPackage.trustedCertificates()); - for (int i = 0; i < certificates.size(); i++) { - applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); - assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); - } - } - - private static byte[] zip(Map<String, String> entries) { - ByteArrayOutputStream zip = new ByteArrayOutputStream(); - try (ZipOutputStream out = new ZipOutputStream(zip)) { - for (Map.Entry<String, String> entry : entries.entrySet()) { - out.putNextEntry(new ZipEntry(entry.getKey())); - out.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); - out.closeEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return zip.toByteArray(); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 989a7c31821..86c21839c96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -32,7 +32,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; @@ -41,8 +40,6 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index d51856b329d..a5655d2e6eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; +import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; @@ -11,8 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; +import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import org.junit.Before; import org.junit.Test; @@ -22,6 +29,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; @@ -36,6 +44,19 @@ import static org.junit.Assert.assertTrue; public class NotificationsDbTest { private static final TenantName tenant = TenantName.from("tenant1"); + private static final String email = "user1@example.com"; + private static final CloudTenant cloudTenant = new CloudTenant(tenant, + Instant.now(), + LastLoginInfo.EMPTY, + Optional.empty(), + ImmutableBiMap.of(), + TenantInfo.empty() + .withContacts(new TenantContacts( + List.of(new TenantContacts.EmailContact( + List.of(TenantContacts.Audience.NOTIFICATIONS), + email)))), + List.of(), + Optional.empty()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), @@ -46,7 +67,8 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(); - private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb); + private final MockMailer mailer = new MockMailer(); + private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, mailer)); @Test public void list_test() { @@ -75,6 +97,29 @@ public class NotificationsDbTest { } @Test + public void notifier_test() { + Notification notification1 = notification(12345, Type.deployment, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + Notification notification2 = notification(12345, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); + Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + + var a = notifications.get(0); + notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Replace the 3rd notification. but don't change source or type + notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Notification for a new app, add without replacement + notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages()); + assertEquals(1, mailer.inbox(email).size()); + + // Notification for new type on existing app + notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages()); + assertEquals(2, mailer.inbox(email).size()); + } + + @Test public void remove_single_test() { // Remove the 3rd notification notificationsDb.removeNotification(NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), Type.deployment); @@ -160,6 +205,8 @@ public class NotificationsDbTest { @Before public void init() { curatorDb.writeNotifications(tenant, notifications); + curatorDb.writeTenant(cloudTenant); + mailer.reset(); } private static List<Notification> notificationIndices(int... indices) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 4668942b61e..1fe6d4cc86c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -21,12 +21,13 @@ import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 class MessageFactory { private final static Logger log = Logger.getLogger(MessageFactory.class.getName()); private final Message requestMsg; - private final LoadType loadType; - private final DocumentProtocol.Priority priority; + private final LoadType loadType; // TODO: Remove on Vespa 8 + private final DocumentProtocol.Priority priority; // TODO: Remove on Vespa 8 @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageFactory(DocumentMessage requestMsg) { diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java index de96c548652..51b068b4712 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.objects.Ids; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.OptionalInt; +import java.util.Objects; /** * A StringFieldValue is a wrapper class that holds a String in {@link com.yahoo.document.Document}s and @@ -57,10 +57,9 @@ public class StringFieldValue extends FieldValue { } private static void validateTextString(String value) { - OptionalInt illegalCodePoint = Text.validateTextString(value); - if (illegalCodePoint.isPresent()) { + if ( ! Text.isValidTextString(value)) { throw new IllegalArgumentException("The string field value contains illegal code point 0x" + - Integer.toHexString(illegalCodePoint.getAsInt()).toUpperCase()); + Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); } } @@ -88,7 +87,7 @@ public class StringFieldValue extends FieldValue { public StringFieldValue clone() { StringFieldValue strfval = (StringFieldValue) super.clone(); if (spanTrees != null) { - strfval.spanTrees = new HashMap<String, SpanTree>(spanTrees.size()); + strfval.spanTrees = new HashMap<>(spanTrees.size()); for (Map.Entry<String, SpanTree> entry : spanTrees.entrySet()) { strfval.spanTrees.put(entry.getKey(), new SpanTree(entry.getValue())); } @@ -240,8 +239,8 @@ public class StringFieldValue extends FieldValue { if (!(o instanceof StringFieldValue)) return false; if (!super.equals(o)) return false; StringFieldValue that = (StringFieldValue) o; - if ((spanTrees != null) ? !spanTrees.equals(that.spanTrees) : that.spanTrees != null) return false; - if ((value != null) ? !value.equals(that.value) : that.value != null) return false; + if (!Objects.equals(spanTrees, that.spanTrees)) return false; + if (!Objects.equals(value, that.value)) return false; return true; } diff --git a/document/src/main/java/com/yahoo/document/idstring/IdString.java b/document/src/main/java/com/yahoo/document/idstring/IdString.java index 55beff9eef9..40ac19dec6d 100644 --- a/document/src/main/java/com/yahoo/document/idstring/IdString.java +++ b/document/src/main/java/com/yahoo/document/idstring/IdString.java @@ -5,8 +5,6 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.text.Text; import com.yahoo.text.Utf8String; -import java.util.OptionalInt; - /** * To be used with DocumentId constructor. * @@ -81,10 +79,9 @@ public abstract class IdString { } private static void validateTextString(String id) { - OptionalInt illegalCodePoint = Text.validateTextString(id); - if (illegalCodePoint.isPresent()) { + if ( ! Text.isValidTextString(id)) { throw new IllegalArgumentException("Unparseable id '" + id + "': Contains illegal code point 0x" + - Integer.toHexString(illegalCodePoint.getAsInt()).toUpperCase()); + Integer.toHexString(Text.validateTextString(id).getAsInt()).toUpperCase()); } } diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 2e68d4803cb..1a2afa3621f 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1844,6 +1844,7 @@ "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriorityByName(java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutingPolicyFactory(java.lang.String, com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory)", @@ -3123,7 +3124,8 @@ ], "methods": [ "public abstract boolean encode(com.yahoo.messagebus.Routable, com.yahoo.document.serialization.DocumentSerializer)", - "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)" + "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", + "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" ], "fields": [] }, diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index a9dfe0128e0..7446c681dec 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -20,6 +20,7 @@ import java.util.TreeMap; * * @author Håkon Humberset */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParameters extends Parameters { private String documentSelection; @@ -47,7 +48,7 @@ public class VisitorParameters extends Parameters { private int maxBucketsPerVisitor = 1; private boolean dynamicallyIncreaseMaxBucketsPerVisitor = false; private float dynamicMaxBucketsIncreaseFactor = 2; - private LoadType loadType = LoadType.DEFAULT; + private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 private DocumentProtocol.Priority priority = null; private int traceLevel = 0; private ThrottlePolicy throttlePolicy = null; @@ -332,10 +333,18 @@ public class VisitorParameters extends Parameters { throttlePolicy = policy; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public void setLoadType(LoadType loadType) { this.loadType = loadType; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public LoadType getLoadType() { return loadType; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index de09049b49e..c2e6dde7f60 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -53,6 +53,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { * * @param params All parameters for construction. */ + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageBusDocumentAccess(MessageBusParams params) { super(params); this.params = params; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java index 628bca4098f..bb8c3a3b1b1 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -14,6 +14,7 @@ import static java.util.Objects.requireNonNull; /** * @author Einar M R Rosenvinge */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class MessageBusParams extends DocumentAccessParams { private String routingConfigId = null; @@ -26,12 +27,16 @@ public class MessageBusParams extends DocumentAccessParams { private RPCNetworkParams rpcNetworkParams = new RPCNetworkParams(); private com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(); private SourceSessionParams sourceSessionParams = new SourceSessionParams(); - private LoadTypeSet loadTypes; + private LoadTypeSet loadTypes; // TODO remove on Vespa 8 public MessageBusParams() { this(new LoadTypeSet()); } + /** + * @deprecated load types are deprecated. Use default constructor instead + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public MessageBusParams(LoadTypeSet loadTypes) { this.loadTypes = loadTypes; } @@ -39,7 +44,9 @@ public class MessageBusParams extends DocumentAccessParams { /** * * @return Returns the set of load types accepted by this Vespa installation + * @deprecated load types are deprecated */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypes() { return loadTypes; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java index 53c09dcbcb6..133736a8542 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java @@ -4,8 +4,10 @@ package com.yahoo.documentapi.messagebus.loadtypes; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; /** + * @deprecated load types are deprecated * @author thomasg */ +@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public class LoadType { private int id; private String name; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java index e28d760eddf..a3fbed472f0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -20,9 +20,15 @@ import java.util.TreeMap; * * For testing, you may want to use the empty constructor and add * load types yourself with addType(). + * + * @deprecated load types are deprecated */ +@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class LoadTypeSet { + public static final LoadTypeSet EMPTY = new LoadTypeSet(); + class DualMap { Map<String, LoadType> nameMap = new TreeMap<String, LoadType>(); Map<Integer, LoadType> idMap = new HashMap<Integer, LoadType>(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java index d1e3b61f998..21f7c243c6f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -9,10 +9,11 @@ import com.yahoo.text.Utf8String; /** * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class DocumentMessage extends Message { private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; - private LoadType loadType = LoadType.DEFAULT; + private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 /** * Constructs a new message with no content. @@ -65,10 +66,20 @@ public abstract class DocumentMessage extends Message { this.priority = priority; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public LoadType getLoadType() { return loadType; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void setLoadType(LoadType loadType) { if (loadType != null) { this.loadType = loadType; @@ -79,7 +90,7 @@ public abstract class DocumentMessage extends Message { @Override public int getApproxSize() { - return 4 + 1; // type + priority + return 4 + 1; // type + priority // TODO update on Vespa 8 to not include deprecated fields } @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index ac946b80429..5db426a5db4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -32,6 +32,7 @@ import static java.util.Objects.requireNonNull; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class DocumentProtocol implements Protocol { private static final Logger log = Logger.getLogger(DocumentProtocol.class.getName()); @@ -246,12 +247,27 @@ public class DocumentProtocol implements Protocol { this(docMan, configId, new LoadTypeSet()); } + public DocumentProtocol(DocumentTypeManager documentTypeManager, + DocumentProtocolPoliciesConfig policiesConfig, + DistributionConfig distributionConfig) { + this(requireNonNull(documentTypeManager), null, new LoadTypeSet(), + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + } + + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public DocumentProtocol(DocumentTypeManager documentTypeManager, LoadTypeSet loadTypes, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(requireNonNull(documentTypeManager), null, requireNonNull(loadTypes), - requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { this(docMan, configId == null ? "client" : configId, set, null, null); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 60c8a613bb5..42172b04b90 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -6,22 +6,17 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import java.util.logging.Level; import com.yahoo.messagebus.Routable; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.objects.Deserializer; -import com.yahoo.vespa.objects.Serializer; import java.util.Map; -import java.util.logging.Logger; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; @@ -86,7 +81,7 @@ public abstract class RoutableFactories60 { DocumentMessage msg = doDecode(in); if (msg != null) { msg.setPriority(DocumentProtocol.getPriority(pri)); - msg.setLoadType(loadTypes.getIdMap().get(loadType)); + msg.setLoadType(loadTypes.getIdMap().get(loadType)); // TODO: ignore on Vespa 8 } return msg; } @@ -136,6 +131,7 @@ public abstract class RoutableFactories60 { return doEncode(reply, out); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { byte pri = in.getByte(null); DocumentReply reply = doDecode(in); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java index d38671fa313..7baa41e5c6a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java @@ -22,7 +22,7 @@ public interface RoutableFactory { /** * <p>This method encodes the content of the given routable into a byte buffer that can later be decoded using the - * {@link #decode(DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)} method.</p> <p>Return false to signal failure.</p> + * {@link #decode(DocumentDeserializer)} method.</p> <p>Return false to signal failure.</p> * <p>This method is NOT exception safe.</p> * * @param obj The routable to encode. @@ -38,7 +38,15 @@ public interface RoutableFactory { * @param in The buffer to read from. * @param loadTypes The LoadTypeSet to inject into the Routable. * @return The decoded routable. + * @deprecated load types are deprecated. Use method without LoadTypeSet instead */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes); + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + default Routable decode(DocumentDeserializer in) { + return decode(in, LoadTypeSet.EMPTY); + } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 24677a9a322..2360cbe8bc3 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -28,13 +28,21 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 final class RoutableRepository { private static final Logger log = Logger.getLogger(RoutableRepository.class.getName()); private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>(); private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>(); - private LoadTypeSet loadTypes; + private LoadTypeSet loadTypes; // TODO remove on Vespa 8 + public RoutableRepository() {} + + /** + * @deprecated load types are deprecated. Use default constructor instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public RoutableRepository(LoadTypeSet set) { loadTypes = set; } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java index caed3867d99..b1187d48374 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -7,7 +7,9 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import org.junit.Test; import static org.junit.Assert.*; +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParametersTestCase { + // TODO: Remove on Vespa 8 private LoadType loadType = new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3); @SuppressWarnings("removal")// TODO: Vespa 8: remove @@ -21,12 +23,12 @@ public class VisitorParametersTestCase { params.setLibraryParameter("groovy", "dudes"); params.setLibraryParameter("ninja", "turtles"); params.setMaxBucketsPerVisitor(55); - params.setPriority(DocumentProtocol.Priority.HIGHEST); + params.setPriority(DocumentProtocol.Priority.HIGHEST); // TODO: Remove on Vespa 8 params.setRoute("extraterrestrial/highway"); params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(AllFields.NAME); - params.setLoadType(loadType); + params.setLoadType(loadType); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java index 73c343174d4..18269971f88 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java @@ -9,6 +9,8 @@ import static org.junit.Assert.assertEquals; /** * @author thomasg */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 +// TODO Vespa 8: remove test case once load types are gone public class LoadTypesTestCase { @Test diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 5250a6b1db7..deecba96aa6 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -143,6 +143,7 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "beartato"; @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); msg.setBucketSpace(BUCKET_SPACE); @@ -151,7 +152,7 @@ public class Messages60TestCase extends MessagesTestBase { for (Language lang : LANGUAGES) { msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("default", msg.getLoadType().getName()); + assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } @@ -162,9 +163,10 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "andrei"; @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); + msg.setLoadType(null); // TODO: Remove on Vespa 8 msg.setBucketSpace(BUCKET_SPACE); assertEquals(BASE_MESSAGE_LENGTH + 27 + serializedLength(BUCKET_SPACE), serialize("StatBucketMessage", msg)); @@ -172,7 +174,7 @@ public class Messages60TestCase extends MessagesTestBase { msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); + assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java index 19f77ee1335..74d06c05383 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java @@ -19,6 +19,7 @@ import static org.junit.Assert.*; /** * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class MessagesTestBase { protected enum Language { @@ -28,7 +29,7 @@ public abstract class MessagesTestBase { protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class); protected final DocumentTypeManager docMan = new DocumentTypeManager(); - protected final LoadTypeSet loadTypes = new LoadTypeSet(); + protected final LoadTypeSet loadTypes = new LoadTypeSet(); // TODO remove on Vespa 8 protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes); public MessagesTestBase() { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java index 0c66c05f35e..ab881e143b7 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java @@ -697,6 +697,7 @@ public class MessageBusVisitorSessionTestCase { } @Test + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void testMessageParameters() { MockSender sender = new MockSender(); MockReceiver receiver = new MockReceiver(); @@ -716,7 +717,7 @@ public class MessageBusVisitorSessionTestCase { params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(DocIdOnly.NAME); - params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); + params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); diff --git a/fastos/src/vespa/fastos/app.cpp b/fastos/src/vespa/fastos/app.cpp index 05e885a7d37..8477f79f00d 100644 --- a/fastos/src/vespa/fastos/app.cpp +++ b/fastos/src/vespa/fastos/app.cpp @@ -16,18 +16,6 @@ FastOS_ApplicationInterface::FastOS_ApplicationInterface() : _argc(0), _argv(nullptr) { -#ifdef __linux__ - char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); - if (fadvise != nullptr) { - int fadviseOptions(0); - if (strstr(fadvise, "SEQUENTIAL")) { fadviseOptions |= POSIX_FADV_SEQUENTIAL; } - if (strstr(fadvise, "RANDOM")) { fadviseOptions |= POSIX_FADV_RANDOM; } - if (strstr(fadvise, "WILLNEED")) { fadviseOptions |= POSIX_FADV_WILLNEED; } - if (strstr(fadvise, "DONTNEED")) { fadviseOptions |= POSIX_FADV_DONTNEED; } - if (strstr(fadvise, "NOREUSE")) { fadviseOptions |= POSIX_FADV_NOREUSE; } - FastOS_FileInterface::setDefaultFAdviseOptions(fadviseOptions); - } -#endif } FastOS_ApplicationInterface::~FastOS_ApplicationInterface () = default; diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index ab273fd2660..27ecbd08917 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -12,9 +12,11 @@ #include <vespa/config/common/configcontext.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/thread.h> +#include <vespa/fastos/file.h> #include <vespa/fastos/app.h> #include <iostream> #include <thread> +#include <fcntl.h> #include <vespa/log/log.h> LOG_SETUP("proton"); @@ -42,6 +44,7 @@ class App : public FastOS_Application { private: static void setupSignals(); + static void setup_fadvise(); Params parseParams(); void startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport); public: @@ -56,6 +59,23 @@ App::setupSignals() SIG::TERM.hook(); } +void +App::setup_fadvise() +{ +#ifdef __linux__ + char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); + if (fadvise != nullptr) { + int fadviseOptions(0); + if (strstr(fadvise, "SEQUENTIAL")) { fadviseOptions |= POSIX_FADV_SEQUENTIAL; } + if (strstr(fadvise, "RANDOM")) { fadviseOptions |= POSIX_FADV_RANDOM; } + if (strstr(fadvise, "WILLNEED")) { fadviseOptions |= POSIX_FADV_WILLNEED; } + if (strstr(fadvise, "DONTNEED")) { fadviseOptions |= POSIX_FADV_DONTNEED; } + if (strstr(fadvise, "NOREUSE")) { fadviseOptions |= POSIX_FADV_NOREUSE; } + FastOS_FileInterface::setDefaultFAdviseOptions(fadviseOptions); + } +#endif +} + Params App::parseParams() { @@ -247,6 +267,7 @@ App::Main() { try { setupSignals(); + setup_fadvise(); FastOS_ThreadPool threadPool(128_Ki); FNET_Transport transport(buildTransportConfig()); transport.Start(&threadPool); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java index 514669fe0ac..d9b1190aaed 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java @@ -18,7 +18,8 @@ public class MessageBusSessionFactory implements SessionFactory { public MessageBusSessionFactory(MessagePropertyProcessor processor) { this(processor, null, null); } - + + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private MessageBusSessionFactory(MessagePropertyProcessor processor, DocumentmanagerConfig documentmanagerConfig, SlobroksConfig slobroksConfig) { diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index e5da51f0918..84fbe63a576 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -33,10 +33,18 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private String defaultDocprocChain = null; private boolean defaultAbortOnDocumentError = true; private boolean defaultAbortOnSendError = true; - private final LoadTypeSet loadTypes; + private final LoadTypeSet loadTypes; // TODO remove on Vespa 8 private boolean configChanged = false; + public MessagePropertyProcessor(FeederConfig config) { + loadTypes = new LoadTypeSet(); + configure(config); + } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeConfig instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public MessagePropertyProcessor(FeederConfig config, LoadTypeConfig loadTypeCfg) { loadTypes = new LoadTypeSet(); configure(config, loadTypeCfg); @@ -127,11 +135,19 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib return feederOptions; } + /** + * @deprecated load types are deprecated. configure without LoadTypeConfig instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public synchronized void configure(FeederConfig config, LoadTypeConfig loadTypeConfig) { loadTypes.configure(loadTypeConfig); configure(config); } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 LoadTypeSet getLoadTypes() { return loadTypes; } @@ -175,7 +191,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private boolean abortOnDocumentError; private boolean abortOnFeedError; private boolean createIfNonExistent; - private LoadType loadType; + private LoadType loadType; // TODO remove on Vespa 8 private int traceLevel; PropertySetter(Route route, long timeout, long totalTimeout, DocumentProtocol.Priority priority, LoadType loadType, diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java index cd524c07f73..c2985996bd0 100755 --- a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -4,7 +4,6 @@ package com.yahoo.dummyreceiver; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; @@ -68,7 +67,7 @@ public class DummyReceiver implements MessageHandler { } private void init() { - MessageBusParams params = new MessageBusParams(new LoadTypeSet()); + MessageBusParams params = new MessageBusParams(); params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name))); params.setDocumentManagerConfigId("client"); params.getMessageBusParams().setMaxPendingCount(0); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index 8f5db4adf97..52f2857c7e5 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -153,7 +153,7 @@ public class Arguments { } } - propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder())); + propertyProcessor = new MessagePropertyProcessor(getFeederConfig()); } private String getParam(List<String> args, String arg) throws IllegalArgumentException { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java index b9917533a62..2454f5c8627 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -27,18 +27,32 @@ import java.util.Map; * * @author bjorncs */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class DocumentRetriever { private final ClusterList clusterList; private final DocumentAccessFactory documentAccessFactory; private final ClientParameters params; - private final LoadTypeSet loadTypeSet; + private final LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private MessageBusSyncSession session; private MessageBusDocumentAccess documentAccess; public DocumentRetriever(ClusterList clusterList, DocumentAccessFactory documentAccessFactory, + ClientParameters params) { + this.clusterList = clusterList; + this.documentAccessFactory = documentAccessFactory; + this.loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 + this.params = params; + } + + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + public DocumentRetriever(ClusterList clusterList, + DocumentAccessFactory documentAccessFactory, LoadTypeSet loadTypeSet, ClientParameters params) { this.clusterList = clusterList; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java index fd2c9e964f7..7596246d16e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java @@ -38,11 +38,12 @@ public class Main { Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown)); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private static DocumentRetriever createDocumentRetriever(ClientParameters params) { return new DocumentRetriever( new ClusterList("client"), new DocumentAccessFactory(), - new LoadTypeSet(params.configId), + new LoadTypeSet(params.configId), // TODO: Remove on Vespa 8 params ); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index 4d79f2f2e1d..0e64f824b63 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -10,7 +10,6 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.StaticThrottlePolicy; @@ -36,7 +35,7 @@ import java.util.stream.Collectors; public class VdsVisit { private VdsVisitParameters params; - private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet()); + private MessageBusParams mbparams = new MessageBusParams(); private VisitorSession session; private final VisitorSessionAccessorFactory sessionAccessorFactory; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index d1fbde7dd42..7dfa3a2cf2e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -8,7 +8,6 @@ import com.yahoo.documentapi.VisitorDestinationParameters; import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.Identity; @@ -209,7 +208,7 @@ public class VdsVisitTarget { public void run() throws Exception { initShutdownHook(); log.log(Level.FINE, "Starting VdsVisitTarget"); - MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); + MessageBusParams mbusParams = new MessageBusParams(); mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); if (port > 0) { diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java index 8d7483c2196..30d117ab105 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -129,7 +129,6 @@ public class DocumentRetrieverTest { return new DocumentRetriever( clusterList, mockedFactory, - new LoadTypeSet(), params); } @@ -145,7 +144,7 @@ public class DocumentRetrieverTest { when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); - LoadTypeSet loadTypeSet = new LoadTypeSet(); + LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1); DocumentRetriever documentRetriever = new DocumentRetriever( new ClusterList(), diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 20c7d435964..e69631e8375 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3232,6 +3232,7 @@ "methods": [ "public static boolean isTextCharacter(int)", "public static java.util.OptionalInt validateTextString(java.lang.String)", + "public static boolean isValidTextString(java.lang.String)", "public static boolean isDisplayable(int)", "public static java.lang.String stripInvalidCharacters(java.lang.String)", "public static java.lang.String truncate(java.lang.String, int)", diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index b4898b14b86..ed6ae3678f4 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -36,6 +36,11 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <scope>compile</scope> + </dependency> <!-- provided scope --> <dependency> diff --git a/vespajlib/src/main/java/ai/vespa/validation/Validation.java b/vespajlib/src/main/java/ai/vespa/validation/Validation.java index 816ca931c80..292cb2f0aa5 100644 --- a/vespajlib/src/main/java/ai/vespa/validation/Validation.java +++ b/vespajlib/src/main/java/ai/vespa/validation/Validation.java @@ -37,36 +37,36 @@ public class Validation { /** Requires the value to match the given pattern. */ public static String requireMatch(String value, String description, Pattern pattern) { - return require(pattern.matcher(value).matches(), value, description, "must match '" + pattern + "'"); + return require(pattern.matcher(value).matches(), value, description + " must match '" + pattern + "'"); } /** Requires the value to be non-blank. */ public static String requireNonBlank(String value, String description) { - return require( ! value.isBlank(), value, description, "cannot be blank"); + return require( ! value.isBlank(), value, description + " cannot be blank"); } /** Requires the value to be at least the lower bound. */ public static <T extends Comparable<? super T>> T requireAtLeast(T value, String description, T lower) { - return require(lower.compareTo(value) <= 0, value, description, "must be at least '" + lower + "'"); + return require(lower.compareTo(value) <= 0, value, description + " must be at least '" + lower + "'"); } /** Requires the value to be at most the upper bound. */ public static <T extends Comparable<? super T>> T requireAtMost(T value, String description, T upper) { - return require(upper.compareTo(value) >= 0, value, description, "must be at most '" + upper + "'"); + return require(upper.compareTo(value) >= 0, value, description + " must be at most '" + upper + "'"); } /** Requires the value to be at least the lower bound, and at most the upper bound. */ public static <T extends Comparable<? super T>> T requireInRange(T value, String description, T lower, T upper) { if (lower.compareTo(upper) > 0) throw new IllegalArgumentException("lower bound cannot be greater than upper bound, " + "but got '" + lower + "' > '" + upper + "'"); - return require(lower.compareTo(value) <= 0 && upper.compareTo(value) >= 0, value, description, - "must be at least '" + lower + "' and at most '" + upper + "'"); + return require(lower.compareTo(value) <= 0 && upper.compareTo(value) >= 0, value, + description + " must be at least '" + lower + "' and at most '" + upper + "'"); } /** Returns the argument if the condition is true, otherwise throws. */ - public static <T> T require(boolean condition, T value, String description, String requirement) { + public static <T> T require(boolean condition, T value, String description) { if (condition) return value; - throw new IllegalArgumentException(description + " " + requirement + ", but got: '" + value + "'"); + throw new IllegalArgumentException(description + ", but got: '" + value + "'"); } }
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java new file mode 100644 index 00000000000..e65a645f5be --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java @@ -0,0 +1,216 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import com.yahoo.path.Path; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.function.Predicate; +import java.util.zip.GZIPInputStream; + +/** + * Helper class for safely reading files from a compressed archive. + * + * @author mpolden + */ +public class ArchiveStreamReader implements AutoCloseable { + + private final ArchiveInputStream archiveInputStream; + private final Options options; + + private long totalRead = 0; + private long entriesRead = 0; + + private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { + this.archiveInputStream = Objects.requireNonNull(archiveInputStream); + this.options = Objects.requireNonNull(options); + } + + /** Create reader for an inputStream containing a tar.gz file */ + public static ArchiveStreamReader ofTarGzip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new TarArchiveInputStream(Exceptions.uncheck(() -> new GZIPInputStream(inputStream))), options); + } + + /** Create reader for an inputStream containing a ZIP file */ + public static ArchiveStreamReader ofZip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new ZipArchiveInputStream(inputStream), options); + } + + /** + * Read the next file in this archive and write it to given outputStream. Returns information about the read archive + * file, or null if there are no more files to read. + */ + public ArchiveFile readNextTo(OutputStream outputStream) { + ArchiveEntry entry; + try { + while ((entry = archiveInputStream.getNextEntry()) != null) { + Path path = Path.fromString(requireNormalized(entry.getName(), options.allowDotSegment)); + if (isSymlink(entry)) throw new IllegalArgumentException("Archive entry " + path + " is a symbolic link, which is unsupported"); + if (entry.isDirectory()) continue; + if (!options.pathPredicate.test(path.toString())) continue; + if (++entriesRead > options.maxEntries) throw new IllegalArgumentException("Attempted to read more entries than entry limit of " + options.maxEntries); + + long size = 0; + byte[] buffer = new byte[2048]; + int read; + while ((read = archiveInputStream.read(buffer)) != -1) { + totalRead += read; + size += read; + if (totalRead > options.maxSize) throw new IllegalArgumentException("Total size of archive exceeds size limit of " + options.maxSize + " bytes"); + if (read > options.maxEntrySize) { + if (!options.truncateEntry) throw new IllegalArgumentException("Size of entry " + path + " exceeded entry size limit of " + options.maxEntrySize + " bytes"); + } else { + outputStream.write(buffer, 0, read); + } + } + return new ArchiveFile(path, crc32(entry), size); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return null; + } + + @Override + public void close() { + Exceptions.uncheck(archiveInputStream::close); + } + + /** Information about a file extracted from a compressed archive */ + public static class ArchiveFile { + + private final Path path; + private final OptionalLong crc32; + private final long size; + + public ArchiveFile(Path name, OptionalLong crc32, long size) { + this.path = Objects.requireNonNull(name); + this.crc32 = Objects.requireNonNull(crc32); + if (crc32.isPresent()) { + requireNonNegative("crc32", crc32.getAsLong()); + } + this.size = requireNonNegative("size", size); + } + + /** The path of this file inside its containing archive */ + public Path path() { + return path; + } + + /** The CRC-32 checksum of this file, if any */ + public OptionalLong crc32() { + return crc32; + } + + /** The decompressed size of this file */ + public long size() { + return size; + } + + } + + /** Get the CRC-32 checksum of given archive entry, if any */ + private static OptionalLong crc32(ArchiveEntry entry) { + long crc32 = -1; + if (entry instanceof ZipArchiveEntry) { + crc32 = ((ZipArchiveEntry) entry).getCrc(); + } + return crc32 > -1 ? OptionalLong.of(crc32) : OptionalLong.empty(); + } + + private static boolean isSymlink(ArchiveEntry entry) { + // Symlinks inside ZIP files are not part of the ZIP spec and are only supported by some implementations, such + // as Info-ZIP. + // + // Commons Compress only has limited support for symlinks as they are only detected when the ZIP file is read + // through org.apache.commons.compress.archivers.zip.ZipFile. This is not the case in this class, because it must + // support reading ZIP files from generic input streams. The check below thus always returns false. + if (entry instanceof ZipArchiveEntry) return ((ZipArchiveEntry) entry).isUnixSymlink(); + if (entry instanceof TarArchiveEntry) return ((TarArchiveEntry) entry).isSymbolicLink(); + throw new IllegalArgumentException("Unsupported archive entry " + entry.getClass().getSimpleName() + ", cannot check for symbolic link"); + } + + private static String requireNormalized(String name, boolean allowDotSegment) { + for (var part : name.split("/")) { + if (part.isEmpty() || (!allowDotSegment && part.equals(".")) || part.equals("..")) { + throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); + } + } + return name; + } + + private static long requireNonNegative(String field, long n) { + if (n < 0) throw new IllegalArgumentException(field + " cannot be negative, got " + n); + return n; + } + + /** Options for reading entries of an archive */ + public static class Options { + + private long maxSize = 8 * (long) Math.pow(1024, 3); // 8 GB + private long maxEntrySize = Long.MAX_VALUE; + private long maxEntries = Long.MAX_VALUE; + private boolean truncateEntry = false; + private boolean allowDotSegment = false; + private Predicate<String> pathPredicate = (path) -> true; + + private Options() {} + + /** Returns the standard set of read options */ + public static Options standard() { + return new Options(); + } + + /** Set the maximum total size of decompressed entries. Default is 8 GB */ + public Options maxSize(long size) { + this.maxSize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum size a decompressed entry. Default is no limit */ + public Options maxEntrySize(long size) { + this.maxEntrySize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum number of entries to decompress. Default is no limit */ + public Options maxEntries(long count) { + this.maxEntries = requireNonNegative("count", count); + return this; + } + + /** + * Set whether to truncate the content of an entry exceeding the configured size limit, instead of throwing. + * Default is to throw. + */ + public Options truncateEntry(boolean truncate) { + this.truncateEntry = truncate; + return this; + } + + /** Set a predicate that an entry path must match in order to be extracted. Default is to extract all entries */ + public Options pathPredicate(Predicate<String> predicate) { + this.pathPredicate = predicate; + return this; + } + + /** Set whether to allow single-dot segments in entry paths. Default is false */ + public Options allowDotSegment(boolean allow) { + this.allowDotSegment = allow; + return this; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 662100aa8ea..30eba3ebd65 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -48,7 +48,11 @@ public final class Text { // The link above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed - // see http://www.w3.org/International/questions/qa-controls - if (codepoint < 0x80) return allowedAsciiChars[codepoint]; + return (codepoint < 0x80) + ? allowedAsciiChars[codepoint] + : isTextCharAboveUsAscii(codepoint); + } + private static boolean isTextCharAboveUsAscii(int codepoint) { if (codepoint < 0xFDD0) return true; if (codepoint <= 0xFDDF) return false; if (codepoint < 0x1FFFE) return true; @@ -110,6 +114,23 @@ public final class Text { return OptionalInt.empty(); } + /** + * Validates that the given string value only contains text characters. + */ + public static boolean isValidTextString(String string) { + for (int i = 0; i < string.length(); ) { + int codePoint = string.codePointAt(i); + if ( ! Text.isTextCharacter(codePoint)) return false; + + int charCount = Character.charCount(codePoint); + if (Character.isHighSurrogate(string.charAt(i))) { + if ( (charCount == 1) || !Character.isLowSurrogate(string.charAt(i+1))) return false; + } + i += charCount; + } + return true; + } + /** Returns whether the given code point is displayable. */ public static boolean isDisplayable(int codePoint) { switch (Character.getType(codePoint)) { diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java new file mode 100644 index 00000000000..b7f019282b7 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java @@ -0,0 +1,131 @@ +package com.yahoo.compress; + +import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author mpolden + */ +class ArchiveStreamReaderTest { + + @Test + void reading() { + Map<String, String> zipFiles = Map.of("foo", "contents of foo", + "bar", "contents of bar", + "baz", "0".repeat(2049)); + Map<String, String> zipContents = new HashMap<>(zipFiles); + zipContents.put("dir/", ""); // Directories are always ignored + Map<String, String> extracted = readAll(zip(zipContents), Options.standard()); + assertEquals(zipFiles, extracted); + } + + @Test + void entry_size_limit() { + Map<String, String> entries = Map.of("foo.xml", "foobar"); + Options options = Options.standard().pathPredicate("foo.xml"::equals).maxEntrySize(1); + try { + readAll(zip(entries), options); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + entries = Map.of("foo.xml", "foobar", + "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit + ); + Map<String, String> extracted = readAll(zip(entries), options.maxEntrySize(10)); + assertEquals(Map.of("foo.xml", "foobar"), extracted); + } + + @Test + void size_limit() { + Map<String, String> entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxSize(4)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void entry_limit() { + Map<String, String> entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxEntries(1)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void paths() { + Map<String, Boolean> tests = Map.of( + "../../services.xml", true, + "/../.././services.xml", true, + "./application/././services.xml", true, + "application//services.xml", true, + "artifacts/", false, // empty dir + "services..xml", false, + "application/services.xml", false, + "components/foo-bar-deploy.jar", false, + "services.xml", false + ); + + Options options = Options.standard().maxEntrySize(1024); + tests.forEach((name, expectException) -> { + try { + readAll(zip(Map.of(name, "foo")), options.pathPredicate(name::equals)); + assertFalse(expectException, "Expected exception for '" + name + "'"); + } catch (IllegalArgumentException ignored) { + assertTrue(expectException, "Unexpected exception for '" + name + "'"); + } + }); + } + + private static Map<String, String> readAll(InputStream inputStream, Options options) { + ArchiveStreamReader reader = ArchiveStreamReader.ofZip(inputStream, options); + ArchiveStreamReader.ArchiveFile file; + Map<String, String> entries = new HashMap<>(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.put(file.path().toString(), baos.toString(StandardCharsets.UTF_8)); + baos.reset(); + } + return entries; + } + + private static InputStream zip(Map<String, String> entries) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipArchiveOutputStream archiveOutputStream = null; + try { + archiveOutputStream = new ZipArchiveOutputStream(baos); + for (var kv : entries.entrySet()) { + String entryName = kv.getKey(); + String contents = kv.getValue(); + ZipArchiveEntry entry = new ZipArchiveEntry(entryName); + archiveOutputStream.putArchiveEntry(entry); + archiveOutputStream.write(contents.getBytes(StandardCharsets.UTF_8)); + archiveOutputStream.closeArchiveEntry(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + if (archiveOutputStream != null) Exceptions.uncheck(archiveOutputStream::close); + } + return new ByteArrayInputStream(baos.toByteArray()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index a2cb2158278..33274380aad 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; +import org.junit.Ignore; import org.junit.Test; import java.util.OptionalInt; @@ -76,4 +77,47 @@ public class TextTestCase { public void testFormat() { assertEquals("foo 3.14", Text.format("%s %.2f", "foo", 3.1415926536)); } + + private static long isValid(String [] strings, int num) { + long sum = 0; + for (int i=0; i < num; i++) { + if (Text.isValidTextString(strings[i%strings.length])) { + sum++; + } + } + return sum; + } + private static long validate(String [] strings, int num) { + long sum = 0; + for (int i=0; i < num; i++) { + if (Text.validateTextString(strings[i%strings.length]).isEmpty()) { + sum++; + } + } + return sum; + } + + @Ignore + @Test + public void benchmarkValidate() { + String [] strings = new String[100]; + for (int i=0; i < strings.length; i++) { + strings[i] = new StringBuilder("some text ").append(i).append("of mine.").appendCodePoint(0xDFFFC).append("foo").toString(); + } + long sum = validate(strings, 1000000); + System.out.println("Warmup num validate = " + sum); + sum = isValid(strings, 1000000); + System.out.println("Warmup num isValid = " + sum); + + long start = System.nanoTime(); + sum = validate(strings, 100000000); + long diff = System.nanoTime() - start; + System.out.println("Validation num validate = " + sum + ". Took " + diff + "ns"); + + start = System.nanoTime(); + sum = isValid(strings, 100000000); + diff = System.nanoTime() - start; + System.out.println("Validation num isValid = " + sum + ". Took " + diff + "ns"); + + } } diff --git a/vespalib/src/tests/slime/slime_binary_format_test.cpp b/vespalib/src/tests/slime/slime_binary_format_test.cpp index 459826d691e..ba2aacb88b9 100644 --- a/vespalib/src/tests/slime/slime_binary_format_test.cpp +++ b/vespalib/src/tests/slime/slime_binary_format_test.cpp @@ -248,9 +248,9 @@ TEST("testCmprUlong") { for (uint32_t n = 1; n <= MAX_CMPR_SIZE; ++n) { TEST_STATE(vespalib::make_string("n = %d", n).c_str()); uint64_t min = (n == 1) ? 0x00 - : (1L << ((n - 1) * 7)); + : (1ULL << ((n - 1) * 7)); uint64_t max = (n == MAX_CMPR_SIZE) ? 0xffffffffffffffff - : (1L << (n * 7)) - 1; + : (1ULL << (n * 7)) - 1; SimpleBuffer expect_min; SimpleBuffer expect_max; for (uint32_t i = 0; i < n; ++i) { diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index 72178f0391b..24136e544b8 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -60,7 +60,9 @@ Array<T>::Array(const Array & rhs) : _array(rhs._array.create(rhs.size() * sizeof(T))), _sz(rhs.size()) { - construct(array(0), rhs.array(0), _sz, std::is_trivially_copyable<T>()); + if (_sz > 0) [[likely]] { + construct(array(0), rhs.array(0), _sz, std::is_trivially_copyable<T>()); + } } template <typename T> diff --git a/vespalib/src/vespa/vespalib/util/fiddle.h b/vespalib/src/vespa/vespalib/util/fiddle.h index 20a13ff4654..f4d2ac33695 100644 --- a/vespalib/src/vespa/vespalib/util/fiddle.h +++ b/vespalib/src/vespa/vespalib/util/fiddle.h @@ -24,8 +24,8 @@ uint32_t mix(uint32_t prefix, uint32_t suffix, uint32_t prefix_bits) { if (prefix_bits >= 32) { return prefix; } - uint32_t suffix_mask = (1 << (32 - prefix_bits)) - 1; - uint32_t prefix_mask = (0 - 1) - suffix_mask; + uint32_t suffix_mask = (1u << (32u - prefix_bits)) - 1u; + uint32_t prefix_mask = (0u - 1u) - suffix_mask; return (prefix & prefix_mask) | (suffix & suffix_mask); } |