diff options
author | jonmv <venstad@gmail.com> | 2023-01-04 08:53:57 +0100 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2023-01-04 08:53:57 +0100 |
commit | eb0bf8c5bb18ad89ba03d290c6cedc6773b321bc (patch) | |
tree | fb27b08113e31c46015659d89eae0d92230b5711 | |
parent | c9746a990c0daf8103dc7a09ab0d80ee87565a3b (diff) |
Use --compression gzip instead
9 files changed, 109 insertions, 64 deletions
diff --git a/vespa-feed-client-api/abi-spec.json b/vespa-feed-client-api/abi-spec.json index 137c7f32bfe..64b049dc75d 100644 --- a/vespa-feed-client-api/abi-spec.json +++ b/vespa-feed-client-api/abi-spec.json @@ -112,6 +112,23 @@ ], "fields" : [ ] }, + "ai.vespa.feed.client.FeedClientBuilder$Compression" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static ai.vespa.feed.client.FeedClientBuilder$Compression[] values()", + "public static ai.vespa.feed.client.FeedClientBuilder$Compression valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum ai.vespa.feed.client.FeedClientBuilder$Compression none", + "public static final enum ai.vespa.feed.client.FeedClientBuilder$Compression gzip" + ] + }, "ai.vespa.feed.client.FeedClientBuilder" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -142,6 +159,7 @@ "public abstract ai.vespa.feed.client.FeedClientBuilder setCaCertificates(java.util.Collection)", "public abstract ai.vespa.feed.client.FeedClientBuilder setEndpointUris(java.util.List)", "public abstract ai.vespa.feed.client.FeedClientBuilder setProxy(java.net.URI)", + "public abstract ai.vespa.feed.client.FeedClientBuilder setCompression(ai.vespa.feed.client.FeedClientBuilder$Compression)", "public abstract ai.vespa.feed.client.FeedClient build()" ], "fields" : [ diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index ca6cd3161ec..d48c3c31348 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -123,8 +123,10 @@ public interface FeedClientBuilder { /** Specify HTTP(S) proxy for all endpoints */ FeedClientBuilder setProxy(URI uri); - /** Whether to gzip request bodies; default false */ - FeedClientBuilder setGzipRequests(boolean gzip); + /** What compression to use for request bodies; default {@code NONE}. */ + FeedClientBuilder setCompression(Compression compression); + + enum Compression { none, gzip } /** Constructs instance of {@link FeedClient} from builder configuration */ FeedClient build(); diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index f827c2b64ca..42f9713c54e 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client.impl; +import ai.vespa.feed.client.FeedClientBuilder.Compression; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -24,6 +25,8 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; +import static ai.vespa.feed.client.FeedClientBuilder.Compression.none; + /** * Parses command line arguments * @@ -58,7 +61,7 @@ class CliArguments { private static final String STDIN_OPTION = "stdin"; private static final String DOOM_OPTION = "max-failure-seconds"; private static final String PROXY_OPTION = "proxy"; - private static final String GZIP_REQUESTS_OPTION = "gzip-requests"; + private static final String COMPRESSION = "compression"; private final CommandLine arguments; @@ -182,7 +185,14 @@ class CliArguments { boolean speedTest() { return has(SPEED_TEST_OPTION); } - boolean gzipRequests() { return has(GZIP_REQUESTS_OPTION); } + Compression compression() throws CliArgumentsException { + try { + return stringValue(COMPRESSION).map(Compression::valueOf).orElse(none); + } + catch (IllegalArgumentException e) { + throw new CliArgumentsException("Invalid " + COMPRESSION + " argument: " + e.getMessage(), e); + } + } OptionalInt testPayloadSize() throws CliArgumentsException { return intValue(TEST_PAYLOAD_SIZE_OPTION); } @@ -359,8 +369,10 @@ class CliArguments { .type(URL.class) .build()) .addOption(Option.builder() - .longOpt(GZIP_REQUESTS_OPTION) - .desc("Compress request bodies with gzip") + .longOpt(COMPRESSION) + .desc("Compression mode for feed requests: 'none' (default), 'gzip'") + .hasArg() + .type(Compression.class) .build()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 6b5e41a6b36..39462d8ba68 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -159,7 +159,7 @@ public class CliClient { cliArgs.headers().forEach(builder::addRequestHeader); builder.setDryrun(cliArgs.dryrunEnabled()); builder.setSpeedTest(cliArgs.speedTest()); - builder.setGzipRequests(cliArgs.gzipRequests()); + builder.setCompression(cliArgs.compression()); cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10), Duration.ofSeconds(doom)))); cliArgs.proxy().ifPresent(builder::setProxy); diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java index dcfa96d5531..21e279b0584 100644 --- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client.impl; +import ai.vespa.feed.client.FeedClientBuilder.Compression; import ai.vespa.feed.client.impl.CliArguments.CliArgumentsException; import org.junit.jupiter.api.Test; @@ -24,12 +25,27 @@ class CliArgumentsTest { @Test void parses_parameters_correctly() throws CliArguments.CliArgumentsException { CliArguments args = CliArguments.fromRawArgs(new String[]{ - "--endpoint=https://vespa.ai:4443/", "--file=feed.json", "--connections=10", - "--max-streams-per-connection=128", "--certificate=cert.pem", "--private-key=key.pem", - "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", - "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value", "--benchmark", - "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", "--gzip-requests", - "--show-errors", "--show-all", "--max-failure-seconds=30", "--proxy", "https://myproxy:1234"}); + "--endpoint", "https://vespa.ai:4443/", + "--file", "feed.json", + "--connections", "10", + "--max-streams-per-connection", "128", + "--certificate", "cert.pem", + "--private-key", "key.pem", + "--ca-certificates", "ca-certs.pem", + "--disable-ssl-hostname-verification", + "--header", "\"My-Header: my-value\"", + "--header", "Another-Header: another-value", + "--benchmark", + "--route", "myroute", + "--timeout", "0.125", + "--trace", "9", + "--verbose", + "--silent", + "--compression", "gzip", + "--show-errors", + "--show-all", + "--max-failure-seconds", "30", + "--proxy", "https://myproxy:1234"}); assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); assertEquals(Paths.get("feed.json"), args.inputFile().get()); assertEquals(10, args.connections().getAsInt()); @@ -52,15 +68,15 @@ class CliArgumentsTest { assertTrue(args.showErrors()); assertTrue(args.showSuccesses()); assertFalse(args.showProgress()); - assertTrue(args.gzipRequests()); + assertEquals(Compression.gzip, args.compression()); assertEquals(URI.create("https://myproxy:1234"), args.proxy().orElse(null)); } @Test void fails_on_missing_parameters() { - CliArguments.CliArgumentsException exception = assertThrows( + CliArguments.CliArgumentsException exception = assertThrows( CliArguments.CliArgumentsException.class, - () -> CliArguments.fromRawArgs(new String[] {"--file=/path/to/file", "--stdin"})); + () -> CliArguments.fromRawArgs(new String[] {"--file", "/path/to/file", "--stdin"})); assertEquals("Endpoint must be specified", exception.getMessage()); } @@ -68,20 +84,20 @@ class CliArgumentsTest { void fails_on_conflicting_parameters() throws CliArgumentsException { assertEquals("Exactly one of 'file' and 'stdin' must be specified", assertThrows(CliArgumentsException.class, - () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint", "--file=/path/to/file", "--stdin"})) + () -> CliArguments.fromRawArgs(new String[] {"--endpoint", "https://endpoint", "--file", "/path/to/file", "--stdin"})) .getMessage()); assertEquals("Exactly one of 'file' and 'stdin' must be specified", assertThrows(CliArgumentsException.class, - () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint"})) + () -> CliArguments.fromRawArgs(new String[] {"--endpoint", "https://endpoint"})) .getMessage()); assertEquals("At most one of 'file', 'stdin' and 'test-payload-size' may be specified", assertThrows(CliArgumentsException.class, - () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint", "--speed-test", "--test-payload-size=123", "--file=file"})) + () -> CliArguments.fromRawArgs(new String[] {"--endpoint", "https://endpoint", "--speed-test", "--test-payload-size", "123", "--file", "file"})) .getMessage()); - CliArguments.fromRawArgs(new String[] {"--endpoint=foo", "--speed-test"}); + CliArguments.fromRawArgs(new String[] {"--endpoint", "foo", "--speed-test"}); } @Test diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 63e4adf0d7d..f33dde82f7b 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -6,6 +6,9 @@ Vespa feed client certificates encoded as PEM --certificate <arg> Path to PEM encoded X.509 certificate file + --compression <arg> Compression mode for feed + requests: 'none' (default), + 'gzip' --connections <arg> Number of concurrent HTTP/2 connections --disable-ssl-hostname-verification Disable SSL hostname @@ -15,8 +18,6 @@ Vespa feed client across the network --endpoint <arg> URI to feed endpoint --file <arg> Path to feed file in JSON format - --gzip-requests Compress request bodies with - gzip --header <arg> HTTP header on the form 'Name: value' --help diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index e2678503c68..1dda8912046 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client.impl; +import ai.vespa.feed.client.FeedClientBuilder.Compression; import ai.vespa.feed.client.HttpResponse; -import org.apache.hc.client5.http.async.methods.SimpleBody; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.entity.GzipCompressingEntity; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.HttpAsyncClients; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; @@ -15,17 +14,7 @@ import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.io.entity.BasicHttpEntity; -import org.apache.hc.core5.http.io.entity.ByteArrayEntity; -import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; -import org.apache.hc.core5.http.message.BasicHttpRequest; -import org.apache.hc.core5.http.nio.AsyncEntityProducer; -import org.apache.hc.core5.http.nio.AsyncRequestProducer; -import org.apache.hc.core5.http.nio.entity.AsyncEntityProducers; -import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer; -import org.apache.hc.core5.http.nio.support.AsyncRequestBuilder; -import org.apache.hc.core5.http.nio.support.BasicRequestProducer; import org.apache.hc.core5.http2.config.H2Config; import org.apache.hc.core5.net.URIAuthority; import org.apache.hc.core5.reactor.IOReactorConfig; @@ -34,7 +23,6 @@ import org.apache.hc.core5.util.Timeout; import javax.net.ssl.SSLContext; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -47,7 +35,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPOutputStream; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeH2Blacklisted; import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeWeak; @@ -71,7 +58,7 @@ class ApacheCluster implements Cluster { for (URI endpoint : builder.endpoints) endpoints.add(new Endpoint(createHttpClient(builder), endpoint)); this.requestConfig = createRequestConfig(builder); - this.gzip = builder.gzipRequests; + this.gzip = builder.compression == Compression.gzip; } @Override diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 13556b60a28..6886dc3d2b9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; +import static ai.vespa.feed.client.FeedClientBuilder.Compression.none; import static java.util.Objects.requireNonNull; /** @@ -50,7 +51,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { boolean benchmark = true; boolean dryrun = false; boolean speedTest = false; - boolean gzipRequests = false; + Compression compression = none; URI proxy; @@ -202,8 +203,8 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { } @Override - public FeedClientBuilderImpl setGzipRequests(boolean gzip) { - this.gzipRequests = gzip; + public FeedClientBuilderImpl setCompression(Compression compression) { + this.compression = compression; return this; } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java index e665a9f0693..33c043ea271 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java @@ -1,6 +1,8 @@ package ai.vespa.feed.client.impl; +import ai.vespa.feed.client.FeedClientBuilder.Compression; import ai.vespa.feed.client.HttpResponse; +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -34,32 +36,38 @@ class ApacheClusterTest { @Test void testClient() throws IOException, ExecutionException, InterruptedException, TimeoutException { - try (ApacheCluster cluster = new ApacheCluster(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:" + server.port()))) - .setGzipRequests(true))) { - server.stubFor(any(anyUrl())) - .setResponse(okJson("{}").build()); + for (Compression compression : Compression.values()) { + try (ApacheCluster cluster = new ApacheCluster(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:" + server.port()))) + .setCompression(compression))) { + server.stubFor(any(anyUrl())) + .setResponse(okJson("{}").build()); - CompletableFuture<HttpResponse> vessel = new CompletableFuture<>(); - cluster.dispatch(new HttpRequest("POST", - "/path", - Map.of("name1", () -> "value1", - "name2", () -> "value2"), - "content".getBytes(UTF_8), - Duration.ofSeconds(1)), - vessel); - HttpResponse response = vessel.get(5, TimeUnit.SECONDS); - assertEquals("{}", new String(response.body(), UTF_8)); - assertEquals(200, response.code()); + CompletableFuture<HttpResponse> vessel = new CompletableFuture<>(); + cluster.dispatch(new HttpRequest("POST", + "/path", + Map.of("name1", () -> "value1", + "name2", () -> "value2"), + "content".getBytes(UTF_8), + Duration.ofSeconds(1)), + vessel); + HttpResponse response = vessel.get(5, TimeUnit.SECONDS); + assertEquals("{}", new String(response.body(), UTF_8)); + assertEquals(200, response.code()); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (OutputStream zip = new GZIPOutputStream(buffer)) { zip.write("content".getBytes(UTF_8)); } - server.verify(1, anyRequestedFor(anyUrl())); - server.verify(1, postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) - .withHeader("name2", equalTo("value2")) - .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) - .withHeader("Content-Encoding", equalTo("gzip")) - .withRequestBody(equalTo("content"))); - server.resetRequests(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (OutputStream zip = new GZIPOutputStream(buffer)) { zip.write("content".getBytes(UTF_8)); } + server.verify(1, anyRequestedFor(anyUrl())); + RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) + .withHeader("name2", equalTo("value2")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) + .withRequestBody(equalTo("content")); + expected = switch (compression) { + case none -> expected.withoutHeader("Content-Encoding"); + case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip")); + }; + server.verify(1, expected); + server.resetRequests(); + } } } |