diff options
9 files changed, 67 insertions, 13 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index a93741fd8fb..e23f0205e5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -79,8 +79,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { @Override protected Optional<ErrorResponse> filter(DiscFilterRequest request) { try { - Principal principal = request.getUserPrincipal(); - if (principal instanceof AthenzPrincipal) { + if (request.getUserPrincipal() instanceof AthenzPrincipal principal) { Optional<DecodedJWT> oktaAt = Optional.ofNullable((String) request.getAttribute("okta.access-token")).map(JWT::decode); Optional<X509Certificate> cert = request.getClientCertificateChain().stream().findFirst(); Instant issuedAt = cert.map(X509Certificate::getNotBefore) @@ -89,9 +88,8 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { Instant expireAt = cert.map(X509Certificate::getNotAfter) .or(() -> oktaAt.map(Payload::getExpiresAt)) .map(Date::toInstant).orElse(Instant.MAX); - request.setAttribute(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, - roles((AthenzPrincipal) principal, request.getUri()), - issuedAt, expireAt)); + request.setAttribute(SecurityContext.ATTRIBUTE_NAME, + new SecurityContext(principal, roles(principal, request.getUri()), issuedAt, expireAt)); } } catch (Exception e) { diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index e8cb5344aff..ca6cd3161ec 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -123,6 +123,9 @@ public interface FeedClientBuilder { /** Specify HTTP(S) proxy for all endpoints */ FeedClientBuilder setProxy(URI uri); + /** Whether to gzip request bodies; default false */ + FeedClientBuilder setGzipRequests(boolean gzip); + /** Constructs instance of {@link FeedClient} from builder configuration */ FeedClient build(); diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index fd36749b109..f827c2b64ca 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -58,6 +58,7 @@ class CliArguments { private static final String STDIN_OPTION = "stdin"; private static final String DOOM_OPTION = "max-failure-seconds"; private static final String PROXY_OPTION = "proxy"; + private static final String GZIP_REQUESTS_OPTION = "gzip-requests"; private final CommandLine arguments; @@ -181,6 +182,8 @@ class CliArguments { boolean speedTest() { return has(SPEED_TEST_OPTION); } + boolean gzipRequests() { return has(GZIP_REQUESTS_OPTION); } + OptionalInt testPayloadSize() throws CliArgumentsException { return intValue(TEST_PAYLOAD_SIZE_OPTION); } Optional<URI> proxy() throws CliArgumentsException { @@ -354,6 +357,10 @@ class CliArguments { .desc("URI to proxy endpoint") .hasArg() .type(URL.class) + .build()) + .addOption(Option.builder() + .longOpt(GZIP_REQUESTS_OPTION) + .desc("Compress request bodies with gzip") .build()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 43a16c2abf0..6b5e41a6b36 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client.impl; -import ai.vespa.feed.client.DocumentId; import ai.vespa.feed.client.FeedClient; import ai.vespa.feed.client.FeedClientBuilder; import ai.vespa.feed.client.FeedException; @@ -10,7 +9,6 @@ import ai.vespa.feed.client.JsonFeeder.ResultCallback; import ai.vespa.feed.client.OperationStats; import ai.vespa.feed.client.Result; import ai.vespa.feed.client.ResultException; -import ai.vespa.feed.client.impl.CliArguments.CliArgumentsException; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -29,14 +27,12 @@ import java.time.Instant; import java.util.Enumeration; import java.util.Map; import java.util.Random; -import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BooleanSupplier; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.stream.Collectors.joining; @@ -163,6 +159,7 @@ public class CliClient { cliArgs.headers().forEach(builder::addRequestHeader); builder.setDryrun(cliArgs.dryrunEnabled()); builder.setSpeedTest(cliArgs.speedTest()); + builder.setGzipRequests(cliArgs.gzipRequests()); cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10), Duration.ofSeconds(doom)))); cliArgs.proxy().ifPresent(builder::setProxy); diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java index 073ea4a58db..dcfa96d5531 100644 --- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java @@ -28,7 +28,7 @@ class CliArgumentsTest { "--max-streams-per-connection=128", "--certificate=cert.pem", "--private-key=key.pem", "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value", "--benchmark", - "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", + "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", "--gzip-requests", "--show-errors", "--show-all", "--max-failure-seconds=30", "--proxy", "https://myproxy:1234"}); assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); assertEquals(Paths.get("feed.json"), args.inputFile().get()); @@ -52,6 +52,7 @@ class CliArgumentsTest { assertTrue(args.showErrors()); assertTrue(args.showSuccesses()); assertFalse(args.showProgress()); + assertTrue(args.gzipRequests()); assertEquals(URI.create("https://myproxy:1234"), args.proxy().orElse(null)); } diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index e41a78bc932..63e4adf0d7d 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -15,6 +15,8 @@ Vespa feed client across the network --endpoint <arg> URI to feed endpoint --file <arg> Path to feed file in JSON format + --gzip-requests Compress request bodies with + gzip --header <arg> HTTP header on the form 'Name: value' --help diff --git a/vespa-feed-client/pom.xml b/vespa-feed-client/pom.xml index 1cc2f2adee1..01b9b00b8a0 100644 --- a/vespa-feed-client/pom.xml +++ b/vespa-feed-client/pom.xml @@ -46,6 +46,11 @@ <artifactId>junit-jupiter</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-jre8-standalone</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index 3ffbaf136f2..e2678503c68 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -2,23 +2,39 @@ package ai.vespa.feed.client.impl; import ai.vespa.feed.client.HttpResponse; +import org.apache.hc.client5.http.async.methods.SimpleBody; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.entity.GzipCompressingEntity; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.HttpAsyncClients; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.core5.concurrent.FutureCallback; import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.io.entity.BasicHttpEntity; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.message.BasicHttpRequest; +import org.apache.hc.core5.http.nio.AsyncEntityProducer; +import org.apache.hc.core5.http.nio.AsyncRequestProducer; +import org.apache.hc.core5.http.nio.entity.AsyncEntityProducers; +import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityProducer; +import org.apache.hc.core5.http.nio.support.AsyncRequestBuilder; +import org.apache.hc.core5.http.nio.support.BasicRequestProducer; import org.apache.hc.core5.http2.config.H2Config; import org.apache.hc.core5.net.URIAuthority; import org.apache.hc.core5.reactor.IOReactorConfig; import org.apache.hc.core5.util.Timeout; import javax.net.ssl.SSLContext; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -29,6 +45,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeH2Blacklisted; @@ -40,9 +57,11 @@ import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeWeak; class ApacheCluster implements Cluster { private final List<Endpoint> endpoints = new ArrayList<>(); - private final List<BasicHeader> defaultHeaders = Arrays.asList(new BasicHeader("User-Agent", String.format("vespa-feed-client/%s", Vespa.VERSION)), + private final List<BasicHeader> defaultHeaders = Arrays.asList(new BasicHeader(HttpHeaders.USER_AGENT, String.format("vespa-feed-client/%s", Vespa.VERSION)), new BasicHeader("Vespa-Client-Version", Vespa.VERSION)); + private final Header gzipEncodingHeader = new BasicHeader(HttpHeaders.CONTENT_ENCODING, "gzip"); private final RequestConfig requestConfig; + private final boolean gzip; private int someNumber = 0; private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(t -> new Thread(t, "request-timeout-thread")); @@ -52,6 +71,7 @@ class ApacheCluster implements Cluster { for (URI endpoint : builder.endpoints) endpoints.add(new Endpoint(createHttpClient(builder), endpoint)); this.requestConfig = createRequestConfig(builder); + this.gzip = builder.gzipRequests; } @Override @@ -77,8 +97,14 @@ class ApacheCluster implements Cluster { request.setConfig(requestConfig); defaultHeaders.forEach(request::setHeader); wrapped.headers().forEach((name, value) -> request.setHeader(name, value.get())); - if (wrapped.body() != null) - request.setBody(wrapped.body(), ContentType.APPLICATION_JSON); + if (wrapped.body() != null) { + byte[] body = wrapped.body(); + if (gzip) { + request.setHeader(gzipEncodingHeader); + body = gzipped(body); + } + request.setBody(body, ContentType.APPLICATION_JSON); + } Future<?> future = endpoint.client.execute(request, new FutureCallback<SimpleHttpResponse>() { @@ -96,6 +122,14 @@ class ApacheCluster implements Cluster { vessel.whenComplete((__, ___) -> endpoint.inflight.decrementAndGet()); } + private byte[] gzipped(byte[] content) throws IOException{ + ByteArrayOutputStream buffer = new ByteArrayOutputStream(1 << 10); + try (GZIPOutputStream zip = new GZIPOutputStream(buffer)) { + zip.write(content); + } + return buffer.toByteArray(); + } + @Override public void close() { Throwable thrown = null; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 1960991792f..13556b60a28 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -50,6 +50,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { boolean benchmark = true; boolean dryrun = false; boolean speedTest = false; + boolean gzipRequests = false; URI proxy; @@ -200,6 +201,12 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { return this; } + @Override + public FeedClientBuilderImpl setGzipRequests(boolean gzip) { + this.gzipRequests = gzip; + return this; + } + /** Constructs instance of {@link ai.vespa.feed.client.FeedClient} from builder configuration */ @Override public FeedClient build() { |