diff options
6 files changed, 120 insertions, 21 deletions
diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index 18dd34ee7be..96aa6defbe6 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -65,6 +65,22 @@ <artifactId>airline</artifactId> <version>0.6</version> </dependency> + <dependency> + <!-- Needed for Vespa TLS configuration. Standard jar artifact --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>http-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <!-- Needed for Vespa TLS configuration. --> + <!-- This artifact is packaged as an OSGi bundle - make sure to manually include or exclude transitive dependencies as necessary --> + <!-- Note: includes BouncyCastle to compile scope transitively. --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> <!-- Test dependencies --> <dependency> @@ -140,6 +156,17 @@ </goals> <configuration> <outputFile>target/${project.artifactId}-jar-with-dependencies.jar</outputFile> + <filters> + <filter> + <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>com.yahoo.vespa.http.client.runner.Runner</mainClass> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index f503190864b..adf61b124ab 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -43,6 +43,16 @@ public final class ConnectionParams { private int traceLevel = 0; private int traceEveryXOperation = 0; private boolean printTraceToStdErr = true; + private boolean useTlsConfigFromEnvironment = false; + + /** + * Use TLS configuration through the standard Vespa environment variables. + * Setting this to 'true' will override any other TLS/HTTPS related configuration. + */ + public Builder setUseTlsConfigFromEnvironment(boolean useTlsConfigFromEnvironment) { + this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; + return this; + } /** * Sets the SSLContext for the connection to the gateway when SSL is enabled for Endpoint. @@ -233,7 +243,8 @@ public final class ConnectionParams { dryRun, traceLevel, traceEveryXOperation, - printTraceToStdErr); + printTraceToStdErr, + useTlsConfigFromEnvironment); } public int getNumPersistentConnectionsPerEndpoint() { @@ -273,6 +284,10 @@ public final class ConnectionParams { public HostnameVerifier getHostnameVerifier() { return hostnameVerifier; } + + public boolean useTlsConfigFromEnvironment() { + return useTlsConfigFromEnvironment; + } } private final SSLContext sslContext; private final HostnameVerifier hostnameVerifier; @@ -288,6 +303,7 @@ public final class ConnectionParams { private final int traceLevel; private final int traceEveryXOperation; private final boolean printTraceToStdErr; + private final boolean useTlsConfigFromEnvironment; private ConnectionParams( SSLContext sslContext, @@ -303,9 +319,11 @@ public final class ConnectionParams { boolean dryRun, int traceLevel, int traceEveryXOperation, - boolean printTraceToStdErr) { + boolean printTraceToStdErr, + boolean useTlsConfigFromEnvironment) { this.sslContext = sslContext; this.hostnameVerifier = hostnameVerifier; + this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; this.headers.putAll(headers); this.headerProviders.putAll(headerProviders); this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; @@ -378,6 +396,10 @@ public final class ConnectionParams { return printTraceToStdErr; } + public boolean useTlsConfigFromEnvironment() { + return useTlsConfigFromEnvironment; + } + /** * A header provider that provides a header value. {@link #getHeaderValue()} is called each time a new HTTP request * is constructed by {@link com.yahoo.vespa.http.client.FeedClient}. diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 5289a7a562a..fcb5e03878d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -17,14 +17,8 @@ import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; import java.io.ByteArrayInputStream; @@ -400,25 +394,26 @@ class ApacheGatewayConnection implements GatewayConnection { } public HttpClient createClient() { - HttpClientBuilder clientBuilder = HttpClientBuilder.create(); - if (useSsl && connectionParams.getSslContext() != null) { - Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() - .register("https", new SSLConnectionSocketFactory( - connectionParams.getSslContext(), connectionParams.getHostnameVerifier())) - .register("http", PlainConnectionSocketFactory.INSTANCE) - .build(); - PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - clientBuilder.setConnectionManager(connMgr); - + HttpClientBuilder clientBuilder; + if (connectionParams.useTlsConfigFromEnvironment()) { + // Note: connection settings are confgured in createVespaTlsAwareClientBuilder. + clientBuilder = createVespaTlsAwareClientBuilder(); + } else { + clientBuilder = HttpClientBuilder.create(); + if (useSsl && connectionParams.getSslContext() != null) { + clientBuilder.setSslcontext(connectionParams.getSslContext()); + clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier()); + } + clientBuilder.setMaxConnPerRoute(1); + clientBuilder.setMaxConnTotal(1); + clientBuilder.setConnectionTimeToLive(15, TimeUnit.SECONDS); } + clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion)); clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion))); - clientBuilder.setMaxConnPerRoute(1); - clientBuilder.setMaxConnTotal(1); clientBuilder.disableContentCompression(); // Try to disable the disabling to see if system tests become stable again. // clientBuilder.disableAutomaticRetries(); - clientBuilder.setConnectionTimeToLive(15, TimeUnit.SECONDS); { RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setSocketTimeout(0); @@ -436,6 +431,19 @@ class ApacheGatewayConnection implements GatewayConnection { ); return clientBuilder.build(); } + + // TODO Remove use of reflection once vespa-http-client only targets JDK11 + // The VespaTlsAwareClientBuilder class refers to classes in security-utils / http-utils that targets JDK11+. + // We cannot use VespaTlsAwareClientBuilder because the classloader might load it even when the createHttpClientBuilder is not invoked. + private HttpClientBuilder createVespaTlsAwareClientBuilder() { + try { + Class<?> builderClass = + Class.forName("com.yahoo.vespa.http.client.core.communication.VespaTlsAwareClientBuilder"); + return (HttpClientBuilder) builderClass.getMethod("createHttpClientBuilder").invoke(null); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } } } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java new file mode 100644 index 00000000000..08da9dfbe2f --- /dev/null +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.client.core.communication; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +import java.util.concurrent.TimeUnit; + +/** + * A static factory for VespaHttpClientBuilder. The main purpose of this class is to isolate references to classes targeting JDK11+. + * + * @author bjorncs + */ +// TODO Inline once vespa-http-client no longer targets JDK8 +@SuppressWarnings("unused") // used through reflection from ApacheGatewayConnection +public class VespaTlsAwareClientBuilder { + + private VespaTlsAwareClientBuilder() {} + + @SuppressWarnings("unused") // used through reflection from ApacheGatewayConnection + public static HttpClientBuilder createHttpClientBuilder() { + return VespaHttpClientBuilder.create(socketFactoryRegistry -> { + PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager( + socketFactoryRegistry, + null, null, null, + 15, TimeUnit.SECONDS); + manager.setDefaultMaxPerRoute(1); + manager.setMaxTotal(1); + return manager; + }); + } +} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 4e2c8f1509e..ea0b3f29509 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -209,6 +209,10 @@ public class CommandLineArguments { description = "Add http header to every request. Header must have the format '<Name>: <Value>'. Use this parameter multiple times for multiple headers") private List<String> headers = new ArrayList<>(); + @Option(name = {"--vespaTls"}, + description = "BETA! Use Vespa TLS configuration from environment if available. Other HTTPS/TLS configuration will be ignored if this is set.") + private boolean useTlsConfigFromEnvironment = false; + private final List<Header> parsedHeaders = new ArrayList<>(); int getWhenVerboseEnabledPrintMessageForEveryXDocuments() { @@ -252,6 +256,7 @@ public class CommandLineArguments { .setTraceEveryXOperation(traceEveryXOperation) .setPrintTraceToStdErr(traceArg > 0) .setNumPersistentConnectionsPerEndpoint(numPersistentConnectionsPerEndpoint) + .setUseTlsConfigFromEnvironment(useTlsConfigFromEnvironment) .build() ) // Enable dynamic throttling. diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 959fb687692..53e708601d7 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -55,6 +55,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> </exclusions> </dependency> <dependency> |