Add Vespa TLS support to vespa-http-client

Use setters for directly assigning ssl context and hostname validator to
client builder. This ensures that all connection settings applied to the
builder are actual used to construct the client. Previously, the
connection settings for timeout and total connections were ignored if
ssl was configured.

6 files changed, 95 insertions, 22 deletions

diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml
index 18dd34ee7be..96aa6defbe6 100644
--- a/vespa-http-client/pom.xml
+++ b/vespa-http-client/pom.xml
@@ -65,6 +65,22 @@
       <artifactId>airline</artifactId>
       <version>0.6</version>
     </dependency>
+    <dependency>
+      <!-- Needed for Vespa TLS configuration. Standard jar artifact -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>http-utils</artifactId>
+      <version>${project.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <!-- Needed for Vespa TLS configuration. -->
+      <!-- This artifact is packaged as an OSGi bundle - make sure to manually include or exclude transitive dependencies as necessary -->
+      <!-- Note: includes BouncyCastle to compile scope transitively. -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>security-utils</artifactId>
+      <version>${project.version}</version>
+      <scope>compile</scope>
+    </dependency>
 
     <!-- Test dependencies -->
     <dependency>
@@ -140,6 +156,17 @@
             </goals>
             <configuration>
               <outputFile>target/${project.artifactId}-jar-with-dependencies.jar</outputFile>
+              <filters>
+                <filter>
+                  <!-- Don't include signature files in uber jar (most likely from bouncycastle).  -->
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
               <transformers>
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                   <mainClass>com.yahoo.vespa.http.client.runner.Runner</mainClass>
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
index f503190864b..adf61b124ab 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
@@ -43,6 +43,16 @@ public final class ConnectionParams {
         private int traceLevel = 0;
         private int traceEveryXOperation = 0;
         private boolean printTraceToStdErr = true;
+        private boolean useTlsConfigFromEnvironment = false;
+
+        /**
+         * Use TLS configuration through the standard Vespa environment variables.
+         * Setting this to 'true' will override any other TLS/HTTPS related configuration.
+         */
+        public Builder setUseTlsConfigFromEnvironment(boolean useTlsConfigFromEnvironment) {
+            this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment;
+            return this;
+        }
 
         /**
          * Sets the SSLContext for the connection to the gateway when SSL is enabled for Endpoint.
@@ -233,7 +243,8 @@ public final class ConnectionParams {
                     dryRun,
                     traceLevel,
                     traceEveryXOperation,
-                    printTraceToStdErr);
+                    printTraceToStdErr,
+                    useTlsConfigFromEnvironment);
         }
 
         public int getNumPersistentConnectionsPerEndpoint() {
@@ -273,6 +284,10 @@ public final class ConnectionParams {
         public HostnameVerifier getHostnameVerifier() {
             return hostnameVerifier;
         }
+
+        public boolean useTlsConfigFromEnvironment() {
+            return useTlsConfigFromEnvironment;
+        }
     }
     private final SSLContext sslContext;
     private final HostnameVerifier hostnameVerifier;
@@ -288,6 +303,7 @@ public final class ConnectionParams {
     private final int traceLevel;
     private final int traceEveryXOperation;
     private final boolean printTraceToStdErr;
+    private final boolean useTlsConfigFromEnvironment;
 
     private ConnectionParams(
             SSLContext sslContext,
@@ -303,9 +319,11 @@ public final class ConnectionParams {
             boolean dryRun,
             int traceLevel,
             int traceEveryXOperation,
-            boolean printTraceToStdErr) {
+            boolean printTraceToStdErr,
+            boolean useTlsConfigFromEnvironment) {
         this.sslContext = sslContext;
         this.hostnameVerifier = hostnameVerifier;
+        this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment;
         this.headers.putAll(headers);
         this.headerProviders.putAll(headerProviders);
         this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint;
@@ -378,6 +396,10 @@ public final class ConnectionParams {
         return printTraceToStdErr;
     }
 
+    public boolean useTlsConfigFromEnvironment() {
+        return useTlsConfigFromEnvironment;
+    }
+
     /**
      * A header provider that provides a header value. {@link #getHeaderValue()} is called each time a new HTTP request
      * is constructed by {@link com.yahoo.vespa.http.client.FeedClient}.
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
index 5289a7a562a..0b92427bcbe 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
@@ -17,14 +17,8 @@ import org.apache.http.StatusLine;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
 
 import java.io.ByteArrayInputStream;
@@ -400,25 +394,24 @@ class ApacheGatewayConnection implements GatewayConnection {
         }
 
         public HttpClient createClient() {
-            HttpClientBuilder clientBuilder = HttpClientBuilder.create();
-            if (useSsl && connectionParams.getSslContext() != null) {
-                Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
-                        .register("https", new SSLConnectionSocketFactory(
-                                connectionParams.getSslContext(), connectionParams.getHostnameVerifier()))
-                        .register("http", PlainConnectionSocketFactory.INSTANCE)
-                        .build();
-                PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
-                clientBuilder.setConnectionManager(connMgr);
-
+            HttpClientBuilder clientBuilder;
+            if (connectionParams.useTlsConfigFromEnvironment()) {
+                clientBuilder = VespaTlsAwareClientBuilder.createHttpClientBuilder();
+            } else {
+                clientBuilder = HttpClientBuilder.create();
+                if (useSsl && connectionParams.getSslContext() != null) {
+                    clientBuilder.setSslcontext(connectionParams.getSslContext());
+                    clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier());
+                }
             }
-            clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
-            clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
             clientBuilder.setMaxConnPerRoute(1);
             clientBuilder.setMaxConnTotal(1);
+            clientBuilder.setConnectionTimeToLive(15, TimeUnit.SECONDS);
+            clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
+            clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
             clientBuilder.disableContentCompression();
             // Try to disable the disabling to see if system tests become stable again.
             // clientBuilder.disableAutomaticRetries();
-            clientBuilder.setConnectionTimeToLive(15, TimeUnit.SECONDS);
             {
                 RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
                 requestConfigBuilder.setSocketTimeout(0);
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java
new file mode 100644
index 00000000000..be67e11963e
--- /dev/null
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java
@@ -0,0 +1,26 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.client.core.communication;
+
+import org.apache.http.impl.client.HttpClientBuilder;
+
+/**
+ * A static factory for VespaHttpClientBuilder.
+ * The main purpose of this class is to avoid references to classes not compiled with JDK8.
+ *
+ * @author bjorncs
+ */
+// TODO Remove use of reflection once vespa-http-client only targets JDK11
+// The VespaTlsAwareClientBuilder class refers to classes in security-utils / http-utils that targets JDK11+.
+class VespaTlsAwareClientBuilder {
+
+    private VespaTlsAwareClientBuilder() {}
+
+    static HttpClientBuilder createHttpClientBuilder() {
+        try {
+            Class<?> builderClass = Class.forName("ai.vespa.util.http.VespaHttpClientBuilder");
+            return (HttpClientBuilder) builderClass.getMethod("create").invoke(null);
+        } catch (ReflectiveOperationException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/Vtag.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/Vtag.java
index f21a86dfc1b..f44a30208e7 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/Vtag.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/Vtag.java
@@ -3,5 +3,5 @@
 package com.yahoo.vespa.http.client.core.communication;
 
 class Vtag {
-    static final String currentVersion = "7.0.0";
+    static final String currentVersion = "7.1.0";
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
index 4e2c8f1509e..ea0b3f29509 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
@@ -209,6 +209,10 @@ public class CommandLineArguments {
             description = "Add http header to every request. Header must have the format '<Name>: <Value>'. Use this parameter multiple times for multiple headers")
     private List<String> headers = new ArrayList<>();
 
+    @Option(name = {"--vespaTls"},
+            description = "BETA! Use Vespa TLS configuration from environment if available. Other HTTPS/TLS configuration will be ignored if this is set.")
+    private boolean useTlsConfigFromEnvironment = false;
+
     private final List<Header> parsedHeaders = new ArrayList<>();
 
     int getWhenVerboseEnabledPrintMessageForEveryXDocuments() {
@@ -252,6 +256,7 @@ public class CommandLineArguments {
                                 .setTraceEveryXOperation(traceEveryXOperation)
                                 .setPrintTraceToStdErr(traceArg > 0)
                                 .setNumPersistentConnectionsPerEndpoint(numPersistentConnectionsPerEndpoint)
+                                .setUseTlsConfigFromEnvironment(useTlsConfigFromEnvironment)
                                 .build()
                 )
                         // Enable dynamic throttling.