Add CLI interface of vespa-feed-client

6 files changed, 493 insertions, 0 deletions

diff --git a/vespa-feed-client/pom.xml b/vespa-feed-client/pom.xml
index d0b0066f07e..cb1e015118e 100644
--- a/vespa-feed-client/pom.xml
+++ b/vespa-feed-client/pom.xml
@@ -42,6 +42,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <scope>compile</scope>
+    </dependency>
 
     <!-- test scope -->
     <dependency>
diff --git a/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliArguments.java b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliArguments.java
new file mode 100644
index 00000000000..3e3cfcc3581
--- /dev/null
+++ b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliArguments.java
@@ -0,0 +1,199 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.feed.client;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.io.File;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.OptionalInt;
+
+/**
+ * Parses command line arguments
+ *
+ * @author bjorncs
+ */
+class CliArguments {
+
+    private static final Options optionsDefinition = createOptions();
+
+    private static final String HELP_OPTION = "help";
+    private static final String VERSION_OPTION = "version";
+    private static final String ENDPOINT_OPTION = "endpoint";
+    private static final String FILE_OPTION = "file";
+    private static final String CONNECTIONS_OPTION = "connections";
+    private static final String MAX_STREAMS_PER_CONNECTION = "max-streams-per-connection";
+    private static final String CERTIFICATE_OPTION = "certificate";
+    private static final String PRIVATE_KEY_OPTION = "key";
+    private static final String CA_CERTIFICATES_OPTION = "ca-certificates";
+    private static final String DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION = "disable-ssl-hostname-verification";
+
+    private final CommandLine arguments;
+
+    private CliArguments(CommandLine arguments) {
+        this.arguments = arguments;
+    }
+
+    static CliArguments fromRawArgs(String[] rawArgs) throws CliArgumentsException {
+        CommandLineParser parser = new DefaultParser();
+        try {
+            return new CliArguments(parser.parse(optionsDefinition, rawArgs));
+        } catch (ParseException e) {
+            throw new CliArgumentsException(e);
+        }
+    }
+
+    URI endpoint() throws CliArgumentsException {
+        try {
+            URL url = (URL) arguments.getParsedOptionValue(ENDPOINT_OPTION);
+            if (url == null) throw new CliArgumentsException("Endpoint must be specified");
+            return url.toURI();
+        } catch (ParseException | URISyntaxException e) {
+            throw new CliArgumentsException("Invalid endpoint: " + e.getMessage(), e);
+        }
+    }
+
+    boolean helpSpecified() { return has(HELP_OPTION); }
+
+    boolean versionSpecified() { return has(VERSION_OPTION); }
+
+    OptionalInt connections() throws CliArgumentsException {  return intValue(CONNECTIONS_OPTION); }
+
+    OptionalInt maxStreamsPerConnection() throws CliArgumentsException { return intValue(MAX_STREAMS_PER_CONNECTION); }
+
+    Optional<CertificateAndKey> certificateAndKey() throws CliArgumentsException {
+        Path certificateFile = fileValue(CERTIFICATE_OPTION).orElse(null);
+        Path privateKeyFile = fileValue(PRIVATE_KEY_OPTION).orElse(null);
+        if ((certificateFile != null && privateKeyFile == null) || (certificateFile == null && privateKeyFile != null)) {
+            throw new CliArgumentsException(String.format("Both '%s' and '%s' must be specified together", CERTIFICATE_OPTION, PRIVATE_KEY_OPTION));
+        }
+        if (privateKeyFile == null && certificateFile == null) return Optional.empty();
+        return Optional.of(new CertificateAndKey(certificateFile, privateKeyFile));
+    }
+
+    Optional<Path> caCertificates() throws CliArgumentsException { return fileValue(CA_CERTIFICATES_OPTION); }
+
+    Path inputFile() throws CliArgumentsException {
+        return fileValue(FILE_OPTION)
+            .orElseThrow(() -> new CliArgumentsException("Feed file must be specified"));
+    }
+
+    boolean sslHostnameVerificationDisabled() { return has(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION); }
+
+    private OptionalInt intValue(String option) throws CliArgumentsException {
+        try {
+            Number number = (Number) arguments.getParsedOptionValue(option);
+            return number != null ? OptionalInt.of(number.intValue()) : OptionalInt.empty();
+        } catch (ParseException e) {
+            throw new CliArgumentsException(String.format("Invalid value for '%s': %s", option, e.getMessage()), e);
+        }
+    }
+
+    private Optional<Path> fileValue(String option) throws CliArgumentsException {
+        try {
+            File certificateFile = (File) arguments.getParsedOptionValue(option);
+            if (certificateFile == null) return Optional.empty();
+            return Optional.of(certificateFile.toPath());
+        } catch (ParseException e) {
+            throw new CliArgumentsException(String.format("Invalid value for '%s': %s", option, e.getMessage()), e);
+        }
+    }
+
+    private boolean has(String option) { return arguments.hasOption(option); }
+
+    private static Options createOptions() {
+        return new Options()
+                .addOption(Option.builder()
+                        .longOpt(HELP_OPTION)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(VERSION_OPTION)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(ENDPOINT_OPTION)
+                        .hasArg()
+                        .type(URL.class)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(FILE_OPTION)
+                        .type(File.class)
+                        .hasArg()
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(CONNECTIONS_OPTION)
+                        .hasArg()
+                        .type(Number.class)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(MAX_STREAMS_PER_CONNECTION)
+                        .hasArg()
+                        .type(Number.class)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(CONNECTIONS_OPTION)
+                        .hasArg()
+                        .type(Number.class)
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(CERTIFICATE_OPTION)
+                        .type(File.class)
+                        .hasArg()
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(PRIVATE_KEY_OPTION)
+                        .type(File.class)
+                        .hasArg()
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(CA_CERTIFICATES_OPTION)
+                        .type(File.class)
+                        .hasArg()
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION)
+                        .build());
+    }
+
+    void printHelp(OutputStream out) {
+        HelpFormatter formatter = new HelpFormatter();
+        PrintWriter writer = new PrintWriter(out);
+        formatter.printHelp(
+                writer,
+                formatter.getWidth(),
+                "vespa-feed-client <options>",
+                "Vespa feed client (" + Vespa.VERSION + ")",
+                optionsDefinition,
+                formatter.getLeftPadding(),
+                formatter.getDescPadding(),
+                "");
+        writer.flush();
+    }
+
+    static class CliArgumentsException extends Exception {
+        CliArgumentsException(String message, Throwable cause) { super(message, cause); }
+        CliArgumentsException(Throwable cause) { super(cause.getMessage(), cause); }
+        CliArgumentsException(String message) { super(message); }
+    }
+
+    static class CertificateAndKey {
+        final Path certificateFile;
+        final Path privateKeyFile;
+
+        CertificateAndKey(Path certificateFile, Path privateKeyFile) {
+            this.certificateFile = certificateFile;
+            this.privateKeyFile = privateKeyFile;
+        }
+    }
+
+}
diff --git a/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliClient.java b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliClient.java
new file mode 100644
index 00000000000..5a4a4ead631
--- /dev/null
+++ b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/CliClient.java
@@ -0,0 +1,94 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.feed.client;
+
+import com.yahoo.vespa.feed.client.CliArguments.CliArgumentsException;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Path;
+import java.util.Properties;
+
+/**
+ * Main method for CLI interface
+ *
+ * @author bjorncs
+ */
+class CliClient {
+
+    private final PrintStream systemOut;
+    private final PrintStream systemError;
+    private final Properties systemProperties;
+
+    CliClient(PrintStream systemOut, PrintStream systemError, Properties systemProperties) {
+        this.systemOut = systemOut;
+        this.systemError = systemError;
+        this.systemProperties = systemProperties;
+    }
+
+    public static void main(String[] args) {
+        CliClient client = new CliClient(System.out, System.err, System.getProperties());
+        int exitCode = client.run(args);
+        System.exit(exitCode);
+    }
+
+    int run(String[] rawArgs) {
+        try {
+            CliArguments cliArgs = CliArguments.fromRawArgs(rawArgs);
+            if (cliArgs.helpSpecified()) {
+                cliArgs.printHelp(systemOut);
+                return 0;
+            }
+            if (cliArgs.versionSpecified()) {
+                systemOut.println(Vespa.VERSION);
+                return 0;
+            }
+            FeedClient feedClient = createFeedClient(cliArgs);
+            return 0;
+        } catch (CliArgumentsException | IOException e) {
+            return handleException(e);
+        }
+    }
+
+    private static FeedClient createFeedClient(CliArguments cliArgs) throws CliArgumentsException, IOException {
+        FeedClientBuilder builder = FeedClientBuilder.create(cliArgs.endpoint());
+        cliArgs.connections().ifPresent(builder::setMaxConnections);
+        cliArgs.maxStreamsPerConnection().ifPresent(builder::setMaxConnections);
+        if (cliArgs.sslHostnameVerificationDisabled()) {
+            builder.setHostnameVerifier(AcceptAllHostnameVerifier.INSTANCE);
+        }
+        CliArguments.CertificateAndKey certificateAndKey = cliArgs.certificateAndKey().orElse(null);
+        Path caCertificates = cliArgs.caCertificates().orElse(null);
+        if (certificateAndKey != null || caCertificates != null) {
+            SslContextBuilder sslContextBuilder = new SslContextBuilder();
+            if (certificateAndKey != null) {
+                sslContextBuilder.withCertificateAndKey(certificateAndKey.certificateFile, certificateAndKey.privateKeyFile);
+            }
+            if (caCertificates != null) {
+                sslContextBuilder.withCaCertificates(caCertificates);
+            }
+            builder.setSslContext(sslContextBuilder.build());
+        }
+        return builder.build();
+    }
+
+    private int handleException(Exception e) { return handleException(e.getMessage(), e); }
+
+    private int handleException(String message, Exception exception) {
+        systemError.println(message);
+        if (debugMode()) {
+            exception.printStackTrace(systemError);
+        }
+        return 1;
+    }
+
+    private boolean debugMode() {
+        return Boolean.parseBoolean(systemProperties.getProperty("VESPA_DEBUG", Boolean.FALSE.toString()));
+    }
+
+    private static class AcceptAllHostnameVerifier implements HostnameVerifier {
+        static final AcceptAllHostnameVerifier INSTANCE = new AcceptAllHostnameVerifier();
+        @Override public boolean verify(String hostname, SSLSession session) { return true; }
+    }
+}
diff --git a/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/SslContextBuilder.java b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/SslContextBuilder.java
new file mode 100644
index 00000000000..326ead6d005
--- /dev/null
+++ b/vespa-feed-client/src/main/java/com/yahoo/vespa/feed/client/SslContextBuilder.java
@@ -0,0 +1,126 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.feed.client;
+
+import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
+import org.bouncycastle.asn1.x9.X9ObjectIdentifiers;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openssl.PEMKeyPair;
+import org.bouncycastle.openssl.PEMParser;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyFactory;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * BouncyCastle integration for creating a {@link SSLContext} instance from PEM encoded material
+ *
+ * @author bjorncs
+ */
+class SslContextBuilder {
+
+    private static final BouncyCastleProvider bcProvider = new BouncyCastleProvider();
+
+    private Path certificateFile;
+    private Path privateKeyFile;
+    private Path caCertificatesFile;
+
+    SslContextBuilder withCertificateAndKey(Path certificate, Path privateKey) {
+        this.certificateFile = certificate;
+        this.privateKeyFile = privateKey;
+        return this;
+    }
+
+    SslContextBuilder withCaCertificates(Path caCertificates) {
+        this.caCertificatesFile = caCertificates;
+        return this;
+    }
+
+    SSLContext build() throws IOException {
+        try {
+            KeyStore keystore = KeyStore.getInstance("PKCS12");
+            if (certificateFile != null && privateKeyFile != null) {
+                keystore.setKeyEntry("cert", privateKey(privateKeyFile), new char[0], certificates(certificateFile));
+            }
+            if (caCertificatesFile != null) {
+                keystore.setCertificateEntry("ca-cert", certificates(caCertificatesFile)[0]);
+            }
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+            kmf.init(keystore, new char[0]);
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            tmf.init(keystore);
+            SSLContext sslContext = SSLContext.getDefault();
+            sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+            return sslContext;
+        } catch (GeneralSecurityException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private static Certificate[] certificates(Path file) throws IOException, GeneralSecurityException {
+        try (PEMParser parser = new PEMParser(Files.newBufferedReader(file))) {
+            List<X509Certificate> result = new ArrayList<>();
+            Object pemObject;
+            while ((pemObject = parser.readObject()) != null) {
+                result.add(toX509Certificate(pemObject));
+            }
+            if (result.isEmpty()) throw new IOException("File contains no PEM encoded certificates: " + file);
+            return result.toArray(new Certificate[0]);
+        }
+    }
+
+    private static PrivateKey privateKey(Path file) throws IOException, GeneralSecurityException {
+        try (PEMParser parser = new PEMParser(Files.newBufferedReader(file))) {
+            Object pemObject;
+            while ((pemObject = parser.readObject()) != null) {
+                if (pemObject instanceof PrivateKeyInfo) {
+                    PrivateKeyInfo keyInfo = (PrivateKeyInfo) pemObject;
+                    PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyInfo.getEncoded());
+                    return createKeyFactory(keyInfo).generatePrivate(keySpec);
+                } else if (pemObject instanceof PEMKeyPair) {
+                    PEMKeyPair pemKeypair = (PEMKeyPair) pemObject;
+                    PrivateKeyInfo keyInfo = pemKeypair.getPrivateKeyInfo();
+                    return createKeyFactory(keyInfo).generatePrivate(new PKCS8EncodedKeySpec(keyInfo.getEncoded()));
+                }
+            }
+            throw new IOException("Could not find private key in PEM file");
+        }
+    }
+
+    private static X509Certificate toX509Certificate(Object pemObject) throws IOException, GeneralSecurityException {
+        if (pemObject instanceof X509Certificate) return (X509Certificate) pemObject;
+        if (pemObject instanceof X509CertificateHolder) {
+            return new JcaX509CertificateConverter()
+                    .setProvider(bcProvider)
+                    .getCertificate((X509CertificateHolder) pemObject);
+        }
+        throw new IOException("Invalid type of PEM object: " + pemObject);
+    }
+
+    private static KeyFactory createKeyFactory(PrivateKeyInfo info) throws IOException, GeneralSecurityException {
+        ASN1ObjectIdentifier algorithm = info.getPrivateKeyAlgorithm().getAlgorithm();
+        if (X9ObjectIdentifiers.id_ecPublicKey.equals(algorithm)) {
+            return KeyFactory.getInstance("EC", bcProvider);
+        } else if (PKCSObjectIdentifiers.rsaEncryption.equals(algorithm)) {
+            return KeyFactory.getInstance("RSA", bcProvider);
+        } else {
+            throw new IOException("Unknown key algorithm: " + algorithm);
+        }
+    }
+
+}
diff --git a/vespa-feed-client/src/test/java/com/yahoo/vespa/feed/client/CliArgumentsTest.java b/vespa-feed-client/src/test/java/com/yahoo/vespa/feed/client/CliArgumentsTest.java
new file mode 100644
index 00000000000..d9229c25ad9
--- /dev/null
+++ b/vespa-feed-client/src/test/java/com/yahoo/vespa/feed/client/CliArgumentsTest.java
@@ -0,0 +1,57 @@
+package com.yahoo.vespa.feed.client;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import com.yahoo.vespa.feed.client.CliArguments.CliArgumentsException;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author bjorncs
+ */
+class CliArgumentsTest {
+
+    @Test
+    void parses_parameters_correctly() throws CliArgumentsException {
+        CliArguments args = CliArguments.fromRawArgs(new String[]{
+                "--endpoint=https://vespa.ai:4443/", "--file=feed.json", "--connections=10",
+                "--max-streams-per-connection=128", "--certificate=cert.pem", "--key=key.pem",
+                "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification"});
+        assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint());
+        assertEquals(Paths.get("feed.json"), args.inputFile());
+        assertEquals(10, args.connections().getAsInt());
+        assertEquals(128, args.maxStreamsPerConnection().getAsInt());
+        assertEquals(Paths.get("cert.pem"), args.certificateAndKey().get().certificateFile);
+        assertEquals(Paths.get("key.pem"), args.certificateAndKey().get().privateKeyFile);
+        assertEquals(Paths.get("ca-certs.pem"), args.caCertificates().get());
+        assertTrue(args.sslHostnameVerificationDisabled());
+        assertFalse(args.helpSpecified());
+    }
+
+    @Test
+    void fails_on_missing_parameters() throws CliArgumentsException {
+        CliArguments cliArguments = CliArguments.fromRawArgs(new String[0]);
+        CliArgumentsException exception =  assertThrows(CliArgumentsException.class, cliArguments::endpoint);
+        assertEquals("Endpoint must be specified", exception.getMessage());
+        exception =  assertThrows(CliArgumentsException.class, cliArguments::inputFile);
+        assertEquals("Feed file must be specified", exception.getMessage());
+    }
+
+    @Test
+    void generated_help_page_contains_expected_description() throws CliArgumentsException, IOException {
+        CliArguments args = CliArguments.fromRawArgs(new String[]{"--help"});
+        assertTrue(args.helpSpecified());
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        args.printHelp(out);
+        String text = out.toString();
+        String expectedHelp = new String(Files.readAllBytes(Paths.get("src", "test", "resources", "help.txt")));
+        assertEquals(expectedHelp, text);
+    }
+
+}
\ No newline at end of file
diff --git a/vespa-feed-client/src/test/resources/help.txt b/vespa-feed-client/src/test/resources/help.txt
new file mode 100644
index 00000000000..1ca7a7dfc7f
--- /dev/null
+++ b/vespa-feed-client/src/test/resources/help.txt
@@ -0,0 +1,12 @@
+usage: vespa-feed-client <options>
+Vespa feed client (7.164.0)
+    --ca-certificates <arg>
+    --certificate <arg>
+    --connections <arg>
+    --disable-ssl-hostname-verification
+    --endpoint <arg>
+    --file <arg>
+    --help
+    --key <arg>
+    --max-streams-per-connection <arg>
+    --version