diff options
author | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-05-25 17:20:33 +0200 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2021-05-25 17:22:10 +0200 |
commit | 34dfcc026213e0a5a4f7c7d1ec6e56d34438d892 (patch) | |
tree | 7de849ec7070f8f2f4d2c8b86167ed86930cc7e4 /vespa-feed-client-cli | |
parent | 69cf843602f1e62582d365acca812357b712e883 (diff) |
Split cli and programmatic API artifacts to separate Maven modules
Diffstat (limited to 'vespa-feed-client-cli')
8 files changed, 567 insertions, 0 deletions
diff --git a/vespa-feed-client-cli/CMakeLists.txt b/vespa-feed-client-cli/CMakeLists.txt new file mode 100644 index 00000000000..a918981dcd3 --- /dev/null +++ b/vespa-feed-client-cli/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(vespa-feed-client-cli) + +vespa_install_script(src/main/sh/vespa-feed-client.sh vespa-feed-client bin) diff --git a/vespa-feed-client-cli/OWNERS b/vespa-feed-client-cli/OWNERS new file mode 100644 index 00000000000..606d074d8a8 --- /dev/null +++ b/vespa-feed-client-cli/OWNERS @@ -0,0 +1,2 @@ +bjorncs +jonmv diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml new file mode 100644 index 00000000000..62ff5c149ec --- /dev/null +++ b/vespa-feed-client-cli/pom.xml @@ -0,0 +1,87 @@ +<?xml version="1.0"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-feed-client-cli</artifactId> + <packaging>jar</packaging> + <version>7-SNAPSHOT</version> + + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> + <!-- Used by internal properties that are still using JDK8--> + <maven.compiler.release>8</maven.compiler.release> + </properties> + + <dependencies> + <!-- compile scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-feed-client</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <scope>compile</scope> + </dependency> + + <!-- test scope --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <archive> + <manifest> + <mainClass>ai.vespa.feed.client.CliClient</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java new file mode 100644 index 00000000000..06c994b12b6 --- /dev/null +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java @@ -0,0 +1,225 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.File; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +/** + * Parses command line arguments + * + * @author bjorncs + */ +class CliArguments { + + private static final Options optionsDefinition = createOptions(); + + private static final String HELP_OPTION = "help"; + private static final String VERSION_OPTION = "version"; + private static final String ENDPOINT_OPTION = "endpoint"; + private static final String FILE_OPTION = "file"; + private static final String CONNECTIONS_OPTION = "connections"; + private static final String MAX_STREAMS_PER_CONNECTION = "max-streams-per-connection"; + private static final String CERTIFICATE_OPTION = "certificate"; + private static final String PRIVATE_KEY_OPTION = "private-key"; + private static final String CA_CERTIFICATES_OPTION = "ca-certificates"; + private static final String DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION = "disable-ssl-hostname-verification"; + private static final String HEADER_OPTION = "header"; + + private final CommandLine arguments; + + private CliArguments(CommandLine arguments) { + this.arguments = arguments; + } + + static CliArguments fromRawArgs(String[] rawArgs) throws CliArgumentsException { + CommandLineParser parser = new DefaultParser(); + try { + return new CliArguments(parser.parse(optionsDefinition, rawArgs)); + } catch (ParseException e) { + throw new CliArgumentsException(e); + } + } + + URI endpoint() throws CliArgumentsException { + try { + URL url = (URL) arguments.getParsedOptionValue(ENDPOINT_OPTION); + if (url == null) throw new CliArgumentsException("Endpoint must be specified"); + return url.toURI(); + } catch (ParseException | URISyntaxException e) { + throw new CliArgumentsException("Invalid endpoint: " + e.getMessage(), e); + } + } + + boolean helpSpecified() { return has(HELP_OPTION); } + + boolean versionSpecified() { return has(VERSION_OPTION); } + + OptionalInt connections() throws CliArgumentsException { return intValue(CONNECTIONS_OPTION); } + + OptionalInt maxStreamsPerConnection() throws CliArgumentsException { return intValue(MAX_STREAMS_PER_CONNECTION); } + + Optional<CertificateAndKey> certificateAndKey() throws CliArgumentsException { + Path certificateFile = fileValue(CERTIFICATE_OPTION).orElse(null); + Path privateKeyFile = fileValue(PRIVATE_KEY_OPTION).orElse(null); + if ((certificateFile == null) != (privateKeyFile == null)) { + throw new CliArgumentsException(String.format("Both '%s' and '%s' must be specified together", CERTIFICATE_OPTION, PRIVATE_KEY_OPTION)); + } + if (privateKeyFile == null && certificateFile == null) return Optional.empty(); + return Optional.of(new CertificateAndKey(certificateFile, privateKeyFile)); + } + + Optional<Path> caCertificates() throws CliArgumentsException { return fileValue(CA_CERTIFICATES_OPTION); } + + Path inputFile() throws CliArgumentsException { + return fileValue(FILE_OPTION) + .orElseThrow(() -> new CliArgumentsException("Feed file must be specified")); + } + + Map<String, String> headers() throws CliArgumentsException { + String[] rawArguments = arguments.getOptionValues(HEADER_OPTION); + if (rawArguments == null) return Collections.emptyMap(); + Map<String, String> headers = new HashMap<>(); + for (String rawArgument : rawArguments) { + if (rawArgument.startsWith("\"") || rawArgument.startsWith("'")) { + rawArgument = rawArgument.substring(1); + } + if (rawArgument.endsWith("\"") || rawArgument.endsWith("'")) { + rawArgument = rawArgument.substring(0, rawArgument.length() - 1); + } + int colonIndex = rawArgument.indexOf(':'); + if (colonIndex == -1) throw new CliArgumentsException("Invalid header: '" + rawArgument + "'"); + headers.put(rawArgument.substring(0, colonIndex), rawArgument.substring(colonIndex + 1).trim()); + } + return Collections.unmodifiableMap(headers); + } + + boolean sslHostnameVerificationDisabled() { return has(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION); } + + private OptionalInt intValue(String option) throws CliArgumentsException { + try { + Number number = (Number) arguments.getParsedOptionValue(option); + return number != null ? OptionalInt.of(number.intValue()) : OptionalInt.empty(); + } catch (ParseException e) { + throw new CliArgumentsException(String.format("Invalid value for '%s': %s", option, e.getMessage()), e); + } + } + + private Optional<Path> fileValue(String option) throws CliArgumentsException { + try { + File certificateFile = (File) arguments.getParsedOptionValue(option); + if (certificateFile == null) return Optional.empty(); + return Optional.of(certificateFile.toPath()); + } catch (ParseException e) { + throw new CliArgumentsException(String.format("Invalid value for '%s': %s", option, e.getMessage()), e); + } + } + + private boolean has(String option) { return arguments.hasOption(option); } + + private static Options createOptions() { + return new Options() + .addOption(Option.builder() + .longOpt(HELP_OPTION) + .build()) + .addOption(Option.builder() + .longOpt(VERSION_OPTION) + .build()) + .addOption(Option.builder() + .longOpt(ENDPOINT_OPTION) + .hasArg() + .type(URL.class) + .build()) + .addOption(Option.builder() + .longOpt(HEADER_OPTION) + .hasArgs() + .build()) + .addOption(Option.builder() + .longOpt(FILE_OPTION) + .type(File.class) + .hasArg() + .build()) + .addOption(Option.builder() + .longOpt(CONNECTIONS_OPTION) + .hasArg() + .type(Number.class) + .build()) + .addOption(Option.builder() + .longOpt(MAX_STREAMS_PER_CONNECTION) + .hasArg() + .type(Number.class) + .build()) + .addOption(Option.builder() + .longOpt(CONNECTIONS_OPTION) + .hasArg() + .type(Number.class) + .build()) + .addOption(Option.builder() + .longOpt(CERTIFICATE_OPTION) + .type(File.class) + .hasArg() + .build()) + .addOption(Option.builder() + .longOpt(PRIVATE_KEY_OPTION) + .type(File.class) + .hasArg() + .build()) + .addOption(Option.builder() + .longOpt(CA_CERTIFICATES_OPTION) + .type(File.class) + .hasArg() + .build()) + .addOption(Option.builder() + .longOpt(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION) + .build()); + } + + void printHelp(OutputStream out) { + HelpFormatter formatter = new HelpFormatter(); + PrintWriter writer = new PrintWriter(out); + formatter.printHelp( + writer, + formatter.getWidth(), + "vespa-feed-client <options>", + "Vespa feed client", + optionsDefinition, + formatter.getLeftPadding(), + formatter.getDescPadding(), + ""); + writer.flush(); + } + + static class CliArgumentsException extends Exception { + CliArgumentsException(String message, Throwable cause) { super(message, cause); } + CliArgumentsException(Throwable cause) { super(cause.getMessage(), cause); } + CliArgumentsException(String message) { super(message); } + } + + static class CertificateAndKey { + final Path certificateFile; + final Path privateKeyFile; + + CertificateAndKey(Path certificateFile, Path privateKeyFile) { + this.certificateFile = certificateFile; + this.privateKeyFile = privateKeyFile; + } + } + +} diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java new file mode 100644 index 00000000000..060f406f38f --- /dev/null +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java @@ -0,0 +1,93 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLSession; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.Properties; + +/** + * Main method for CLI interface + * + * @author bjorncs + */ +public class CliClient { + + private final PrintStream systemOut; + private final PrintStream systemError; + private final Properties systemProperties; + + private CliClient(PrintStream systemOut, PrintStream systemError, Properties systemProperties) { + this.systemOut = systemOut; + this.systemError = systemError; + this.systemProperties = systemProperties; + } + + public static void main(String[] args) { + CliClient client = new CliClient(System.out, System.err, System.getProperties()); + int exitCode = client.run(args); + System.exit(exitCode); + } + + private int run(String[] rawArgs) { + try { + CliArguments cliArgs = CliArguments.fromRawArgs(rawArgs); + if (cliArgs.helpSpecified()) { + cliArgs.printHelp(systemOut); + return 0; + } + if (cliArgs.versionSpecified()) { + systemOut.println(Vespa.VERSION); + return 0; + } + FeedClient feedClient = createFeedClient(cliArgs); + return 0; + } catch (CliArguments.CliArgumentsException | IOException e) { + return handleException(e); + } + } + + private static FeedClient createFeedClient(CliArguments cliArgs) throws CliArguments.CliArgumentsException, IOException { + FeedClientBuilder builder = FeedClientBuilder.create(cliArgs.endpoint()); + cliArgs.connections().ifPresent(builder::setMaxConnections); + cliArgs.maxStreamsPerConnection().ifPresent(builder::setMaxConnections); + if (cliArgs.sslHostnameVerificationDisabled()) { + builder.setHostnameVerifier(AcceptAllHostnameVerifier.INSTANCE); + } + CliArguments.CertificateAndKey certificateAndKey = cliArgs.certificateAndKey().orElse(null); + Path caCertificates = cliArgs.caCertificates().orElse(null); + if (certificateAndKey != null || caCertificates != null) { + SslContextBuilder sslContextBuilder = new SslContextBuilder(); + if (certificateAndKey != null) { + sslContextBuilder.withCertificateAndKey(certificateAndKey.certificateFile, certificateAndKey.privateKeyFile); + } + if (caCertificates != null) { + sslContextBuilder.withCaCertificates(caCertificates); + } + builder.setSslContext(sslContextBuilder.build()); + } + cliArgs.headers().forEach(builder::addRequestHeader); + return builder.build(); + } + + private int handleException(Exception e) { return handleException(e.getMessage(), e); } + + private int handleException(String message, Exception exception) { + systemError.println(message); + if (debugMode()) { + exception.printStackTrace(systemError); + } + return 1; + } + + private boolean debugMode() { + return Boolean.parseBoolean(systemProperties.getProperty("VESPA_DEBUG", Boolean.FALSE.toString())); + } + + private static class AcceptAllHostnameVerifier implements HostnameVerifier { + static final AcceptAllHostnameVerifier INSTANCE = new AcceptAllHostnameVerifier(); + @Override public boolean verify(String hostname, SSLSession session) { return true; } + } +} diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh new file mode 100755 index 00000000000..2a166dd40bb --- /dev/null +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh @@ -0,0 +1,82 @@ +#!/bin/sh +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + empty_if_start_slash=${mypath%%/*} + if [ "${empty_if_start_slash}" ]; then + mypath=$(pwd)/${mypath} + fi + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findhost () { + if [ "${VESPA_HOSTNAME}" = "" ]; then + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 + fi + export VESPA_HOSTNAME +} + +findroot +findhost + +# END environment bootstrap section + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-Djava.awt.headless=true \ +-Xms128m -Xmx2048m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespa-feed-client-cli.jar ai.vespa.feed.client.CliClient "$@" diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/CliArgumentsTest.java new file mode 100644 index 00000000000..be479d294d5 --- /dev/null +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/CliArgumentsTest.java @@ -0,0 +1,61 @@ +package ai.vespa.feed.client;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author bjorncs + */ +class CliArgumentsTest { + + @Test + void parses_parameters_correctly() throws CliArguments.CliArgumentsException { + CliArguments args = CliArguments.fromRawArgs(new String[]{ + "--endpoint=https://vespa.ai:4443/", "--file=feed.json", "--connections=10", + "--max-streams-per-connection=128", "--certificate=cert.pem", "--private-key=key.pem", + "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", + "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value"}); + assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); + assertEquals(Paths.get("feed.json"), args.inputFile()); + assertEquals(10, args.connections().getAsInt()); + assertEquals(128, args.maxStreamsPerConnection().getAsInt()); + assertEquals(Paths.get("cert.pem"), args.certificateAndKey().get().certificateFile); + assertEquals(Paths.get("key.pem"), args.certificateAndKey().get().privateKeyFile); + assertEquals(Paths.get("ca-certs.pem"), args.caCertificates().get()); + assertTrue(args.sslHostnameVerificationDisabled()); + assertFalse(args.helpSpecified()); + assertFalse(args.versionSpecified()); + assertEquals(2, args.headers().size()); + assertEquals("my-value", args.headers().get("My-Header")); + assertEquals("another-value", args.headers().get("Another-Header")); + } + + @Test + void fails_on_missing_parameters() throws CliArguments.CliArgumentsException { + CliArguments cliArguments = CliArguments.fromRawArgs(new String[0]); + CliArguments.CliArgumentsException exception = assertThrows(CliArguments.CliArgumentsException.class, cliArguments::endpoint); + assertEquals("Endpoint must be specified", exception.getMessage()); + exception = assertThrows(CliArguments.CliArgumentsException.class, cliArguments::inputFile); + assertEquals("Feed file must be specified", exception.getMessage()); + } + + @Test + void generated_help_page_contains_expected_description() throws CliArguments.CliArgumentsException, IOException { + CliArguments args = CliArguments.fromRawArgs(new String[]{"--help"}); + assertTrue(args.helpSpecified()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + args.printHelp(out); + String text = out.toString(); + String expectedHelp = new String(Files.readAllBytes(Paths.get("src", "test", "resources", "help.txt"))); + assertEquals(expectedHelp, text); + } + +}
\ No newline at end of file diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt new file mode 100644 index 00000000000..8ad153bc0e0 --- /dev/null +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -0,0 +1,13 @@ +usage: vespa-feed-client <options> +Vespa feed client + --ca-certificates <arg> + --certificate <arg> + --connections <arg> + --disable-ssl-hostname-verification + --endpoint <arg> + --file <arg> + --header <arg> + --help + --max-streams-per-connection <arg> + --private-key <arg> + --version |