diff options
8 files changed, 73 insertions, 50 deletions
diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java index a3e33699a4d..18c1c6a22fa 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliArguments.java @@ -38,6 +38,7 @@ class CliArguments { private static final String CERTIFICATE_OPTION = "certificate"; private static final String CONNECTIONS_OPTION = "connections"; private static final String DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION = "disable-ssl-hostname-verification"; + private static final String DRYRUN_OPTION = "dryrun"; private static final String ENDPOINT_OPTION = "endpoint"; private static final String FILE_OPTION = "file"; private static final String HEADER_OPTION = "header"; @@ -150,6 +151,8 @@ class CliArguments { boolean readFeedFromStandardInput() { return has(STDIN_OPTION); } + boolean dryrunEnabled() { return has(DRYRUN_OPTION); } + private OptionalInt intValue(String option) throws CliArgumentsException { try { Number number = (Number) arguments.getParsedOptionValue(option); @@ -272,6 +275,10 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(VERBOSE_OPTION) + .build()) + .addOption(Option.builder() + .longOpt(DRYRUN_OPTION) + .desc("Enable dryrun mode where each operation succeeds after " + DryrunCluster.DELAY.toMillis() + "ms") .build()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java index 83abe0bb872..1c20f6b5c1b 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/CliClient.java @@ -11,11 +11,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Files; -import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.concurrent.atomic.AtomicInteger; /** * Main method for CLI interface @@ -86,6 +84,7 @@ public class CliClient { cliArgs.certificateAndKey().ifPresent(c -> builder.setCertificate(c.certificateFile, c.privateKeyFile)); cliArgs.caCertificates().ifPresent(builder::setCaCertificatesFile); cliArgs.headers().forEach(builder::addRequestHeader); + builder.setDryrun(cliArgs.dryrunEnabled()); return builder.build(); } diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 1c537defb01..b5d59e02f7e 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -9,6 +9,8 @@ Vespa feed client connections --disable-ssl-hostname-verification Disable SSL hostname verification + --dryrun Enable dryrun mode where each + operation succeeds after 1ms --endpoint <arg> URI to feed endpoint --file <arg> Path to feed file in JSON format --header <arg> HTTP header on the form 'Name: diff --git a/vespa-feed-client/abi-spec.json b/vespa-feed-client/abi-spec.json index 70cb4c3f09f..d794fcf851a 100644 --- a/vespa-feed-client/abi-spec.json +++ b/vespa-feed-client/abi-spec.json @@ -175,6 +175,7 @@ "public ai.vespa.feed.client.FeedClientBuilder setCertificate(java.nio.file.Path, java.nio.file.Path)", "public ai.vespa.feed.client.FeedClientBuilder setCertificate(java.util.Collection, java.security.PrivateKey)", "public ai.vespa.feed.client.FeedClientBuilder setCertificate(java.security.cert.X509Certificate, java.security.PrivateKey)", + "public ai.vespa.feed.client.FeedClientBuilder setDryrun(boolean)", "public ai.vespa.feed.client.FeedClientBuilder setCaCertificatesFile(java.nio.file.Path)", "public ai.vespa.feed.client.FeedClientBuilder setCaCertificates(java.util.Collection)", "public ai.vespa.feed.client.FeedClient build()" diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DryrunCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DryrunCluster.java new file mode 100644 index 00000000000..9e6ad0150c7 --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DryrunCluster.java @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Dryrun implementation that reports every request/operation as successful + * + * @author bjorncs + */ +class DryrunCluster implements Cluster { + + private final static Logger log = Logger.getLogger(DryrunCluster.class.getName()); + + static final Duration DELAY = Duration.ofMillis(1); + + @Override + public void dispatch(HttpRequest request, CompletableFuture<HttpResponse> vessel) { + long millis = DELAY.toMillis(); + log.log(Level.FINE, "Dryrun of request '{0}' with delay of {1}ms", new Object[]{request, millis}); + if (millis > 0) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + vessel.cancel(true); + Thread.currentThread().interrupt(); + return; + } + } + vessel.complete(new SimpleOkResponse()); + } + + private static class SimpleOkResponse implements HttpResponse { + @Override public int code() { return 200; } + @Override public byte[] body() { return "{\"message\":\"dummy dryrun message\"}".getBytes(StandardCharsets.UTF_8); } + } + +} diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index 0f685ec5b7f..57aaf67c2d9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -45,6 +45,7 @@ public class FeedClientBuilder { PrivateKey privateKey; Collection<X509Certificate> caCertificates; boolean benchmark; + boolean dryrun; /** Creates a builder for a single container endpoint **/ public static FeedClientBuilder create(URI endpoint) { return new FeedClientBuilder(Collections.singletonList(endpoint)); } @@ -158,6 +159,11 @@ public class FeedClientBuilder { return setCertificate(Collections.singletonList(certificate), privateKey); } + public FeedClientBuilder setDryrun(boolean enabled) { + this.dryrun = enabled; + return this; + } + /** * Overrides JVM default SSL truststore * @param caCertificatesFile Path to PEM encoded file containing trusted certificates diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 1987bae18f9..1b939a0361c 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -59,7 +59,7 @@ class HttpRequestStrategy implements RequestStrategy { }); HttpRequestStrategy(FeedClientBuilder builder) throws IOException { - this(builder, new ApacheCluster(builder)); + this(builder, builder.dryrun ? new DryrunCluster() : new ApacheCluster(builder)); } HttpRequestStrategy(FeedClientBuilder builder, Cluster cluster) { diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 84dbe919c5c..9bd285b7cca 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -1,15 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hadoop.mapreduce; -import ai.vespa.feed.client.DocumentId; -import ai.vespa.feed.client.DryrunResult; import ai.vespa.feed.client.FeedClient; import ai.vespa.feed.client.FeedClientBuilder; import ai.vespa.feed.client.JsonFeeder; import ai.vespa.feed.client.OperationParseException; -import ai.vespa.feed.client.OperationParameters; -import ai.vespa.feed.client.OperationStats; -import ai.vespa.feed.client.Result; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; import com.yahoo.vespa.http.client.config.FeedParams; @@ -21,7 +16,6 @@ import java.net.URI; import java.time.Duration; import java.util.Arrays; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Level; import java.util.logging.Logger; @@ -132,22 +126,19 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { } private FeedClient createFeedClient() { - if (config.dryrun()) { - return new DryrunClient(); - } else { - List<URI> endpoints = endpointUris(config); - log.info("Using endpoints " + endpoints); - int streamsPerConnection = streamsPerConnection(config); - log.log(Level.INFO, "Using {0} max streams per connection", new Object[] {streamsPerConnection}); - log.log(Level.INFO, "Using {0} connections", new Object[] {config.numConnections()}); - FeedClientBuilder feedClientBuilder = FeedClientBuilder.create(endpoints) - .setConnectionsPerEndpoint(config.numConnections()) - .setMaxStreamPerConnection(streamsPerConnection) - .setRetryStrategy(retryStrategy(config)); - - onFeedClientInitialization(feedClientBuilder); - return feedClientBuilder.build(); - } + List<URI> endpoints = endpointUris(config); + log.info("Using endpoints " + endpoints); + int streamsPerConnection = streamsPerConnection(config); + log.log(Level.INFO, "Using {0} max streams per connection", new Object[] {streamsPerConnection}); + log.log(Level.INFO, "Using {0} connections", new Object[] {config.numConnections()}); + FeedClientBuilder feedClientBuilder = FeedClientBuilder.create(endpoints) + .setConnectionsPerEndpoint(config.numConnections()) + .setMaxStreamPerConnection(streamsPerConnection) + .setDryrun(config.dryrun()) + .setRetryStrategy(retryStrategy(config)); + + onFeedClientInitialization(feedClientBuilder); + return feedClientBuilder.build(); } private static FeedClient.RetryStrategy retryStrategy(VespaConfiguration config) { @@ -166,29 +157,4 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { .map(hostname -> URI.create(String.format("https://%s:%d/", hostname, config.defaultPort()))) .collect(toList()); } - - private static class DryrunClient implements FeedClient { - - @Override - public CompletableFuture<Result> put(DocumentId documentId, String documentJson, OperationParameters params) { - return createSuccessResult(documentId); - } - - @Override - public CompletableFuture<Result> update(DocumentId documentId, String updateJson, OperationParameters params) { - return createSuccessResult(documentId); - } - - @Override - public CompletableFuture<Result> remove(DocumentId documentId, OperationParameters params) { - return createSuccessResult(documentId); - } - - @Override public OperationStats stats() { return null; } - @Override public void close(boolean graceful) {} - - private static CompletableFuture<Result> createSuccessResult(DocumentId documentId) { - return CompletableFuture.completedFuture(DryrunResult.create(Result.Type.success, documentId, "ok", null)); - } - } } |