aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorValerij Fredriksen <freva@users.noreply.github.com>2022-08-11 16:31:47 +0200
committerGitHub <noreply@github.com>2022-08-11 16:31:47 +0200
commit046cae924701cf6446db73b12720194874daad01 (patch)
tree2f71bc196852527e933f72150ef3be7c236a6c65
parent482ba60da0512eba63ecb766915d333052cb351d (diff)
parent5f46b531be2a88e42e69e94af24330ad68fafa35 (diff)
Merge pull request #23632 from vespa-engine/jonmv/speed-test
Add --speedTest to feed client CLI, and dryRun to /doc/v1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java9
-rw-r--r--vespa-feed-client-api/abi-spec.json1
-rw-r--r--vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java4
-rw-r--r--vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java33
-rw-r--r--vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java79
-rw-r--r--vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java26
-rw-r--r--vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliClientTest.java31
-rw-r--r--vespa-feed-client-cli/src/test/resources/help.txt16
-rw-r--r--vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java7
-rw-r--r--vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java7
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java36
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java95
12 files changed, 283 insertions, 61 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index b9432fdc375..78063a383dc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -550,10 +550,9 @@ public class ApplicationController {
controller.jobController().deploymentStatus(application.get());
for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) {
- if ( ! notification.source().instance().map(declaredInstances::contains).orElse(true))
- controller.notificationsDb().removeNotifications(notification.source());
- if (notification.source().instance().isPresent() &&
- ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false))
+ if ( notification.source().instance().isPresent()
+ && ( ! declaredInstances.contains(notification.source().instance().get())
+ || ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false)))
controller.notificationsDb().removeNotifications(notification.source());
}
@@ -647,7 +646,7 @@ public class ApplicationController {
.filter(zone -> deploymentSpec.instance(instance).isEmpty()
|| ! deploymentSpec.requireInstance(instance).deploysTo(zone.environment(),
zone.region()))
- .collect(toList());
+ .toList();
if (deploymentsToRemove.isEmpty())
return application;
diff --git a/vespa-feed-client-api/abi-spec.json b/vespa-feed-client-api/abi-spec.json
index 5bd0acf82d3..a41932323b5 100644
--- a/vespa-feed-client-api/abi-spec.json
+++ b/vespa-feed-client-api/abi-spec.json
@@ -137,6 +137,7 @@
"public abstract ai.vespa.feed.client.FeedClientBuilder setCertificate(java.util.Collection, java.security.PrivateKey)",
"public abstract ai.vespa.feed.client.FeedClientBuilder setCertificate(java.security.cert.X509Certificate, java.security.PrivateKey)",
"public abstract ai.vespa.feed.client.FeedClientBuilder setDryrun(boolean)",
+ "public abstract ai.vespa.feed.client.FeedClientBuilder setSpeedTest(boolean)",
"public abstract ai.vespa.feed.client.FeedClientBuilder setCaCertificatesFile(java.nio.file.Path)",
"public abstract ai.vespa.feed.client.FeedClientBuilder setCaCertificates(java.util.Collection)",
"public abstract ai.vespa.feed.client.FeedClientBuilder setEndpointUris(java.util.List)",
diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
index 7ec5fbb02b7..b7c51a0c5c9 100644
--- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
+++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
@@ -95,8 +95,12 @@ public interface FeedClientBuilder {
/** Sets client SSL certificate/key */
FeedClientBuilder setCertificate(X509Certificate certificate, PrivateKey privateKey);
+ /** Turns on dryrun mode, where each operation succeeds after a given delay, rather than being sent across the network. */
FeedClientBuilder setDryrun(boolean enabled);
+ /** Turns on speed test mode, where all feed operations are immediately acknowledged by the server. */
+ FeedClientBuilder setSpeedTest(boolean enabled);
+
/**
* Overrides JVM default SSL truststore
* @param caCertificatesFile Path to PEM encoded file containing trusted certificates
diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java
index e024f961e26..fd36749b109 100644
--- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java
+++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java
@@ -39,6 +39,8 @@ class CliArguments {
private static final String CONNECTIONS_OPTION = "connections";
private static final String DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION = "disable-ssl-hostname-verification";
private static final String DRYRUN_OPTION = "dryrun";
+ private static final String SPEED_TEST_OPTION = "speed-test";
+ private static final String TEST_PAYLOAD_SIZE_OPTION = "test-payload-size";
private static final String ENDPOINT_OPTION = "endpoint";
private static final String FILE_OPTION = "file";
private static final String HEADER_OPTION = "header";
@@ -78,8 +80,19 @@ class CliArguments {
if (!args.hasOption(ENDPOINT_OPTION)) {
throw new CliArgumentsException("Endpoint must be specified");
}
- if (args.hasOption(FILE_OPTION) == args.hasOption(STDIN_OPTION)) {
- throw new CliArgumentsException(String.format("Either option '%s' or '%s' must be specified", FILE_OPTION, STDIN_OPTION));
+ if (args.hasOption(SPEED_TEST_OPTION)) {
+ if ( args.hasOption(FILE_OPTION) && (args.hasOption(STDIN_OPTION) || args.hasOption(TEST_PAYLOAD_SIZE_OPTION))
+ || args.hasOption(STDIN_OPTION) && args.hasOption(TEST_PAYLOAD_SIZE_OPTION)) {
+ throw new CliArgumentsException(String.format("At most one of '%s', '%s' and '%s' may be specified", FILE_OPTION, STDIN_OPTION, TEST_PAYLOAD_SIZE_OPTION));
+ }
+ }
+ else {
+ if (args.hasOption(FILE_OPTION) == args.hasOption(STDIN_OPTION)) {
+ throw new CliArgumentsException(String.format("Exactly one of '%s' and '%s' must be specified", FILE_OPTION, STDIN_OPTION));
+ }
+ if (args.hasOption(TEST_PAYLOAD_SIZE_OPTION)) {
+ throw new CliArgumentsException(String.format("Option '%s' can only be specified together with '%s'", TEST_PAYLOAD_SIZE_OPTION, SPEED_TEST_OPTION));
+ }
}
if (args.hasOption(CERTIFICATE_OPTION) != args.hasOption(PRIVATE_KEY_OPTION)) {
throw new CliArgumentsException(
@@ -166,6 +179,10 @@ class CliArguments {
boolean dryrunEnabled() { return has(DRYRUN_OPTION); }
+ boolean speedTest() { return has(SPEED_TEST_OPTION); }
+
+ OptionalInt testPayloadSize() throws CliArgumentsException { return intValue(TEST_PAYLOAD_SIZE_OPTION); }
+
Optional<URI> proxy() throws CliArgumentsException {
try {
URL url = (URL) arguments.getParsedOptionValue(PROXY_OPTION);
@@ -304,7 +321,17 @@ class CliArguments {
.build())
.addOption(Option.builder()
.longOpt(DRYRUN_OPTION)
- .desc("Enable dryrun mode where each operation succeeds after " + DryrunCluster.DELAY.toMillis() + "ms")
+ .desc("Let each operation succeed after " + DryrunCluster.DELAY.toMillis() + "ms, instead of sending it across the network ")
+ .build())
+ .addOption(Option.builder()
+ .longOpt(SPEED_TEST_OPTION)
+ .desc("Perform a network speed test, where the server immediately responds to each feed operation with a successful response. Requires Vespa version ≥ 8.35 on the server")
+ .build())
+ .addOption(Option.builder()
+ .longOpt(TEST_PAYLOAD_SIZE_OPTION)
+ .desc("Document JSON test payload size in bytes, for use with --speed-test; requires --file and -stdin to not be set; default is 1024")
+ .hasArg()
+ .type(Number.class)
.build())
.addOption(Option.builder()
.longOpt(VERBOSE_OPTION)
diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java
index 68b9cf6af0e..e4fe07bedb2 100644
--- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java
+++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package ai.vespa.feed.client.impl;
+import ai.vespa.feed.client.DocumentId;
import ai.vespa.feed.client.FeedClient;
import ai.vespa.feed.client.FeedClientBuilder;
import ai.vespa.feed.client.FeedException;
@@ -9,22 +10,35 @@ import ai.vespa.feed.client.JsonFeeder.ResultCallback;
import ai.vespa.feed.client.OperationStats;
import ai.vespa.feed.client.Result;
import ai.vespa.feed.client.ResultException;
+import ai.vespa.feed.client.impl.CliArguments.CliArgumentsException;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLSession;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.io.SequenceInputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.Duration;
+import java.time.Instant;
+import java.util.Enumeration;
import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.util.stream.Collectors.joining;
/**
* Main method for CLI interface
@@ -76,7 +90,7 @@ public class CliClient {
if (cliArgs.showProgress()) {
Thread progressPrinter = new Thread(() -> {
try {
- while ( ! latch.await(10, TimeUnit.SECONDS)) {
+ while (!latch.await(10, TimeUnit.SECONDS)) {
synchronized (printMonitor) {
printBenchmarkResult(System.nanoTime() - startNanos, successes.get(), failures.get(), feedClient.stats(), systemError);
}
@@ -89,9 +103,17 @@ public class CliClient {
}
feeder.feedMany(in, new ResultCallback() {
- @Override public void onNextResult(Result result, FeedException error) { handleResult(result, error, successes, failures, cliArgs); }
- @Override public void onError(FeedException error) { fatal.set(error); latch.countDown(); }
- @Override public void onComplete() { latch.countDown(); }
+ @Override
+ public void onNextResult(Result result, FeedException error) { handleResult(result, error, successes, failures, cliArgs); }
+
+ @Override
+ public void onError(FeedException error) {
+ fatal.set(error);
+ latch.countDown();
+ }
+
+ @Override
+ public void onComplete() { latch.countDown(); }
});
latch.await();
@@ -99,9 +121,11 @@ public class CliClient {
if (fatal.get() != null) throw fatal.get();
}
return 0;
- } catch (CliArguments.CliArgumentsException | IOException | FeedException e) {
+ }
+ catch (CliArguments.CliArgumentsException | IOException | FeedException e) {
return handleException(verbose, e);
- } catch (Exception e) {
+ }
+ catch (Exception e) {
return handleException(verbose, "Unknown failure: " + e.getMessage(), e);
}
}
@@ -111,7 +135,8 @@ public class CliClient {
failures.incrementAndGet();
if (args.showErrors()) synchronized (printMonitor) {
systemError.println(error.getMessage());
- if (error instanceof ResultException) ((ResultException) error).getTrace().ifPresent(systemError::println);
+ if (error instanceof ResultException)
+ ((ResultException) error).getTrace().ifPresent(systemError::println);
if (args.verboseSpecified()) error.printStackTrace(systemError);
}
}
@@ -136,6 +161,7 @@ public class CliClient {
cliArgs.caCertificates().ifPresent(builder::setCaCertificatesFile);
cliArgs.headers().forEach(builder::addRequestHeader);
builder.setDryrun(cliArgs.dryrunEnabled());
+ builder.setSpeedTest(cliArgs.speedTest());
cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10),
Duration.ofSeconds(doom))));
cliArgs.proxy().ifPresent(builder::setProxy);
@@ -151,7 +177,9 @@ public class CliClient {
}
private InputStream createFeedInputStream(CliArguments cliArgs) throws CliArguments.CliArgumentsException, IOException {
- return cliArgs.readFeedFromStandardInput() ? systemIn : Files.newInputStream(cliArgs.inputFile().get());
+ return cliArgs.readFeedFromStandardInput() ? systemIn
+ : cliArgs.inputFile().isPresent() ? Files.newInputStream(cliArgs.inputFile().get())
+ : createDummyInputStream(cliArgs.testPayloadSize().orElse(1024));
}
private int handleException(boolean verbose, Exception e) { return handleException(verbose, e.getMessage(), e); }
@@ -165,8 +193,12 @@ public class CliClient {
}
private static class AcceptAllHostnameVerifier implements HostnameVerifier {
+
static final AcceptAllHostnameVerifier INSTANCE = new AcceptAllHostnameVerifier();
- @Override public boolean verify(String hostname, SSLSession session) { return true; }
+
+ @Override
+ public boolean verify(String hostname, SSLSession session) { return true; }
+
}
static void printBenchmarkResult(long durationNanos, long successes, long failures,
@@ -206,4 +238,31 @@ public class CliClient {
generator.writeNumber(String.format("%." + precision + "f", value));
}
-}
+ /** Creates an input stream that spits out random documents (id and data) for one minute. */
+ static InputStream createDummyInputStream(int payloadSize) {
+ Instant end = Instant.now().plusSeconds(60);
+ return createDummyInputStream(payloadSize, new Random(), () -> Instant.now().isBefore(end));
+ }
+
+ static InputStream createDummyInputStream(int payloadSize, Random random, BooleanSupplier hasNext) {
+ int idSize = 8;
+ String template = String.format("{ \"put\": \"id:test:test::%s\", \"fields\": { \"test\": \"%s\" } }\n",
+ IntStream.range(0, idSize).mapToObj(__ -> "*").collect(joining()),
+ IntStream.range(0, payloadSize).mapToObj(__ -> "#").collect(joining()));
+ byte[] buffer = template.getBytes(StandardCharsets.UTF_8);
+ int idIndex = template.indexOf('*');
+ int dataIndex = template.indexOf('#');
+
+ return new SequenceInputStream(new Enumeration<InputStream>() {
+ @Override public boolean hasMoreElements() {
+ return hasNext.getAsBoolean();
+ }
+ @Override public InputStream nextElement() {
+ for (int i = 0; i < idSize; i++) buffer[ idIndex + i] = (byte) ('a' + (random.nextInt(26)));
+ for (int i = 0; i < payloadSize; i++) buffer[dataIndex + i] = (byte) ('a' + (random.nextInt(26)));
+ return new ByteArrayInputStream(buffer);
+ }
+ });
+ }
+
+} \ No newline at end of file
diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java
index 201a85ed09d..073ea4a58db 100644
--- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java
+++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package ai.vespa.feed.client.impl;
+import ai.vespa.feed.client.impl.CliArguments.CliArgumentsException;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayOutputStream;
@@ -63,16 +64,23 @@ class CliArgumentsTest {
}
@Test
- void fails_on_conflicting_parameters() {
- CliArguments.CliArgumentsException exception = assertThrows(
- CliArguments.CliArgumentsException.class,
- () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint", "--file=/path/to/file", "--stdin"}));
- assertEquals("Either option 'file' or 'stdin' must be specified", exception.getMessage());
+ void fails_on_conflicting_parameters() throws CliArgumentsException {
+ assertEquals("Exactly one of 'file' and 'stdin' must be specified",
+ assertThrows(CliArgumentsException.class,
+ () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint", "--file=/path/to/file", "--stdin"}))
+ .getMessage());
- exception = assertThrows(
- CliArguments.CliArgumentsException.class,
- () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint"}));
- assertEquals("Either option 'file' or 'stdin' must be specified", exception.getMessage());
+ assertEquals("Exactly one of 'file' and 'stdin' must be specified",
+ assertThrows(CliArgumentsException.class,
+ () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint"}))
+ .getMessage());
+
+ assertEquals("At most one of 'file', 'stdin' and 'test-payload-size' may be specified",
+ assertThrows(CliArgumentsException.class,
+ () -> CliArguments.fromRawArgs(new String[] {"--endpoint=https://endpoint", "--speed-test", "--test-payload-size=123", "--file=file"}))
+ .getMessage());
+
+ CliArguments.fromRawArgs(new String[] {"--endpoint=foo", "--speed-test"});
}
@Test
diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliClientTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliClientTest.java
new file mode 100644
index 00000000000..44dbf62966e
--- /dev/null
+++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliClientTest.java
@@ -0,0 +1,31 @@
+package ai.vespa.feed.client.impl;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author jonmv
+ */
+class CliClientTest {
+
+ @Test
+ void testDummyStream() throws IOException {
+ AtomicInteger count = new AtomicInteger(3);
+ InputStream in = CliClient.createDummyInputStream(4, new Random(0), () -> count.decrementAndGet() >= 0);
+ byte[] buffer = new byte[1 << 20];
+ int offset = 0, read;
+ while ((read = in.read(buffer, offset, buffer.length - offset)) >= 0) offset += read;
+ assertEquals("{ \"put\": \"id:test:test::ssxvnjhp\", \"fields\": { \"test\": \"dqdx\" } }\n" +
+ "{ \"put\": \"id:test:test::vcrastvy\", \"fields\": { \"test\": \"bcwv\" } }\n" +
+ "{ \"put\": \"id:test:test::mgnykrxv\", \"fields\": { \"test\": \"zxkg\" } }\n",
+ new String(buffer, 0, offset, StandardCharsets.UTF_8));
+ }
+
+}
diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt
index 66d7c3521c2..e41a78bc932 100644
--- a/vespa-feed-client-cli/src/test/resources/help.txt
+++ b/vespa-feed-client-cli/src/test/resources/help.txt
@@ -10,8 +10,9 @@ Vespa feed client
connections
--disable-ssl-hostname-verification Disable SSL hostname
verification
- --dryrun Enable dryrun mode where each
- operation succeeds after 1ms
+ --dryrun Let each operation succeed after
+ 1ms, instead of sending it
+ across the network
--endpoint <arg> URI to feed endpoint
--file <arg> Path to feed file in JSON format
--header <arg> HTTP header on the form 'Name:
@@ -34,8 +35,19 @@ Vespa feed client
failure
--silent Disable periodic status printing
to stderr
+ --speed-test Perform a network speed test,
+ where the server immediately
+ responds to each feed operation
+ with a successful response.
+ Requires Vespa version ≥ 8.35 on
+ the server
--stdin Read JSON input from standard
input
+ --test-payload-size <arg> Document JSON test payload size
+ in bytes, for use with
+ --speed-test; requires --file
+ and -stdin to not be set;
+ default is 1024
--timeout <arg> Feed operation timeout (in
seconds)
--trace <arg> The trace level of network
diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java
index 134ad464618..c70cb7cd850 100644
--- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java
+++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java
@@ -49,6 +49,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder {
Collection<X509Certificate> caCertificates;
boolean benchmark = true;
boolean dryrun = false;
+ boolean speedTest = false;
URI proxy;
@@ -170,6 +171,12 @@ public class FeedClientBuilderImpl implements FeedClientBuilder {
return this;
}
+ @Override
+ public FeedClientBuilder setSpeedTest(boolean enabled) {
+ this.speedTest = enabled;
+ return this;
+ }
+
/**
* Overrides JVM default SSL truststore
* @param caCertificatesFile Path to PEM encoded file containing trusted certificates
diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java
index c136d697a0b..80a107fbdc6 100644
--- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java
+++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java
@@ -41,6 +41,7 @@ class HttpFeedClient implements FeedClient {
private final Map<String, Supplier<String>> requestHeaders;
private final RequestStrategy requestStrategy;
private final AtomicBoolean closed = new AtomicBoolean();
+ private final boolean speedTest;
HttpFeedClient(FeedClientBuilderImpl builder) throws IOException {
this(builder, new HttpRequestStrategy(builder));
@@ -49,6 +50,7 @@ class HttpFeedClient implements FeedClient {
HttpFeedClient(FeedClientBuilderImpl builder, RequestStrategy requestStrategy) {
this.requestHeaders = new HashMap<>(builder.requestHeaders);
this.requestStrategy = requestStrategy;
+ this.speedTest = builder.speedTest;
}
@Override
@@ -90,7 +92,7 @@ class HttpFeedClient implements FeedClient {
throw new IllegalStateException("Client is closed");
HttpRequest request = new HttpRequest(method,
- getPath(documentId) + getQuery(params),
+ getPath(documentId) + getQuery(params, speedTest),
requestHeaders,
operationJson == null ? null : operationJson.getBytes(UTF_8), // TODO: make it bytes all the way?
params.timeout().orElse(null));
@@ -217,13 +219,14 @@ class HttpFeedClient implements FeedClient {
}
}
- static String getQuery(OperationParameters params) {
+ static String getQuery(OperationParameters params, boolean speedTest) {
StringJoiner query = new StringJoiner("&", "?", "").setEmptyValue("");
if (params.createIfNonExistent()) query.add("create=true");
params.testAndSetCondition().ifPresent(condition -> query.add("condition=" + encode(condition)));
params.timeout().ifPresent(timeout -> query.add("timeout=" + timeout.toMillis() + "ms"));
params.route().ifPresent(route -> query.add("route=" + encode(route)));
params.tracelevel().ifPresent(tracelevel -> query.add("tracelevel=" + tracelevel));
+ if (speedTest) query.add("dryRun=true");
return query.toString();
}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
index 7f940e5695a..c72bc1ef4c5 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
@@ -20,7 +20,6 @@ import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.FixedBucketSpaces;
import com.yahoo.document.TestAndSetCondition;
import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.document.fieldset.AllFields;
import com.yahoo.document.fieldset.DocIdOnly;
import com.yahoo.document.fieldset.DocumentOnly;
import com.yahoo.document.idstring.IdIdString;
@@ -169,6 +168,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
private static final String STREAM = "stream";
private static final String SLICES = "slices";
private static final String SLICE_ID = "sliceId";
+ private static final String DRY_RUN = "dryRun";
private final Clock clock;
private final Duration handlerTimeout;
@@ -362,6 +362,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
}
private ContentChannel getDocuments(HttpRequest request, DocumentPath path, ResponseHandler handler) {
+ disallow(request, DRY_RUN);
enqueueAndDispatch(request, handler, () -> {
boolean streamed = getProperty(request, STREAM, booleanParser).orElse(false);
VisitorParameters parameters = parseGetParameters(request, path, streamed);
@@ -374,6 +375,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
}
private ContentChannel postDocuments(HttpRequest request, DocumentPath path, ResponseHandler handler) {
+ disallow(request, DRY_RUN);
enqueueAndDispatch(request, handler, () -> {
StorageCluster destination = resolveCluster(Optional.of(requireProperty(request, DESTINATION_CLUSTER)), clusters);
VisitorParameters parameters = parseParameters(request, path);
@@ -388,6 +390,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
}
private ContentChannel putDocuments(HttpRequest request, DocumentPath path, ResponseHandler handler) {
+ disallow(request, DRY_RUN);
return new ForwardingContentChannel(in -> {
enqueueAndDispatch(request, handler, () -> {
StorageCluster cluster = resolveCluster(Optional.of(requireProperty(request, CLUSTER)), clusters);
@@ -406,6 +409,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
}
private ContentChannel deleteDocuments(HttpRequest request, DocumentPath path, ResponseHandler handler) {
+ disallow(request, DRY_RUN);
enqueueAndDispatch(request, handler, () -> {
VisitorParameters parameters = parseParameters(request, path);
parameters.setFieldSet(DocIdOnly.NAME);
@@ -420,6 +424,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
}
private ContentChannel getDocument(HttpRequest request, DocumentPath path, ResponseHandler handler) {
+ disallow(request, DRY_RUN);
enqueueAndDispatch(request, handler, () -> {
DocumentOperationParameters rawParameters = parametersFromRequest(request, CLUSTER, FIELD_SET);
if (rawParameters.fieldSet().isEmpty())
@@ -442,6 +447,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
private ContentChannel postDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) {
ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.PUT, clock.instant());
+ if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) {
+ handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1));
+ return ignoredContent;
+ }
+
return new ForwardingContentChannel(in -> {
enqueueAndDispatch(request, handler, () -> {
DocumentPut put = parser.parsePut(in, path.id().toString());
@@ -459,6 +469,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
private ContentChannel putDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) {
ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.UPDATE, clock.instant());
+ if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) {
+ handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1));
+ return ignoredContent;
+ }
+
return new ForwardingContentChannel(in -> {
enqueueAndDispatch(request, handler, () -> {
DocumentUpdate update = parser.parseUpdate(in, path.id().toString());
@@ -477,6 +492,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
private ContentChannel deleteDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) {
ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.REMOVE, clock.instant());
+ if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) {
+ handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1));
+ return ignoredContent;
+ }
+
enqueueAndDispatch(request, handler, () -> {
DocumentRemove remove = new DocumentRemove(path.id());
getProperty(request, CONDITION).map(TestAndSetCondition::new).ifPresent(remove::setCondition);
@@ -1062,19 +1082,19 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
private void updatePutMetrics(Outcome outcome) {
switch (outcome) {
- case SUCCESS: metric.add(MetricNames.SUCCEEDED, 1, null); break;
- case CONDITION_FAILED: metric.add(MetricNames.CONDITION_NOT_MET, 1, null); break;
- default: metric.add(MetricNames.FAILED, 1, null); break;
+ case SUCCESS -> metric.add(MetricNames.SUCCEEDED, 1, null);
+ case CONDITION_FAILED -> metric.add(MetricNames.CONDITION_NOT_MET, 1, null);
+ default -> metric.add(MetricNames.FAILED, 1, null);
}
}
private void updateUpdateMetrics(Outcome outcome, boolean create) {
if (create && outcome == Outcome.NOT_FOUND) outcome = Outcome.SUCCESS; // >_<
switch (outcome) {
- case SUCCESS: metric.add(MetricNames.SUCCEEDED, 1, null); break;
- case NOT_FOUND: metric.add(MetricNames.NOT_FOUND, 1, null); break;
- case CONDITION_FAILED: metric.add(MetricNames.CONDITION_NOT_MET, 1, null); break;
- default: metric.add(MetricNames.FAILED, 1, null); break;
+ case SUCCESS -> metric.add(MetricNames.SUCCEEDED, 1, null);
+ case NOT_FOUND -> metric.add(MetricNames.NOT_FOUND, 1, null);
+ case CONDITION_FAILED -> metric.add(MetricNames.CONDITION_NOT_MET, 1, null);
+ default -> metric.add(MetricNames.FAILED, 1, null);
}
}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
index 74a86b6a7b7..7f77ce9d0d5 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
@@ -524,6 +524,79 @@ public class DocumentV1ApiTest {
"}", response.readAll());
assertEquals(404, response.getStatus());
+ // GET with dryRun=true is an error
+ access.session.expect((__, ___) -> {
+ fail("Should not cause an actual feed operation");
+ return null;
+ });
+ response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?dryRun=true");
+ assertSameJson("{" +
+ " \"pathId\": \"/document/v1/space/music/number/1/two\"," +
+ " \"message\": \"May not specify 'dryRun' at '/document/v1/space/music/number/1/two'\"\n" +
+ "}", response.readAll());
+ assertEquals(400, response.getStatus());
+
+ // POST with dryRun=true returns an immediate OK response
+ access.session.expect((__, ___) -> {
+ fail("Should not cause an actual feed operation");
+ return null;
+ });
+ response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?dryRun=true", POST,
+ "NOT JSON, NOT PARSED");
+ assertSameJson("{" +
+ " \"pathId\": \"/document/v1/space/music/number/1/two\"," +
+ " \"id\": \"id:space:music:n=1:two\"" +
+ "}", response.readAll());
+ assertEquals(200, response.getStatus());
+
+ // PUT with dryRun=true returns an immediate OK response
+ access.session.expect((__, ___) -> {
+ fail("Should not cause an actual feed operation");
+ return null;
+ });
+ response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?dryRun=true", PUT,
+ "NOT JSON, NOT PARSED");
+ assertSameJson("{" +
+ " \"pathId\": \"/document/v1/space/music/number/1/two\"," +
+ " \"id\": \"id:space:music:n=1:two\"" +
+ "}", response.readAll());
+ assertEquals(200, response.getStatus());
+
+ // DELETE with dryRun=true returns an immediate OK response
+ access.session.expect((__, ___) -> {
+ fail("Should not cause an actual feed operation");
+ return null;
+ });
+ response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?dryRun=true", DELETE,
+ "NOT JSON, NOT PARSED");
+ assertSameJson("{" +
+ " \"pathId\": \"/document/v1/space/music/number/1/two\"," +
+ " \"id\": \"id:space:music:n=1:two\"" +
+ "}", response.readAll());
+ assertEquals(200, response.getStatus());
+
+ // PUT with a document update payload is a document update operation.
+ access.session.expect((update, parameters) -> {
+ DocumentUpdate expectedUpdate = new DocumentUpdate(doc3.getDataType(), doc3.getId());
+ expectedUpdate.addFieldUpdate(FieldUpdate.createAssign(doc3.getField("artist"), new StringFieldValue("Lisa Ekdahl")));
+ expectedUpdate.setCreateIfNonExistent(true);
+ assertEquals(expectedUpdate, update);
+ assertEquals(parameters(), parameters);
+ parameters.responseHandler().get().handleResponse(new UpdateResponse(0, true));
+ return new Result();
+ });
+ response = driver.sendRequest("http://localhost/document/v1/space/music/group/a/three?create=true&timeout=1e1s&dryRun=false", PUT,
+ "{" +
+ " \"fields\": {" +
+ " \"artist\": { \"assign\": \"Lisa Ekdahl\" }" +
+ " }" +
+ "}");
+ assertSameJson("{" +
+ " \"pathId\": \"/document/v1/space/music/group/a/three\"," +
+ " \"id\": \"id:space:music:g=a:three\"" +
+ "}", response.readAll());
+ assertEquals(200, response.getStatus());
+
// POST with a document payload is a document put operation.
access.session.expect((put, parameters) -> {
DocumentPut expectedPut = new DocumentPut(doc2);
@@ -566,28 +639,6 @@ public class DocumentV1ApiTest {
"}", response.readAll());
assertEquals(200, response.getStatus());
- // PUT with a document update payload is a document update operation.
- access.session.expect((update, parameters) -> {
- DocumentUpdate expectedUpdate = new DocumentUpdate(doc3.getDataType(), doc3.getId());
- expectedUpdate.addFieldUpdate(FieldUpdate.createAssign(doc3.getField("artist"), new StringFieldValue("Lisa Ekdahl")));
- expectedUpdate.setCreateIfNonExistent(true);
- assertEquals(expectedUpdate, update);
- assertEquals(parameters(), parameters);
- parameters.responseHandler().get().handleResponse(new UpdateResponse(0, true));
- return new Result();
- });
- response = driver.sendRequest("http://localhost/document/v1/space/music/group/a/three?create=true&timeout=1e1s", PUT,
- "{" +
- " \"fields\": {" +
- " \"artist\": { \"assign\": \"Lisa Ekdahl\" }" +
- " }" +
- "}");
- assertSameJson("{" +
- " \"pathId\": \"/document/v1/space/music/group/a/three\"," +
- " \"id\": \"id:space:music:g=a:three\"" +
- "}", response.readAll());
- assertEquals(200, response.getStatus());
-
// POST with no payload is a 400
access.session.expect((__, ___) -> { throw new AssertionError("Not supposed to happen"); });
response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?condition=test%20it", POST, "");