summaryrefslogtreecommitdiffstats
path: root/vespaclient-container-plugin
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespaclient-container-plugin
Publish
Diffstat (limited to 'vespaclient-container-plugin')
-rw-r--r--vespaclient-container-plugin/.gitignore2
-rw-r--r--vespaclient-container-plugin/OWNERS1
-rw-r--r--vespaclient-container-plugin/pom.xml109
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java69
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java39
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java260
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java48
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java21
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java128
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java229
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java5
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java60
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java33
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java26
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java77
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java78
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java71
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java29
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java37
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java100
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java43
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java21
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java116
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java469
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java37
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java194
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java5
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java326
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java40
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java98
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java24
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java43
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java354
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java172
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java43
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java61
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java88
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java542
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java75
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java27
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java25
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java86
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java14
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java7
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java81
-rw-r--r--vespaclient-container-plugin/src/test/application/services.xml17
-rw-r--r--vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg113
-rw-r--r--vespaclient-container-plugin/src/test/files/feedhandler/test10.xml91
-rw-r--r--vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml53
-rw-r--r--vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml48
-rwxr-xr-xvespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml43
-rwxr-xr-xvespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml44
-rwxr-xr-xvespaclient-container-plugin/src/test/files/feedhandler/test_removes2
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java75
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java109
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json0
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java59
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java54
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java298
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java36
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore0
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java103
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java1015
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java134
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java103
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java129
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java98
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java1090
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java25
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java248
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java31
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java70
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java39
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java69
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java35
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java236
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java535
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java224
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java162
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java160
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java104
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java91
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java28
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java75
84 files changed, 10259 insertions, 0 deletions
diff --git a/vespaclient-container-plugin/.gitignore b/vespaclient-container-plugin/.gitignore
new file mode 100644
index 00000000000..016c6f704f0
--- /dev/null
+++ b/vespaclient-container-plugin/.gitignore
@@ -0,0 +1,2 @@
+pom.xml.build
+/target
diff --git a/vespaclient-container-plugin/OWNERS b/vespaclient-container-plugin/OWNERS
new file mode 100644
index 00000000000..0e39145d8c3
--- /dev/null
+++ b/vespaclient-container-plugin/OWNERS
@@ -0,0 +1 @@
+dybdahl
diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml
new file mode 100644
index 00000000000..017216c3cbd
--- /dev/null
+++ b/vespaclient-container-plugin/pom.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>vespaclient-container-plugin</artifactId>
+ <version>6-SNAPSHOT</version>
+ <packaging>container-plugin</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespaclient-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-dev</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-provisioning</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespa-http-client</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:rawtypes</arg>
+ <arg>-Xlint:deprecation</arg>
+ <arg>-Xlint:unchecked</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java
new file mode 100644
index 00000000000..4dc47f20889
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.documentapi.DumpVisitorDataHandler;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Handling data from visit.
+ *
+ * @author dybdahl
+ */
+class LocalDataVisitorHandler extends DumpVisitorDataHandler {
+
+ StringBuilder commaSeparatedJsonDocuments = new StringBuilder();
+ final StringBuilder errors = new StringBuilder();
+
+ private boolean isFirst = true;
+ private final Object monitor = new Object();
+
+ String getErrors() {
+ return errors.toString();
+ }
+
+ String getCommaSeparatedJsonDocuments() {
+ return commaSeparatedJsonDocuments.toString();
+ }
+
+ @Override
+ public void onDocument(Document document, long l) {
+ try {
+ final String docJson = new String(JsonWriter.toByteArray(document), StandardCharsets.UTF_8.name());
+ synchronized (monitor) {
+ if (!isFirst) {
+ commaSeparatedJsonDocuments.append(",");
+ }
+ isFirst = false;
+ commaSeparatedJsonDocuments.append(docJson);
+ }
+ } catch (Exception e) {
+ synchronized (monitor) {
+ errors.append(ExceptionUtils.getStackTrace(e)).append("\n");
+ }
+ }
+ }
+
+ // TODO: Not sure if we should support removeal or not. Do nothing here maybe?
+ @Override
+ public void onRemove(DocumentId documentId) {
+ try {
+ final String removeJson = new String(JsonWriter.documentRemove(documentId), StandardCharsets.UTF_8.name());
+ synchronized (monitor) {
+ if (!isFirst) {
+ commaSeparatedJsonDocuments.append(",");
+ }
+ isFirst = false;
+ commaSeparatedJsonDocuments.append(removeJson);
+ }
+ } catch (Exception e) {
+ synchronized (monitor) {
+ errors.append(ExceptionUtils.getStackTrace(e)).append("\n");
+ }
+ }
+ }
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java
new file mode 100644
index 00000000000..240cc3c3c61
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.util.Optional;
+
+/**
+ * Abstract the backend stuff for the REST API, such as retrieving or updating documents.
+ *
+ * @author dybdahl
+ */
+public interface OperationHandler {
+
+ class VisitResult{
+
+ public final Optional<String> token;
+ public final String documentsAsJsonList;
+
+ public VisitResult(Optional<String> token, String documentsAsJsonList) {
+ this.token = token;
+ this.documentsAsJsonList = documentsAsJsonList;
+ }
+ }
+
+ VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException;
+
+ void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException;
+
+ void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException;
+
+ void delete(RestUri restUri, String condition) throws RestApiException;
+
+ Optional<String> get(RestUri restUri) throws RestApiException;
+
+ /** Called just before this is disposed of */
+ default void shutdown() {}
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java
new file mode 100644
index 00000000000..21fd930e9d5
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java
@@ -0,0 +1,260 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.TestAndSetCondition;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.documentapi.DocumentAccess;
+import com.yahoo.documentapi.DocumentAccessException;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.SyncSession;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorParameters;
+import com.yahoo.documentapi.VisitorSession;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.storage.searcher.ContinuationHit;
+import com.yahoo.vdslib.VisitorOrdering;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Sends operations to messagebus via document api.
+ *
+ * @author dybdahl
+ */
+public class OperationHandlerImpl implements OperationHandler {
+
+ public static final int VISIT_TIMEOUT_MS = 120000;
+ private final DocumentAccess documentAccess;
+
+ public OperationHandlerImpl(DocumentAccess documentAccess) {
+ this.documentAccess = documentAccess;
+ }
+
+ @Override
+ public void shutdown() {
+ documentAccess.shutdown();
+ }
+
+ private static final int HTTP_STATUS_BAD_REQUEST = 400;
+ private static final int HTTP_STATUS_INSUFFICIENT_STORAGE = 507;
+
+ private static int getHTTPStatusCode(Set<Integer> errorCodes) {
+ if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) {
+ return HTTP_STATUS_INSUFFICIENT_STORAGE;
+ }
+ return HTTP_STATUS_BAD_REQUEST;
+ }
+
+ private static Response createErrorResponse(DocumentAccessException documentException, RestUri restUri) {
+ return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), documentException.getMessage(), restUri);
+ }
+
+ @Override
+ public VisitResult visit(
+ RestUri restUri,
+ String documentSelection,
+ Optional<String> cluster,
+ Optional<String> continuation) throws RestApiException {
+
+ VisitorParameters visitorParameters = createVisitorParameters(restUri, documentSelection, cluster, continuation);
+
+ VisitorControlHandler visitorControlHandler = new VisitorControlHandler();
+ visitorParameters.setControlHandler(visitorControlHandler);
+ LocalDataVisitorHandler localDataVisitorHandler = new LocalDataVisitorHandler();
+ visitorParameters.setLocalDataHandler(localDataVisitorHandler);
+
+ final VisitorSession visitorSession;
+ try {
+ visitorSession = documentAccess.createVisitorSession(visitorParameters);
+ // Not sure if this line is required
+ visitorControlHandler.setSession(visitorSession);
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(
+ 500,
+ "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTrace(e),
+ restUri));
+ }
+ try {
+ if (! visitorControlHandler.waitUntilDone(VISIT_TIMEOUT_MS)) {
+ throw new RestApiException(Response.createErrorResponse(500, "Timed out", restUri));
+ }
+ if (visitorControlHandler.getResult().code != VisitorControlHandler.CompletionCode.SUCCESS) {
+ throw new RestApiException(Response.createErrorResponse(400, visitorControlHandler.getResult().toString()));
+ }
+ } catch (InterruptedException e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ if (localDataVisitorHandler.getErrors().isEmpty()) {
+ final Optional<String> continuationToken;
+ if (! visitorControlHandler.getProgress().isFinished()) {
+ final ContinuationHit continuationHit = new ContinuationHit(visitorControlHandler.getProgress());
+ continuationToken = Optional.of(continuationHit.getValue());
+ } else {
+ continuationToken = Optional.empty();
+ }
+ return new VisitResult(continuationToken, localDataVisitorHandler.getCommaSeparatedJsonDocuments());
+ }
+ throw new RestApiException(Response.createErrorResponse(500, localDataVisitorHandler.getErrors(), restUri));
+ }
+
+ @Override
+ public void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException {
+ try {
+ SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters());
+ DocumentPut put = new DocumentPut(data.getDocument());
+ put.setCondition(data.getCondition());
+ syncSession.put(put);
+ } catch (DocumentAccessException documentException) {
+ throw new RestApiException(createErrorResponse(documentException, restUri));
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ }
+
+ @Override
+ public void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException {
+ try {
+ SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters());
+ syncSession.update(data.getDocumentUpdate());
+ } catch (DocumentAccessException documentException) {
+ throw new RestApiException(createErrorResponse(documentException, restUri));
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ }
+
+ @Override
+ public void delete(RestUri restUri, String condition) throws RestApiException {
+ try {
+ DocumentId id = new DocumentId(restUri.generateFullId());
+ SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters());
+ DocumentRemove documentRemove = new DocumentRemove(id);
+ if (condition != null && ! condition.isEmpty()) {
+ documentRemove.setCondition(new TestAndSetCondition(condition));
+ }
+ syncSession.remove(documentRemove);
+ } catch (DocumentAccessException documentException) {
+ throw new RestApiException(Response.createErrorResponse(400, documentException.getMessage(), restUri));
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ }
+
+ @Override
+ public Optional<String> get(RestUri restUri) throws RestApiException {
+ try {
+ DocumentId id = new DocumentId(restUri.generateFullId());
+ SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters());
+ final Document document = syncSession.get(id);
+ if (document == null) {
+ return Optional.empty();
+ }
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ JsonWriter jsonWriter = new JsonWriter(outputStream);
+ jsonWriter.write(document);
+ return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name()));
+
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ }
+
+ private static String resolveClusterRoute(Optional<String> wantedCluster) throws RestApiException {
+ List<ClusterDef> clusters = new ClusterList("client").getStorageClusters();
+ return resolveClusterRoute(wantedCluster, clusters);
+ }
+
+ // Based on resolveClusterRoute in VdsVisit, protected for testability
+ protected static String resolveClusterRoute(Optional<String> wantedCluster, List<ClusterDef> clusters) throws RestApiException {
+ if (clusters.size() == 0) {
+ throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " +
+ "declared. Visiting feature is not available.");
+ }
+ if (! wantedCluster.isPresent()) {
+ if (clusters.size() != 1) {
+ new RestApiException(Response.createErrorResponse(400, "Several clusters exist: " +
+ clusterListToString(clusters) + " you must specify one.. "));
+ }
+ return clusterDefToRoute(clusters.get(0));
+ }
+
+ for (ClusterDef clusterDef : clusters) {
+ if (clusterDef.getName().equals(wantedCluster.get())) {
+ return clusterDefToRoute(clusterDef);
+ }
+ }
+ throw new RestApiException(Response.createErrorResponse(400, "Your vespa cluster contains the content clusters " +
+ clusterListToString(clusters) + " not " + wantedCluster.get() + ". Please select a valid vespa cluster."));
+
+ }
+
+ private static String clusterDefToRoute(ClusterDef clusterDef) {
+ return "[Storage:cluster=" + clusterDef.getName() + ";clusterconfigid=" + clusterDef.getConfigId() + "]";
+ }
+
+ private static String clusterListToString(List<ClusterDef> clusters) {
+ StringBuilder clusterListString = new StringBuilder();
+ clusters.forEach(x -> clusterListString.append(x.getName()).append(" (").append(x.getConfigId()).append("), "));
+ return clusterListString.toString();
+ }
+
+ private VisitorParameters createVisitorParameters(
+ RestUri restUri,
+ String documentSelection,
+ Optional<String> clusterName,
+ Optional<String> continuation)
+ throws RestApiException {
+
+ StringBuilder selection = new StringBuilder();
+
+ if (! documentSelection.isEmpty()) {
+ selection.append("(").append(documentSelection).append(" and ");
+ }
+ selection.append(restUri.getDocumentType()).append(" and (id.namespace=='").append(restUri.getNamespace()).append("')");
+ if (! documentSelection.isEmpty()) {
+ selection.append(")");
+ }
+
+ VisitorParameters params = new VisitorParameters(selection.toString());
+
+ params.setMaxBucketsPerVisitor(1);
+ params.setMaxPending(32);
+ params.setMaxFirstPassHits(1);
+ params.setMaxTotalHits(10);
+ params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(1));
+ params.setToTimestamp(0L);
+ params.setFromTimestamp(0L);
+
+ params.visitInconsistentBuckets(true);
+ params.setVisitorOrdering(VisitorOrdering.ASCENDING);
+
+ params.setRoute(resolveClusterRoute(clusterName));
+
+ params.setTraceLevel(0);
+ params.setPriority(DocumentProtocol.Priority.NORMAL_4);
+ params.setVisitRemoves(false);
+
+ if (continuation.isPresent()) {
+ try {
+ params.setResumeToken(ContinuationHit.getToken(continuation.get()));
+ } catch (Exception e) {
+ throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri));
+ }
+ }
+ return params;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java
new file mode 100644
index 00000000000..9c846e9ce38
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+public class Response extends HttpResponse {
+
+ private final static ObjectMapper objectMapper = new ObjectMapper();
+ private final String jsonMessage;
+
+ public Response(int code, Optional<ObjectNode> element, Optional<RestUri> restPath) {
+ super(code);
+ ObjectNode objectNode = element.orElse(objectMapper.createObjectNode());
+ if (restPath.isPresent()) {
+ objectNode.put("id", restPath.get().generateFullId());
+ objectNode.put("pathId", restPath.get().getRawPath());
+ }
+ jsonMessage = objectNode.toString();
+ }
+
+ public static Response createErrorResponse(int code, String errorMessage) {
+ ObjectNode objectNode = objectMapper.createObjectNode();
+ objectNode.putArray("errors").add(errorMessage);
+ return new Response(code, Optional.of(objectNode), Optional.<RestUri>empty());
+ }
+
+ public static Response createErrorResponse(int code, String errorMessage, RestUri restUri) {
+ ObjectNode objectNode = objectMapper.createObjectNode();
+ objectNode.putArray("errors").add(errorMessage);
+ return new Response(code, Optional.of(objectNode), Optional.of(restUri));
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ stream.write(jsonMessage.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java
new file mode 100644
index 00000000000..b553d83d848
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+/**
+ * Exceptions for Rest API
+ *
+ * @author dybdahl
+ */
+public class RestApiException extends Exception {
+
+ private final Response response;
+
+ public RestApiException(Response response) {
+ this.response = response;
+ }
+
+ public Response getResponse() {
+ return response;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java
new file mode 100644
index 00000000000..98293508168
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+import static com.yahoo.jdisc.Response.Status.*;
+
+/**
+ * Represents the request URI with its values.
+ *
+ * @author dybdahl
+ */
+public class RestUri {
+
+ public static final char NUMBER_STREAMING = 'n';
+ public static final char GROUP_STREAMING = 'g';
+ public static final String DOCUMENT = "document";
+ public static final String V_1 = "v1";
+ public static final String ID = "id:";
+
+ /**
+ * Represents the "grouping" part of document id which can be used with streaming model.
+ */
+ public static class Group {
+ public final char name;
+ public final String value;
+ Group(char name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+ }
+ private final String namespace;
+ private final String documentType;
+ private final String docId;
+ private Optional<Group> group = Optional.empty();
+ private final String rawPath;
+
+ public String getRawPath() {
+ return rawPath;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public String getDocumentType() {
+ return documentType;
+ }
+
+ public String getDocId() {
+ return docId;
+ }
+
+ public Optional<Group> getGroup() {
+ return group;
+ }
+
+ public String generateFullId() {
+ return ID + namespace + ":" + documentType + ":"
+ + (getGroup().isPresent() ? group.get().name + "=" + group.get().value : "")
+ + ":" + docId;
+ }
+
+ static class PathParser {
+ final List<String> rawParts;
+ final String originalPath;
+ int readPos = 0;
+ public PathParser(String path) {
+ this.originalPath = path;
+ this.rawParts = Splitter.on('/').splitToList(path);
+ }
+ String nextTokenOrException() throws RestApiException {
+ if (readPos >= rawParts.size()) {
+ throwUsage(originalPath);
+ }
+ return rawParts.get(readPos++);
+ }
+
+ String restOfPath() throws RestApiException {
+ String rawId = Joiner.on("/").join(rawParts.listIterator(readPos));
+ try {
+ return URLDecoder.decode(rawId, StandardCharsets.UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
+ throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,"Problems decoding the URI: " + e.getMessage()));
+ }
+ }
+ }
+
+ public RestUri(URI uri) throws RestApiException {
+ rawPath = uri.getRawPath();
+ PathParser pathParser = new PathParser(rawPath);
+ if (! pathParser.nextTokenOrException().equals("") ||
+ ! pathParser.nextTokenOrException().equals(DOCUMENT) ||
+ ! pathParser.nextTokenOrException().equals(V_1)) {
+ throwUsage(uri.getRawPath());
+ }
+ namespace = pathParser.nextTokenOrException();
+ documentType = pathParser.nextTokenOrException();
+ switch (pathParser.nextTokenOrException()) {
+ case "number":
+ group = Optional.of(new Group(NUMBER_STREAMING, pathParser.nextTokenOrException()));
+ break;
+ case "docid": group = Optional.empty();
+ break;
+ case "group":
+ group = Optional.of(new Group(GROUP_STREAMING, pathParser.nextTokenOrException()));
+ break;
+ default: throwUsage(uri.getRawPath());
+ }
+ docId = pathParser.restOfPath();
+ }
+
+ private static void throwUsage(String inputPath) throws RestApiException {
+ throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,
+ "Expected:\n" +
+ ".../{namespace}/{document-type}/group/{name}/[{user-specified}]\n" +
+ ".../{namespace}/{document-type}/docid/[{user-specified}]\n: but got " + inputPath));
+ }
+
+}
+
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java
new file mode 100644
index 00000000000..8b863a69f5c
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java
@@ -0,0 +1,229 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi.resource;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Inject;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.TestAndSetCondition;
+import com.yahoo.document.config.DocumentmanagerConfig;
+
+import com.yahoo.document.json.SingleDocumentParser;
+import com.yahoo.document.restapi.OperationHandler;
+import com.yahoo.document.restapi.OperationHandlerImpl;
+import com.yahoo.document.restapi.Response;
+import com.yahoo.document.restapi.RestApiException;
+import com.yahoo.document.restapi.RestUri;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * API for handling single operation on a document and visiting.
+ *
+ * @author dybdahl
+ */
+public class RestApi extends LoggingRequestHandler {
+
+ private static final String CREATE_PARAMETER_NAME = "create";
+ private static final String CONDITION_PARAMETER_NAME = "condition";
+ private static final String DOCUMENTS = "documents";
+ private static final String FIELDS = "fields";
+ private static final String DOC_ID_NAME = "id";
+ private static final String PATH_NAME = "pathId";
+ private static final String SELECTION = "selection";
+ private static final String CLUSTER = "cluster";
+ private static final String CONTINUATION = "continuation";
+ private final OperationHandler operationHandler;
+ private SingleDocumentParser singleDocumentParser;
+ private ObjectMapper mapper = new ObjectMapper();
+ private AtomicInteger threadsAvailableForApi = new AtomicInteger(20 /*max concurrent requests */);
+
+ @Inject
+ public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig) {
+ super(executor, accessLog);
+ final LoadTypeSet loadTypes = new LoadTypeSet("client");
+ this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(new MessageBusParams(loadTypes)));
+ this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig));
+ }
+
+ // For testing and development
+ public RestApi(
+ Executor executor,
+ AccessLog accessLog,
+ OperationHandler operationHandler) {
+ super(executor, accessLog);
+ this.operationHandler = operationHandler;
+ }
+
+ @Override
+ public void destroy() {
+ operationHandler.shutdown();
+ }
+
+ // For testing and development
+ protected void setDocTypeManagerForTests(DocumentTypeManager docTypeManager) {
+ this.singleDocumentParser = new SingleDocumentParser(docTypeManager);
+ }
+
+ // Returns null if invalid value.
+ private Optional<Boolean> parseBoolean(String parameter, HttpRequest request) {
+ final String property = request.getProperty(parameter);
+ if (property != null && ! property.isEmpty()) {
+ switch (property) {
+ case "true" : return Optional.of(true);
+ case "false": return Optional.of(false);
+ default : return null;
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ if (threadsAvailableForApi.decrementAndGet() < 1) {
+ return Response.createErrorResponse(
+ 429 /* Too Many Requests */,
+ "Too many parallel requests, consider using http-vespa-java-client. Please try again later.");
+ }
+ return handleInternal(request);
+ } finally {
+ threadsAvailableForApi.incrementAndGet();
+ }
+ }
+
+ // protected for testing
+ protected HttpResponse handleInternal(HttpRequest request) {
+ final RestUri restUri;
+ try {
+ restUri = new RestUri(request.getUri());
+ } catch (RestApiException e) {
+ return e.getResponse();
+ } catch (Exception e2) {
+ return Response.createErrorResponse(500, "Exception while parsing URI: " + e2.getMessage());
+ }
+
+ Optional<Boolean> create = parseBoolean(CREATE_PARAMETER_NAME, request);
+ if (create == null) {
+ return Response.createErrorResponse(403, "Non valid value for 'create' parameter, must be empty, true, or " +
+ "false: " + request.getProperty(CREATE_PARAMETER_NAME));
+ }
+ String condition = request.getProperty(CONDITION_PARAMETER_NAME);
+ Optional<ObjectNode> resultJson = Optional.empty();
+ try {
+ switch (request.getMethod()) {
+ case GET: // Vespa Visit/Get
+ return restUri.getDocId().isEmpty() ? handleVisit(restUri, request) : handleGet(restUri);
+ case POST: // Vespa Put
+ operationHandler.put(restUri, createPutOperation(request, restUri.generateFullId(), condition));
+ break;
+ case PUT: // Vespa Update
+ operationHandler.update(restUri, createUpdateOperation(request, restUri.generateFullId(), condition, create));
+ break;
+ case DELETE: // Vespa Delete
+ operationHandler.delete(restUri, condition);
+ break;
+ default:
+ return new Response(405, Optional.empty(), Optional.of(restUri));
+ }
+ } catch (RestApiException e) {
+ return e.getResponse();
+ } catch (Exception e2) {
+ // We always blame the user. This might be a bit nasty, but the parser throws various kind of exception
+ // types, but with nice descriptions.
+ return Response.createErrorResponse(400, e2.getMessage(), restUri);
+ }
+ return new Response(200, resultJson, Optional.of(restUri));
+ }
+
+ private VespaXMLFeedReader.Operation createPutOperation(HttpRequest request, String id, String condition) {
+ final VespaXMLFeedReader.Operation operationPut =
+ singleDocumentParser.parsePut(request.getData(), id);
+ if (condition != null && ! condition.isEmpty()) {
+ operationPut.setCondition(new TestAndSetCondition(condition));
+ }
+ return operationPut;
+ }
+
+ private VespaXMLFeedReader.Operation createUpdateOperation(HttpRequest request, String id, String condition, Optional<Boolean> create) {
+ final VespaXMLFeedReader.Operation operationUpdate =
+ singleDocumentParser.parseUpdate(request.getData(), id);
+ if (condition != null && ! condition.isEmpty()) {
+ operationUpdate.getDocumentUpdate().setCondition(new TestAndSetCondition(condition));
+ }
+ if (create.isPresent()) {
+ operationUpdate.getDocumentUpdate().setCreateIfNonExistent(create.get());
+ }
+ return operationUpdate;
+ }
+
+ private HttpResponse handleGet(RestUri restUri) throws RestApiException {
+ final Optional<String> getDocument = operationHandler.get(restUri);
+ final ObjectNode resultNode = mapper.createObjectNode();
+ if (getDocument.isPresent()) {
+ final JsonNode parseNode;
+ try {
+ parseNode = mapper.readTree(getDocument.get());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed while parsing my own results", e);
+ }
+ resultNode.putPOJO(FIELDS, parseNode.get(FIELDS));
+ }
+ resultNode.put(DOC_ID_NAME, restUri.generateFullId());
+ resultNode.put(PATH_NAME, restUri.getRawPath());
+
+ return new HttpResponse(getDocument.isPresent() ? 200 : 404) {
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8.name()));
+ }
+ };
+ }
+
+ private HttpResponse handleVisit(RestUri restUri, HttpRequest request) throws RestApiException {
+ if (restUri.getGroup().isPresent() && ! restUri.getGroup().get().value.isEmpty()) {
+ return Response.createErrorResponse(
+ 400,
+ "Visiting does not support setting value for group/value, try using expression parameter instead.",
+ restUri);
+
+ }
+ String documentSelection = Optional.ofNullable(request.getProperty(SELECTION)).orElse("");
+ Optional<String> cluster = Optional.ofNullable(request.getProperty(CLUSTER));
+ Optional<String> continuation = Optional.ofNullable(request.getProperty(CONTINUATION));
+ final OperationHandler.VisitResult visit = operationHandler.visit(restUri, documentSelection, cluster, continuation);
+ final ObjectNode resultNode = mapper.createObjectNode();
+ if (visit.token.isPresent()) {
+ resultNode.put(CONTINUATION, visit.token.get());
+ }
+ resultNode.putArray(DOCUMENTS).addPOJO(visit.documentsAsJsonList);
+ resultNode.put(PATH_NAME, restUri.getRawPath());
+
+ HttpResponse httpResponse = new HttpResponse(200) {
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ try {
+ outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ return httpResponse;
+ }
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java
new file mode 100644
index 00000000000..52e32009e00
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.document.restapi.resource;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java
new file mode 100755
index 00000000000..d8989e347f6
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.metrics.MetricManager;
+import com.yahoo.metrics.MetricSnapshot;
+import com.yahoo.text.Utf8String;
+import com.yahoo.text.XMLWriter;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+public class StatusResponse extends HttpResponse {
+
+ MetricManager manager;
+ int verbosity;
+ int snapshotTime;
+
+ StatusResponse(MetricManager manager, int verbosity, int snapshotTime) {
+ super(com.yahoo.jdisc.http.HttpResponse.Status.OK);
+ this.manager = manager;
+ this.snapshotTime = snapshotTime;
+ this.verbosity = verbosity;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream));
+ writer.openTag("status");
+ if (verbosity >= 2) {
+ writer.attribute(new Utf8String("description"), "Metrics since start");
+ }
+
+ if (snapshotTime == 0) {
+ MetricSnapshot snapshot = (new MetricSnapshot(
+ "Total metrics from start until current time", 0,
+ manager.getActiveMetrics().getMetrics(), false));
+ manager.getTotalMetricSnapshot().addToSnapshot(snapshot, (int)(System.currentTimeMillis() / 1000), false);
+ snapshot.printXml(manager, "", verbosity, writer);
+ } else {
+ try {
+ manager.getMetricSnapshotSet(snapshotTime).getSnapshot().printXml(manager, "", verbosity, writer);
+ } catch (Exception e) {
+ writer.openTag("error");
+ writer.attribute(new Utf8String("details"), "No metric snapshot with period " + snapshotTime +
+ " was found. Legal snapshot periods are: " + manager.getSnapshotPeriods());
+ writer.closeTag();
+ }
+ }
+ writer.closeTag();
+ writer.flush();
+ }
+
+ @Override
+ public java.lang.String getContentType() {
+ return "application/xml";
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java
new file mode 100755
index 00000000000..a4e0ddd1748
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import java.util.concurrent.Executor;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+
+public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler {
+
+ private final VespaFeedHandlerGet getHandler;
+ private final VespaFeedHandler feedHandler;
+
+ public VespaFeedHandlerCompatibility(Executor executor, VespaFeedHandlerGet getHandler,
+ VespaFeedHandler feedHandler) {
+ super(executor);
+ this.getHandler = getHandler;
+ this.feedHandler = feedHandler;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ boolean hasType = request.hasProperty("type");
+ // If we have an ID and no document type, redirect to Get
+ if (request.hasProperty("id") && !hasType) {
+ return getHandler.handle(request);
+ } else {
+ return feedHandler.handle(request);
+ }
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java
new file mode 100755
index 00000000000..aadfe5852ce
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import java.util.Collections;
+import java.util.concurrent.Executor;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+import com.yahoo.search.handler.SearchHandler;
+
+public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler {
+
+ private final SearchHandler searchHandler;
+
+ public VespaFeedHandlerGet(SearchHandler searchHandler, Executor executor) {
+ super(executor, null, true);
+ this.searchHandler = searchHandler;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget")));
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
new file mode 100755
index 00000000000..14b2d86ae75
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import com.google.inject.Inject;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.document.DocumentId;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.feedapi.SingleSender;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.concurrent.Executor;
+
+public class VespaFeedHandlerRemove extends VespaFeedHandlerBase {
+
+ @Inject
+ public VespaFeedHandlerRemove(FeederConfig feederConfig,
+ LoadTypeConfig loadTypeConfig, Executor executor, Metric metric) throws Exception {
+ super(feederConfig, loadTypeConfig, executor, metric);
+ }
+
+ VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception {
+ super(context, executor);
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ if (request.getProperty("status") != null) {
+ return new MetricResponse(context.getMetrics().getMetricSet());
+ }
+
+ MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request);
+ String route = properties.getRoute().toString();
+ FeedResponse response = new FeedResponse(new RouteMetricSet(route, null));
+ SingleSender sender = new SingleSender(response, getSharedSender(route));
+ sender.addMessageProcessor(properties);
+
+ response.setAbortOnFeedError(properties.getAbortOnFeedError());
+
+ if (request.hasProperty("id")) {
+ sender.remove(new DocumentId(request.getProperty("id")));
+ } else if (request.hasProperty("id[0]")) {
+ int index = 0;
+ while (request.hasProperty("id[" + index + "]")) {
+ sender.remove(new DocumentId(request.getProperty("id[" + index + "]")));
+ ++index;
+ }
+ }
+
+ if (request.getData() != null) {
+ try {
+ String line;
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(getRequestInputStream(request), "UTF-8"));
+ while ((line = reader.readLine()) != null) {
+ sender.remove(new DocumentId(line));
+ }
+ } catch (Exception e) {
+ response.addError(e.getClass() + ": " + e.getCause());
+ }
+ }
+
+ sender.done();
+ long millis = getTimeoutMillis(request);
+ boolean completed = sender.waitForPending(millis);
+ if ( ! completed)
+ response.addError("Timed out after "+millis+" ms waiting for responses");
+ return response;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
new file mode 100644
index 00000000000..3b2f82c865e
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import com.google.inject.Inject;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.feedapi.SingleSender;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.util.concurrent.Executor;
+
+public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase {
+
+ @Inject
+ public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor,
+ Metric metric) throws Exception {
+ super(feederConfig, loadTypeConfig, executor, metric);
+ }
+
+ VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception {
+ super(context, executor);
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request);
+ FeedResponse response;
+
+ if (request.getProperty("route") == null) {
+ if (context.getClusterList().getStorageClusters().size() == 0) {
+ return new FeedResponse(null).addError("No storage clusters configured and no alternate route specified.");
+ } else if (context.getClusterList().getStorageClusters().size() > 1) {
+ return new FeedResponse(null).addError("More than one storage cluster configured and no route specified.");
+ } else {
+ properties.setRoute(Route.parse(context.getClusterList().getStorageClusters().get(0).getName()));
+ }
+ }
+
+ response = new FeedResponse(new RouteMetricSet(properties.getRoute().toString(), null));
+
+ SingleSender sender = new SingleSender(response, getSharedSender(properties.getRoute().toString()));
+ sender.addMessageProcessor(properties);
+
+ String user = request.getProperty("user");
+ String group = request.getProperty("group");
+ String selection = request.getProperty("selection");
+
+ boolean oneFound = (user != null) ^ (group != null) ^ (selection != null);
+
+ if (!oneFound) {
+ response.addError("Exactly one of \"user\", \"group\" or \"selection\" must be specified for removelocation");
+ return response;
+ }
+
+ if (user != null) {
+ selection = "id.user=" + user;
+ }
+ if (group != null) {
+ selection = "id.group=\"" + group + "\"";
+ }
+
+ sender.send(new RemoveLocationMessage(selection));
+ sender.done();
+ long millis = getTimeoutMillis(request);
+ boolean completed = sender.waitForPending(millis);
+ if ( ! completed)
+ response.addError("Timed out after "+millis+" ms waiting for responses");
+ return response;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java
new file mode 100755
index 00000000000..77930ae5a94
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import java.util.concurrent.Executor;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.metrics.MetricManager;
+import com.yahoo.metrics.MetricSet;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler {
+
+ private MetricManager manager;
+
+ public VespaFeedHandlerStatus(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor) {
+ this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()), true, true, executor);
+ }
+
+ VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) {
+ super(executor);
+ manager = new MetricManager();
+ final MetricSet metricSet = context.getMetrics().getMetricSet();
+ metricSet.unregister();
+ manager.registerMetric(metricSet);
+ if (doLog) {
+ manager.addMetricToConsumer("log", "routes.total.putdocument.count");
+ manager.addMetricToConsumer("log", "routes.total.removedocument.count");
+ manager.addMetricToConsumer("log", "routes.total.updatedocument.count");
+ manager.addMetricToConsumer("log", "routes.total.getdocument.count");
+
+ manager.addMetricToConsumer("log", "routes.total.putdocument.errors.total");
+ manager.addMetricToConsumer("log", "routes.total.removedocument.errors.total");
+ manager.addMetricToConsumer("log", "routes.total.updatedocument.errors.total");
+ manager.addMetricToConsumer("log", "routes.total.getdocument.errors.total");
+
+ manager.addMetricToConsumer("log", "routes.total.putdocument.latency");
+ manager.addMetricToConsumer("log", "routes.total.removedocument.latency");
+ manager.addMetricToConsumer("log", "routes.total.updatedocument.latency");
+ manager.addMetricToConsumer("log", "routes.total.getdocument.latency");
+ }
+
+ if (doLog || makeSnapshots) {
+ new Thread(manager).start();
+ }
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ return new StatusResponse(manager, asInt(request.getProperty("verbosity"), 0), asInt(request.getProperty("snapshotperiod"), 0));
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ private int asInt(String value, int defaultValue) {
+ if (value == null) return defaultValue;
+ return Integer.parseInt(value);
+ }
+
+ @Override
+ public void destroy() {
+ manager.stop();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java
new file mode 100644
index 00000000000..dba6bb78be1
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import java.util.Collections;
+import java.util.concurrent.Executor;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
+import com.yahoo.search.handler.SearchHandler;
+
+/**
+ * @author thomasg
+ */
+public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler {
+
+ private final SearchHandler searchHandler;
+
+ public VespaFeedHandlerVisit(SearchHandler searchHandler, Executor executor) {
+ super(executor, null, true);
+ this.searchHandler = searchHandler;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespavisit")));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java
new file mode 100755
index 00000000000..cae35669c02
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.search.result.Hit;
+import java.io.IOException;
+import java.util.Base64;
+
+public class ContinuationHit extends Hit {
+
+ private final String value;
+
+ public ContinuationHit(ProgressToken token) {
+ super("continuation");
+
+ final byte[] serialized = token.serialize();
+ value = Base64.getUrlEncoder().encodeToString(serialized);
+ }
+
+ public static ProgressToken getToken(String continuation) throws IOException {
+ byte[] serialized;
+ try {
+ serialized = Base64.getUrlDecoder().decode(continuation);
+ } catch (IllegalArgumentException e) {
+ // Legacy visitor tokens were encoded with MIME Base64 which may fail decoding as URL-safe.
+ // Try again with MIME decoder to avoid breaking upgrade scenarios.
+ // TODO(vekterli): remove once this is no longer a risk.
+ serialized = Base64.getMimeDecoder().decode(continuation);
+ }
+ return new ProgressToken(serialized);
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java
new file mode 100755
index 00000000000..901ea44102b
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.io.ByteWriter;
+import com.yahoo.prelude.templates.Context;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.text.XML;
+
+import java.io.IOException;
+import java.io.Writer;
+
+/**
+ * Template used to render a single field for a single Document. Fields
+ * that are either of type CONTENT or RAW are written directly, while
+ * all other fields are wrapped in Vespa XML and escaped.
+ */
+public class DocumentFieldTemplate extends UserTemplate<Writer> {
+
+ Field field;
+ String contentType;
+ String encoding;
+ boolean wrapXml;
+
+ public DocumentFieldTemplate(Field field, String contentType,
+ String encoding, boolean wrapXml) {
+ // Defaults here are chosen as to give the same expected results as
+ // HTTPGateway's GetRequest
+ super("documentfield", contentType,
+ encoding);
+
+ this.field = field;
+ this.contentType = contentType;
+ this.encoding = encoding;
+ this.wrapXml = wrapXml;
+ }
+
+ @Override
+ public void error(Context context, Writer writer) throws IOException {
+ // Error shouldn't be handled by this template, but rather
+ // delegated to the searcher
+ }
+
+ @Override
+ public Writer wrapWriter(Writer writer) {
+ /* TODO: uncomment
+ if (!(writer instanceof ByteWriter)) {
+ throw new IllegalArgumentException("ByteWriter required, but got " + writer.getClass().getName());
+ }
+ */
+
+ return writer;
+ }
+
+ @Override
+ public void header(Context context, Writer writer) throws IOException {
+ if (wrapXml) {
+ // XML wrapping should only be used for default field rendering
+ writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n");
+ writer.write("<result>");
+ }
+ }
+
+ @Override
+ public void footer(Context context, Writer writer) throws IOException {
+ if (wrapXml) {
+ writer.write("</result>\n");
+ }
+ }
+
+ @Override
+ public void hit(Context context, Writer writer) throws IOException {
+ DocumentHit hit = (DocumentHit)context.get("hit");
+ Document doc = hit.getDocument();
+ // Assume field existence has been checked before we ever get here.
+ // Also assume that relevant encoding/content type is set
+ // appropriately according to the request and the field's content
+ // type, as this is immutable in the template set.
+ FieldValue value = doc.getFieldValue(field);
+ if (field.getDataType() == DataType.RAW) {
+ ByteWriter bw = (ByteWriter)writer;
+ bw.append(((Raw) value).getByteBuffer().array());
+ } else {
+ writer.write(XML.xmlEscape(value.toString(), false));
+ }
+ }
+
+ @Override
+ public void hitFooter(Context context, Writer writer) throws IOException {
+ }
+
+ @Override
+ public void noHits(Context context, Writer writer) throws IOException {
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java
new file mode 100755
index 00000000000..8520b5352e3
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.search.result.Hit;
+
+import java.util.Iterator;
+import java.util.Map;
+
+public class DocumentHit extends Hit {
+
+ private Document document;
+ private int index;
+
+ public DocumentHit(Document document, int index) {
+ super(document.getId().toString());
+ this.document = document;
+ this.index = index;
+ }
+
+ public void populateHitFields() {
+ // Create hit fields for all document fields
+ Iterator<Map.Entry<Field, FieldValue>> fieldIter = document.iterator();
+ while (fieldIter.hasNext()) {
+ Map.Entry<Field, FieldValue> field = fieldIter.next();
+ setField(field.getKey().getName(), field.getValue());
+ }
+
+ // Assign an explicit document id field
+ setField("documentid", document.getId().toString());
+ }
+
+ public Document getDocument() {
+ return document;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java
new file mode 100644
index 00000000000..cf5dfbcffc1
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.search.result.Hit;
+
+public class DocumentRemoveHit extends Hit {
+
+ private final DocumentId idOfRemovedDoc;
+
+ public DocumentRemoveHit(DocumentId idOfRemovedDoc) {
+ super(idOfRemovedDoc.toString());
+ this.idOfRemovedDoc = idOfRemovedDoc;
+ setField("documentid", idOfRemovedDoc.toString());
+ }
+
+ public DocumentId getIdOfRemovedDoc() {
+ return idOfRemovedDoc;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java
new file mode 100755
index 00000000000..5266a2e1d09
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.ErrorHit;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.prelude.templates.Context;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.search.result.Hit;
+import com.yahoo.text.XML;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.logging.Logger;
+
+public class DocumentXMLTemplate extends UserTemplate<Writer> {
+
+ private static final Logger log = Logger.getLogger(DocumentXMLTemplate.class.getName());
+
+ public DocumentXMLTemplate() {
+ super("vespa_xml");
+ }
+
+ public DocumentXMLTemplate(String mimeType, String encoding) {
+ super("vespa_xml", mimeType, encoding);
+ }
+
+ private void writeErrorMessage(Writer writer, String type, int code,
+ String message, String detailedMessage) throws IOException {
+ writer.write("<error type=\"" + type + "\" code=\"" + code + "\" message=\"");
+ writer.write(XML.xmlEscape(message, true));
+ if (detailedMessage != null) {
+ writer.write(": ");
+ writer.write(XML.xmlEscape(detailedMessage, true));
+ }
+ writer.write("\"/>\n");
+ }
+
+ private void writeGenericErrorMessage(Writer writer, ErrorMessage message) throws IOException {
+ // A bit dirty, but we don't have to support many different types
+ if (message instanceof MessageBusErrorMessage) {
+ writeErrorMessage(writer, "messagebus",
+ ((MessageBusErrorMessage)message).getMessageBusCode(),
+ message.getMessage(), message.getDetailedMessage());
+ } else {
+ writeErrorMessage(writer, "searcher", message.getCode(),
+ message.getMessage(), message.getDetailedMessage());
+ }
+ }
+
+ @Override
+ public void error(Context context, Writer writer) throws IOException {
+ writer.write("<errors>\n");
+ // If the error contains no error hits, use a single error with the main
+ // code and description. Otherwise, use the error hits explicitly
+ ErrorHit errorHit = ((Result)context.get("result")).hits().getErrorHit();
+ if (errorHit == null || errorHit.errors().isEmpty()) {
+ ErrorMessage message = ((Result)context.get("result")).hits().getError();
+ writeGenericErrorMessage(writer, message);
+ } else {
+ for (ErrorMessage message : errorHit.errors()) {
+ writeGenericErrorMessage(writer, message);
+ }
+ }
+ writer.write("</errors>\n");
+ }
+
+ @Override
+ public void header(Context context, Writer writer) throws IOException {
+ writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+ writer.write("<result>\n");
+ HitGroup rootGroup = ((Result) context.get("result")).hits();
+ if (rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) != null) {
+ writer.write("<continuation>" + rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) + "</continuation>");
+ }
+ }
+
+ @Override
+ public void footer(Context context, Writer writer) throws IOException {
+ writer.write("</result>\n");
+ }
+
+ @Override
+ public void hit(Context context, Writer writer) throws IOException {
+ Hit hit = (Hit)context.get("hit");
+ if (hit instanceof DocumentHit) {
+ DocumentHit docHit = (DocumentHit) hit;
+ if (docHit.getDocument() != null) {
+ writer.write(docHit.getDocument().toXML(" "));
+ }
+ } else if (hit instanceof DocumentRemoveHit) {
+ writeDocumentRemoveHit(writer, (DocumentRemoveHit) hit);
+ } else {
+ log.log(LogLevel.WARNING, "Cannot render document XML; expected hit of type " +
+ "com.yahoo.storage.searcher.Document[Remove]Hit, got " + hit.getClass().getName() +
+ ". Is there another backend searcher present?");
+ }
+ }
+
+ private void writeDocumentRemoveHit(Writer writer, DocumentRemoveHit remove) throws IOException {
+ writer.write("<remove documentid=\"");
+ writer.write(XML.xmlEscape(remove.getIdOfRemovedDoc().toString()));
+ writer.write("\"/>\n");
+ }
+
+ @Override
+ public void hitFooter(Context context, Writer writer) throws IOException {
+ }
+
+ @Override
+ public void noHits(Context context, Writer writer) throws IOException {
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java
new file mode 100755
index 00000000000..661fcac6a64
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java
@@ -0,0 +1,469 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.feedapi.*;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.DefaultErrorHit;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Searcher component to make GET requests to a content cluster.
+ * <p>
+ * Document ID must be given either as 1 "id=docid" query parameter
+ * for single-document GETs and 1-n "id[0]=docid_1&amp;id[1]=...&amp;id[n-1]=docid_n"
+ * parameters for multi-document GETs.
+ *
+ * <p>
+ * Standard gateway query parameters are implicitly supported:
+ * priority, timeout, route
+ *
+ * <p>
+ * The searcher also accepts the following (optional) query parameters:
+ * headersonly=true|false (default: false)
+ * For specifying whether or not to return only header fields.
+ *
+ * <p>
+ * field=string (default: no parameter specified)
+ * For getting a single document field.
+ *
+ * <p>
+ * contenttype=string (default: no content type specified)
+ * For specifiying the returned HTTP header content type for a returned
+ * document field's content. field must also be specified.
+ */
+public class GetSearcher extends Searcher {
+
+ private static final Logger log = Logger.getLogger(GetSearcher.class.getName());
+
+ FeedContext context;
+
+ private final long defaultTimeoutMillis;
+
+ private class GetResponse implements SharedSender.ResultCallback {
+
+ /**
+ * We have to maintain the same ordering of results as that
+ * given in the request. Do this by remembering the index of
+ * each requested document ID.
+ */
+ private Map<String, Integer> ordering;
+ private List<DocumentHit> documentHits = new ArrayList<>();
+ private List<DefaultErrorHit> errorHits = new ArrayList<>();
+
+ public GetResponse(List<String> documentIds) {
+ ordering = new HashMap<>(documentIds.size());
+ for (int i = 0; i < documentIds.size(); ++i) {
+ ordering.put(documentIds.get(i), i);
+ }
+ }
+
+ public boolean isAborted() {
+ return false;
+ }
+
+ private String stackTraceFromException(Exception e) {
+ StringWriter sw = new StringWriter();
+ PrintWriter ps = new PrintWriter(sw);
+ e.printStackTrace(ps);
+ ps.flush();
+ return sw.toString();
+ }
+
+ public boolean handleReply(Reply reply, int numPending) {
+ if (!reply.hasErrors()) {
+ try {
+ addDocumentHit(reply);
+ } catch (Exception e) {
+ String msg = "Got exception of type " + e.getClass().getName()
+ + " during document deserialization: " + e.getMessage();
+ errorHits.add(new DefaultErrorHit("GetSearcher",
+ ErrorMessage.createInternalServerError(msg)));
+ log.log(LogLevel.DEBUG, "Got exception during document deserialization: " + stackTraceFromException(e));
+ }
+ } else {
+ errorHits.add(new DefaultErrorHit("GetSearcher", new MessageBusErrorMessage(
+ reply.getError(0).getCode(), 0, reply.getError(0).getMessage())));
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received error reply with message " + reply.getError(0).getMessage());
+ }
+ }
+
+ return (numPending > 0);
+ }
+
+ private void addDocumentHit(Reply reply) {
+ Document doc = ((GetDocumentReply)reply).getDocument();
+ GetDocumentMessage msg = (GetDocumentMessage)reply.getMessage();
+ Integer index = ordering.get(msg.getDocumentId().toString());
+ if (index == null) { // Shouldn't happen
+ throw new IllegalStateException("Received GetDocumentReply for unknown document: "
+ + doc.getId().toString());
+ }
+ if (doc != null) {
+ documentHits.add(new DocumentHit(doc, index));
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received GetDocumentReply for "
+ + doc.getId().toString());
+ }
+ } else {
+ // Don't add a hit for documents that can't be found
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received empty (not found) GetDocumentReply for "
+ + msg.getDocumentId().toString());
+ }
+ }
+ }
+
+ private class IndexComparator implements Comparator<DocumentHit> {
+ public int compare(DocumentHit o1, DocumentHit o2) {
+ return o1.getIndex() - o2.getIndex();
+ }
+ }
+
+ public void addHitsToResult(Result result, boolean populateHitFields) {
+ for (DefaultErrorHit hit : errorHits) {
+ result.hits().add(hit);
+ }
+ // Sort document hits according to their request index
+ Collections.sort(documentHits, new IndexComparator());
+ for (DocumentHit hit : documentHits) {
+ if (populateHitFields) {
+ hit.populateHitFields();
+ }
+ result.hits().add(hit);
+ }
+ result.setTotalHitCount(documentHits.size());
+ }
+
+ public List<DocumentHit> getDocumentHits() {
+ return documentHits;
+ }
+
+ public List<DefaultErrorHit> getErrorHits() {
+ return errorHits;
+ }
+ }
+
+ @Inject
+ public GetSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception {
+ this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()),
+ (long)(feederConfig.timeout() * 1000));
+ }
+
+ GetSearcher(FeedContext context) throws Exception {
+ this.context = context;
+ this.defaultTimeoutMillis = context.getPropertyProcessor().getDefaultTimeoutMillis();
+ }
+
+ GetSearcher(FeedContext context, long defaultTimeoutMillis) throws Exception {
+ this.context = context;
+ this.defaultTimeoutMillis = defaultTimeoutMillis;
+ }
+
+ private void postValidateDocumentIdParameters(Properties properties, int arrayIdsFound) throws Exception {
+ for (Map.Entry<String, Object> kv : properties.listProperties().entrySet()) {
+ if (!kv.getKey().startsWith("id[")) {
+ continue;
+ }
+ if (!kv.getKey().endsWith("]")) {
+ throw new IllegalArgumentException("Malformed document ID array parameter");
+ }
+ String indexStr = kv.getKey().substring(3, kv.getKey().length() - 1);
+ int idx = Integer.parseInt(indexStr);
+ if (idx >= arrayIdsFound) {
+ throw new IllegalArgumentException("query contains document ID array " +
+ "that is not zero-based and/or linearly increasing");
+ }
+ }
+ }
+
+ private List<String> getDocumentIds(Query query) throws Exception {
+ Properties properties = query.properties();
+ List<String> docIds = new ArrayList<>();
+
+ // First check for regular "id=XX" syntax. If found, return vector with that
+ // document id only
+ String singleId = properties.getString("id");
+
+ int index = 0;
+ if (singleId != null) {
+ docIds.add(singleId);
+ } else {
+ // Check for id[0]=XX&id[1]=YY...id[n]=ZZ syntax. Indices always start
+ // at 0 and are always increased by 1.
+ while (true) {
+ String docId = properties.getString("id[" + index + "]");
+ if (docId == null) {
+ break;
+ }
+ docIds.add(docId);
+ ++index;
+ }
+ }
+ postValidateDocumentIdParameters(properties, index);
+
+ handleData(query.getHttpRequest(), docIds);
+ return docIds;
+ }
+
+ private void handleData(HttpRequest request, List<String> docIds) throws IOException {
+ if (request.getData() != null) {
+ InputStream input;
+ if ("gzip".equals(request.getHeader("Content-Encoding"))) {
+ input = new GZIPInputStream(request.getData());
+ } else {
+ input = request.getData();
+ }
+ InputStreamReader reader = new InputStreamReader(input, "UTF-8");
+ BufferedReader lineReader = new BufferedReader(reader);
+ String line;
+ while ((line = lineReader.readLine()) != null) {
+ docIds.add(line);
+ }
+ }
+ }
+
+ private void handleFieldFiltering(GetResponse response, Result result,
+ String fieldName, String contentType,
+ boolean headersOnly) {
+
+ if (response.getDocumentHits().isEmpty()) {
+ result.hits().addError(ErrorMessage.createNotFound(
+ "Document not found, could not return field '" + fieldName + "'"));
+ return;
+ }
+
+ if (result.hits().getErrorHit() == null) {
+ Document doc = response.getDocumentHits().get(0).getDocument();
+ Field field = doc.getDataType().getField(fieldName);
+ boolean wrapXml = false;
+
+ if (field == null) {
+ result.hits().addError(ErrorMessage.createIllegalQuery(
+ "Field '" + fieldName + "' not found in document type"));
+ return;
+ }
+ FieldValue value = doc.getFieldValue(field);
+ // If the field exists but hasn't been set in this document, the
+ // content will be null. We treat this as an error.
+ if (value == null) {
+ if (!field.isHeader() && headersOnly) {
+ // TODO(vekterli): make this work with field sets as well.
+ result.hits().addError(ErrorMessage.createInvalidQueryParameter(
+ "Field '" + fieldName + "' is located in document body, but headersonly "
+ + "prevents it from being retrieved in " + doc.getId().toString()));
+ } else {
+ result.hits().addError(ErrorMessage.createNotFound(
+ "Field '" + fieldName + "' found in document type, but had "
+ + "no content in " + doc.getId().toString()));
+ }
+ return;
+ }
+ String encoding = null;
+ if (field.getDataType() == DataType.RAW) {
+ if (contentType == null) {
+ contentType = "application/octet-stream";
+ }
+ encoding = "ISO-8859-1";
+ } else {
+ // By default, return field wrapped in a blanket of vespa XML
+ contentType = "text/xml";
+ wrapXml = true;
+ }
+ if (encoding == null) {
+ // Encoding doesn't matter for binary content, since we're always
+ // writing directly to the byte buffer and not through a charset
+ // encoder. Presumably, the client is intelligent enough to not
+ // attempt to UTF-8 decode binary data.
+ encoding = "UTF-8";
+ }
+ // Add hit now that we know there aren't any field errors. Otherwise,
+ // there would be both an error hit and a document hit in the result
+ response.addHitsToResult(result, false);
+ // Override Vespa XML template
+ result.getTemplating().setTemplates(new DocumentFieldTemplate(field, contentType, encoding, wrapXml));
+ }
+ // else: return with error hit, invoking regular Vespa XML error template
+ }
+
+ private void validateParameters(String fieldName, String contentType,
+ List<String> documentIds) {
+ // Content-type only makes sense for single document queries with a field
+ // set
+ if (contentType != null) {
+ if (documentIds.size() > 1) {
+ throw new IllegalArgumentException(
+ "contenttype parameter only valid for single document id query");
+ }
+ if (fieldName == null) {
+ throw new IllegalArgumentException(
+ "contenttype set without document field being specified");
+ }
+ }
+ if (fieldName != null && documentIds.size() > 1) {
+ throw new IllegalArgumentException(
+ "Field only valid for single document id query");
+ }
+ }
+
+ // For unit testing
+ protected MessagePropertyProcessor getMessagePropertyProcessor() {
+ return context.getPropertyProcessor();
+ }
+
+ private void doGetDocuments(Query query, Result result, List<String> documentIds) {
+ GetResponse response = new GetResponse(documentIds);
+ Properties properties = query.properties();
+
+ boolean headersOnly = properties.getBoolean("headersonly", false);
+ boolean populateHitFields = properties.getBoolean("populatehitfields", false);
+ String fieldSet = properties.getString("fieldset");
+ String fieldName = properties.getString("field");
+ String contentType = properties.getString("contenttype");
+ long timeoutMillis = properties.getString("timeout") != null ? query.getTimeout() : defaultTimeoutMillis;
+
+ if (fieldSet == null) {
+ fieldSet = headersOnly ? "[header]" : "[all]";
+ }
+
+ validateParameters(fieldName, contentType, documentIds);
+
+ MessagePropertyProcessor.PropertySetter propertySetter;
+ propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest());
+
+ SingleSender sender = new SingleSender(response, context.getSharedSender(propertySetter.getRoute().toString()));
+ sender.addMessageProcessor(propertySetter);
+
+ sendDocumentGetMessages(documentIds, fieldSet, sender);
+ // Twiddle thumbs until we've received a reply for all documents
+ sender.done();
+ boolean completed = sender.waitForPending(timeoutMillis);
+ if ( ! completed) {
+ result.hits().addError(ErrorMessage.createTimeout(
+ "Timed out after waiting "+timeoutMillis+" ms for responses"));
+ }
+ if (fieldName != null) {
+ handleFieldFiltering(response, result, fieldName, contentType, headersOnly);
+ } else {
+ response.addHitsToResult(result, populateHitFields);
+ }
+ }
+
+ private void sendDocumentGetMessages(List<String> documentIds, String fieldSet, SingleSender sender) {
+ for (String docIdStr : documentIds) {
+ DocumentId docId = new DocumentId(docIdStr);
+ GetDocumentMessage getMsg = new GetDocumentMessage(docId, fieldSet);
+
+ sender.send(getMsg);
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Sent GetDocumentMessage for "
+ + docId.toString());
+ }
+ }
+ }
+
+ boolean verifyBackendDocumentHitsOnly(Result result) {
+ if (result.hits().size() != 0) {
+ log.log(LogLevel.DEBUG, "Result had hits after being sent down");
+ for (int i = 0; i < result.hits().size(); ++i) {
+ if (!(result.hits().get(i) instanceof DocumentHit)) {
+ log.log(LogLevel.WARNING, "Got hit from backend searcher which was "
+ + "not a com.yahoo.storage.searcher.DocumentHit instance: "
+ + result.hits().get(i).getClass().getName());
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Result search(Query query, Execution execution) {
+ // Pass through to next searcher
+ Result result = execution.search(query);
+
+ List<String> documentIds;
+ try {
+ documentIds = getDocumentIds(query);
+ } catch (Exception e) {
+ setOutputFormat(query, result);
+ result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage()));
+ return result;
+ }
+ // Early-out for pass-through queries
+ if (documentIds.isEmpty()) {
+ return result;
+ }
+ // Make sure we don't try to combine non-document hits and document hits
+ // in the same result set.
+ if (!verifyBackendDocumentHitsOnly(result)) {
+ result = new Result(query); // Don't include unknown hits
+ setOutputFormat(query, result);
+ result.hits().addError(ErrorMessage.createInternalServerError(
+ "A backend searcher to com.yahoo.storage.searcher.GetSearcher " +
+ "returned a hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " +
+ "Only DocumentHit instances are supported in the backend hit result set when doing " +
+ "queries that contain document identifier sets recognised by the Get Searcher."));
+ return result;
+ }
+ setOutputFormat(query, result);
+ // Do not propagate exceptions back up, as we want to have all errors
+ // be reported using the proper template
+ try {
+ doGetDocuments(query, result, documentIds);
+ query.setHits(result.hits().size());
+ } catch (IllegalArgumentException e) {
+ result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage()));
+ } catch (Exception e) {
+ result.hits().addError(ErrorMessage.createUnspecifiedError(e.getClass().getName() + ": " + e.getMessage()));
+ }
+
+ return result;
+ }
+
+ private static final CompoundName formatShortcut = new CompoundName("format");
+ private static final CompoundName format = new CompoundName("presentation.format");
+
+ /**
+ * Use custom XML output format unless the default JSON renderer is specified in the request.
+ */
+ @SuppressWarnings("deprecation")
+ static void setOutputFormat(Query query, Result result) {
+ if (getRequestProperty(formatShortcut, "", query).equals("JsonRenderer")) return;
+ if (getRequestProperty(format, "", query).equals("JsonRenderer")) return;
+ if (getRequestProperty(formatShortcut, "", query).equals("json")) return;
+ if (getRequestProperty(format, "", query).equals("json")) return;
+ result.getTemplating().setTemplates(new DocumentXMLTemplate());
+ }
+
+ private static String getRequestProperty(CompoundName propertyName, String defaultValue, Query query) {
+ String propertyValue = query.getHttpRequest().getProperty(propertyName.toString());
+ if (propertyValue == null) return defaultValue;
+ return propertyValue;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java
new file mode 100755
index 00000000000..7cc8a6514f7
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.search.result.ErrorMessage;
+
+/**
+ * Simple ErrorMessage extension that includes a message bus error code, not
+ * just the searcher error code (which isn't very useful for a Vespa XML consumer)
+ */
+public class MessageBusErrorMessage extends ErrorMessage {
+
+ private int mbusCode;
+
+ public MessageBusErrorMessage(int mbusCode, int qrsCode, String message) {
+ super(qrsCode, message);
+ this.mbusCode = mbusCode;
+ }
+
+ public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage) {
+ super(qrsCode, message, detailedMessage);
+ this.mbusCode = mbusCode;
+ }
+
+ public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage, Throwable cause) {
+ super(qrsCode, message, detailedMessage, cause);
+ this.mbusCode = mbusCode;
+ }
+
+ public int getMessageBusCode() {
+ return mbusCode;
+ }
+
+ public void setMessageBusCode(int code) {
+ this.mbusCode = code;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java
new file mode 100644
index 00000000000..621ffcefbe1
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java
@@ -0,0 +1,194 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.documentapi.*;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vdslib.VisitorOrdering;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+/**
+ * A searcher that allows you to iterate through a storage cluster using visiting.
+ */
+public class VisitSearcher extends Searcher {
+
+ public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken";
+ FeedContext context;
+
+ public VisitSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception {
+ this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()));
+ }
+
+ VisitSearcher(FeedContext context) throws Exception {
+ this.context = context;
+ }
+
+ class HitDataHandler extends DumpVisitorDataHandler {
+ private final Result result;
+ private final boolean populateHits;
+ private final Object monitor = new Object();
+
+ HitDataHandler(Result result, boolean populateHits) {
+ this.result = result;
+ this.populateHits = populateHits;
+ }
+
+ @Override
+ public void onDocument(Document document, long l) {
+ final DocumentHit hit = new DocumentHit(document, 0);
+ if (populateHits) {
+ hit.populateHitFields();
+ }
+ synchronized (monitor) {
+ result.hits().add(hit);
+ }
+ }
+
+ @Override
+ public void onRemove(DocumentId documentId) {
+ final DocumentRemoveHit hit = new DocumentRemoveHit(documentId);
+ synchronized (monitor) {
+ result.hits().add(hit);
+ }
+ }
+ }
+
+ public VisitorParameters getVisitorParameters(Query query, Result result) throws Exception {
+ String documentSelection = query.properties().getString("visit.selection");
+ if (documentSelection == null) {
+ documentSelection = "";
+ }
+
+ VisitorParameters params = new VisitorParameters(documentSelection);
+ params.setMaxBucketsPerVisitor(query.properties().getInteger("visit.maxBucketsPerVisitor", 1));
+ params.setMaxPending(query.properties().getInteger("visit.maxPendingMessagesPerVisitor", 32));
+ params.setMaxFirstPassHits(query.properties().getInteger("visit.approxMaxDocs", 1));
+ params.setMaxTotalHits(query.properties().getInteger("visit.approxMaxDocs", 1));
+ params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(
+ query.properties().getInteger("visit.maxPendingVisitors", 1)));
+ params.setToTimestamp(query.properties().getLong("visit.toTimestamp", 0L));
+ params.setFromTimestamp(query.properties().getLong("visit.fromTimestamp", 0L));
+
+ String pri = query.properties().getString("visit.priority");
+ if (pri != null) {
+ params.setPriority(DocumentProtocol.Priority.valueOf(pri));
+ }
+
+ if (query.properties().getBoolean("visit.visitInconsistentBuckets")) {
+ params.visitInconsistentBuckets(true);
+ }
+
+ String ordering = query.properties().getString("visit.order");
+ if (!"ascending".equalsIgnoreCase(ordering)) {
+ params.setVisitorOrdering(VisitorOrdering.ASCENDING);
+ } else {
+ params.setVisitorOrdering(VisitorOrdering.DESCENDING);
+ }
+
+ String remoteCluster = query.properties().getString("visit.dataHandler");
+ if (remoteCluster != null) {
+ params.setRemoteDataHandler(remoteCluster);
+ } else {
+ params.setLocalDataHandler(new HitDataHandler(
+ result, query.properties().getBoolean("populatehitfields", false)));
+ }
+
+ String fieldSet = query.properties().getString("visit.fieldSet");
+ if (fieldSet != null) {
+ params.fieldSet(fieldSet);
+ }
+
+ String continuation = query.properties().getString("visit.continuation");
+ if (continuation != null) {
+ params.setResumeToken(ContinuationHit.getToken(continuation));
+ }
+
+ params.setVisitRemoves(query.properties().getBoolean("visit.visitRemoves"));
+
+ MessagePropertyProcessor.PropertySetter propertySetter;
+ propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest());
+
+ propertySetter.process(params);
+
+ if (context.getClusterList().getStorageClusters().size() == 0) {
+ throw new IllegalArgumentException("No content clusters have been defined");
+ }
+
+ String route = query.properties().getString("visit.cluster");
+ ClusterDef found = null;
+ if (route != null) {
+ String names = "";
+ for (ClusterDef c : context.getClusterList().getStorageClusters()) {
+ if (c.getName().equals(route)) {
+ found = c;
+ }
+ if (!names.isEmpty()) {
+ names += ", ";
+ }
+ names += c.getName();
+ }
+ if (found == null) {
+ throw new IllegalArgumentException("Your vespa cluster contains the storage clusters " + names + ", not " + route + ". Please select a valid vespa cluster.");
+ }
+ } else if (context.getClusterList().getStorageClusters().size() == 1) {
+ found = context.getClusterList().getStorageClusters().get(0);
+ } else {
+ throw new IllegalArgumentException("Multiple content clusters are defined, select one using the \"visit.cluster\" option");
+ }
+
+ params.setRoute("[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]");
+ return params;
+ }
+
+ @Override
+ public Result search(Query query, Execution execution) {
+ Result result = execution.search(query);
+
+ VisitorParameters parameters;
+
+ try {
+ parameters = getVisitorParameters(query, result);
+ } catch (Exception e) {
+ return new Result(query, ErrorMessage.createBadRequest("Illegal parameters: " + e.toString()));
+ }
+
+ if (parameters != null) {
+ VisitorSession session = context.getSessionFactory().createVisitorSession(parameters);
+
+ try {
+ if (!session.waitUntilDone(query.getTimeout())) {
+ return new Result(query, ErrorMessage.createTimeout("Visitor timed out"));
+ }
+
+ ProgressToken token = session.getProgress();
+ if (!token.isFinished()) {
+ final ContinuationHit continuation = new ContinuationHit(token);
+ result.hits().setField(VISITOR_CONTINUATION_TOKEN_FIELDNAME, continuation.getValue());
+ }
+ } catch (InterruptedException e) {
+ } finally {
+ session.destroy();
+ }
+ }
+
+ GetSearcher.setOutputFormat(query, result);
+ query.setHits(result.hits().size());
+ return result;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java
new file mode 100644
index 00000000000..0aeaf714bf3
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.storage.searcher;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java
new file mode 100644
index 00000000000..5f2952931f3
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java
@@ -0,0 +1,326 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.Result;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.vespa.http.client.core.ErrorCode;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL;
+
+/**
+ * An instance of this class handles all requests from one client using VespaHttpClient.
+ *
+ * The implementation is based on the code from V2, but the object model is rewritten to simplify the logic and
+ * avoid using a threadpool that has no effect with all the extra that comes with it. V2 has one instance per thread
+ * on the client, while this is one instance for all threads.
+ */
+class ClientFeederV3 {
+
+ protected static final Logger log = Logger.getLogger(ClientFeederV3.class.getName());
+ // This is for all clients on this gateway, for load balancing from client.
+ private final static AtomicInteger outstandingOperations = new AtomicInteger(0);
+ private final BlockingQueue<OperationStatus> feedReplies = new LinkedBlockingQueue<>();
+ private final ReferencedResource<SharedSourceSession> sourceSession;
+ private final String clientId;
+ private final ReplyHandler feedReplyHandler;
+ private final Metric metric;
+ private Instant prevOpsPerSecTime = Instant.now();
+ private double operationsForOpsPerSec = 0d;
+
+ private final Object monitor = new Object();
+ private final StreamReaderV3 streamReaderV3;
+ private final AtomicInteger ongoingRequests = new AtomicInteger(0);
+ private String hostName;
+ private AtomicInteger threadsAvailableForFeeding;
+
+ ClientFeederV3(
+ ReferencedResource<SharedSourceSession> sourceSession,
+ FeedReaderFactory feedReaderFactory,
+ DocumentTypeManager docTypeManager,
+ String clientId,
+ Metric metric,
+ ReplyHandler feedReplyHandler,
+ AtomicInteger threadsAvailableForFeeding) {
+ this.sourceSession = sourceSession;
+ this.clientId = clientId;
+ this.feedReplyHandler = feedReplyHandler;
+ this.metric = metric;
+ this.threadsAvailableForFeeding = threadsAvailableForFeeding;
+ this.streamReaderV3 = new StreamReaderV3(feedReaderFactory, docTypeManager);
+ try {
+ this.hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ log.info("Could not find hostname, randomizing. " + e.getMessage());
+ hostName = "UNKNOWNHOSTNAME" + System.nanoTime() % 10000;
+ }
+ }
+
+ public boolean timedOut() {
+ synchronized (monitor) {
+ return Instant.now().isAfter(prevOpsPerSecTime.plusSeconds(6000)) && ongoingRequests.get() == 0;
+ }
+ }
+
+ public void kill() {
+ // No new requests should be sent to this object, but there can be old one, even though this is very unlikely.
+ while (ongoingRequests.get() > 0) {
+ try {
+ ongoingRequests.wait(100);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ sourceSession.getReference().close();
+ }
+
+ private void transferPreviousRepliesToResponse(BlockingQueue<OperationStatus> operations) throws InterruptedException {
+ OperationStatus status = feedReplies.poll();
+ while (status != null) {
+ outstandingOperations.decrementAndGet();
+ operations.put(status);
+ status = feedReplies.poll();
+ }
+ }
+
+ public HttpResponse handleRequest(HttpRequest request) throws IOException {
+ threadsAvailableForFeeding.decrementAndGet();
+ ongoingRequests.incrementAndGet();
+ try {
+ InputStream inputStream = StreamReaderV3.unzipStreamIfNeeded(request);
+ final BlockingQueue<OperationStatus> replies = new LinkedBlockingQueue<>();
+ FeederSettings feederSettings = new FeederSettings(request);
+ try {
+ feed(feederSettings, inputStream, replies, threadsAvailableForFeeding);
+ synchronized (monitor) {
+ // Handshake requests do not have DATA_FORMAT, we do not want to give responses to
+ // handshakes as it won't be processed by the client.
+ if (request.getJDiscRequest().headers().get(Headers.DATA_FORMAT) != null) {
+ transferPreviousRepliesToResponse(replies);
+ }
+ }
+ } catch (InterruptedException e) {
+ // NOP, just terminate
+ } catch (Throwable e) {
+ log.log(LogLevel.WARNING, "Unhandled exception while feeding: "
+ + Exceptions.toMessageString(e), e);
+ } finally {
+ try {
+ replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, null));
+ } catch (InterruptedException e) {
+ // NOP, we are already exiting the thread
+ }
+ }
+ return new FeedResponse(200, replies, 3 /* protocol version */, clientId, outstandingOperations.get(), hostName);
+ } finally {
+ ongoingRequests.decrementAndGet();
+ threadsAvailableForFeeding.incrementAndGet();
+ }
+ }
+
+ private Optional<DocumentOperationMessageV3> pullMessageFromRequest(
+ FeederSettings settings, InputStream requestInputStream, BlockingQueue<OperationStatus> repliesFromOldMessages) {
+ while (true) {
+ final Optional<String> operationId;
+ try {
+ operationId = streamReaderV3.getNextOperationId(requestInputStream);
+ } catch (IOException ioe) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe);
+ }
+ return Optional.empty();
+ }
+ if (! operationId.isPresent()) {
+ return Optional.empty();
+ }
+ final DocumentOperationMessageV3 msg;
+ try {
+ msg = getNextMessage(operationId.get(), requestInputStream, settings);
+ } catch (Exception e) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e);
+ }
+ repliesFromOldMessages.add(new OperationStatus(
+ Exceptions.toMessageString(e), operationId.get(), ErrorCode.ERROR, ""));
+
+ continue;
+ }
+ setRoute(msg, settings);
+ return Optional.of(msg);
+ }
+ }
+
+ private Result sendMessage(
+ FeederSettings settings,
+ DocumentOperationMessageV3 msg,
+ AtomicInteger threadsAvailableForFeeding) throws InterruptedException {
+ Result result = null;
+ while (result == null || result.getError().getCode() == SEND_QUEUE_FULL) {
+ msg.getMessage().pushHandler(feedReplyHandler);
+
+ if (settings.denyIfBusy && threadsAvailableForFeeding.get() < 1) {
+ return sourceSession.getResource().sendMessage(msg.getMessage());
+ } else {
+ result = sourceSession.getResource().sendMessageBlocking(msg.getMessage());
+ }
+ if (result.isAccepted()) {
+ return result;
+ }
+ Thread.sleep(100);
+ }
+ return result;
+ }
+
+ private void feed(
+ FeederSettings settings,
+ InputStream requestInputStream,
+ BlockingQueue<OperationStatus> repliesFromOldMessages,
+ AtomicInteger threadsAvailableForFeeding) throws InterruptedException {
+ while (true) {
+
+ Optional<DocumentOperationMessageV3> msg = pullMessageFromRequest(settings, requestInputStream, repliesFromOldMessages);
+
+ if (! msg.isPresent()) {
+ break;
+ }
+ setMessageParameters(msg.get(), settings);
+
+ final Result result;
+ try {
+ result = sendMessage(settings, msg.get(), threadsAvailableForFeeding);
+
+ } catch (RuntimeException e) {
+ repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), Exceptions.toMessageString(e),
+ ErrorCode.ERROR, msg.get().getMessage()));
+ continue;
+ }
+
+ if (result.isAccepted()) {
+ outstandingOperations.incrementAndGet();
+ updateOpsPerSec();
+ log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.get().getOperationId());
+ } else if (!result.getError().isFatal()) {
+ repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(),
+ ErrorCode.TRANSIENT_ERROR, msg.get().getMessage()));
+ continue;
+ } else {
+ // should probably not happen, but everybody knows stuff that
+ // shouldn't happen, happens all the time
+ repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(),
+ ErrorCode.ERROR, msg.get().getMessage()));
+ continue;
+ }
+ }
+ }
+
+ private OperationStatus createOperationStatus(String id, String message, ErrorCode code, Message msg)
+ throws InterruptedException {
+ String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0
+ ? msg.getTrace().toString()
+ : "";
+ return new OperationStatus(message, id, code, traceMessage);
+ }
+
+ // protected for mocking
+ protected DocumentOperationMessageV3 getNextMessage(
+ String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception {
+ VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings);
+
+ // This is a bit hard to set up while testing, so we accept that things are not perfect.
+ if (sourceSession.getResource().session() != null) {
+ metric.set(
+ MetricNames.PENDING,
+ Double.valueOf(sourceSession.getResource().session().getPendingCount()),
+ null);
+ }
+
+ DocumentOperationMessageV3 msg = DocumentOperationMessageV3.create(operation, operationId, metric);
+ if (msg == null) {
+ // typical end of feed
+ return null;
+ }
+ metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/);
+ log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.getOperationId());
+ return msg;
+ }
+
+ private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) {
+ msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies));
+ if (settings.traceLevel != null) {
+ msg.getMessage().getTrace().setLevel(settings.traceLevel);
+ }
+ if (settings.priority != null) {
+ try {
+ DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority);
+ if (msg.getMessage() instanceof DocumentMessage) {
+ ((DocumentMessage) msg.getMessage()).setPriority(priority);
+ }
+ }
+ catch (IllegalArgumentException i) {
+ log.severe(i.getMessage());
+ }
+ }
+ }
+
+ private void setRoute(DocumentOperationMessageV3 msg, FeederSettings settings) {
+ if (settings.route != null) {
+ msg.getMessage().setRoute(settings.route);
+ }
+ }
+
+ protected final void log(LogLevel level, Object... msgParts) {
+ StringBuilder s;
+
+ if (!log.isLoggable(level)) {
+ return;
+ }
+
+ s = new StringBuilder();
+ for (Object part : msgParts) {
+ s.append(part.toString());
+ }
+
+ log.log(level, s.toString());
+ }
+
+ private void updateOpsPerSec() {
+ Instant now = Instant.now();
+ synchronized (monitor) {
+ if (now.plusSeconds(1).isAfter(prevOpsPerSecTime)) {
+ Duration duration = Duration.between(now, prevOpsPerSecTime);
+ final double opsPerSec = operationsForOpsPerSec / (duration.toMillis() / 1000.);
+ metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, null /*metricContext*/);
+ operationsForOpsPerSec = 1.0d;
+ prevOpsPerSecTime = now;
+ } else {
+ operationsForOpsPerSec += 1.0d;
+ }
+ }
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java
new file mode 100644
index 00000000000..240b02fabc7
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * The state of a client session, used to save replies when client disconnects.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ClientState {
+
+ public final int pending;
+ public final long creationTime;
+ public final BlockingQueue<OperationStatus> feedReplies;
+ public final ReferencedResource<SharedSourceSession> sourceSession;
+ public final Metric.Context metricContext;
+
+ public final long prevOpsPerSecTime; // previous measurement time of OPS
+ public final double operationsForOpsPerSec;
+
+ public ClientState(int pending, BlockingQueue<OperationStatus> feedReplies,
+ ReferencedResource<SharedSourceSession> sourceSession, Metric.Context metricContext,
+ long prevOpsPerSecTime, double operationsForOpsPerSec) {
+ super();
+ this.pending = pending;
+ this.feedReplies = feedReplies;
+ this.sourceSession = sourceSession;
+ this.metricContext = metricContext;
+ creationTime = System.currentTimeMillis();
+ this.prevOpsPerSecTime = prevOpsPerSecTime;
+ this.operationsForOpsPerSec = operationsForOpsPerSec;
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java
new file mode 100644
index 00000000000..cd61ae91dae
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.routing.ErrorDirective;
+import com.yahoo.messagebus.routing.Hop;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+import com.yahoo.yolean.Exceptions;
+
+/**
+ * Keeps an operation with its message.
+ *
+ * This implementation is based on V2, but the code is restructured.
+ *
+ * @author dybdahl
+ */
+class DocumentOperationMessageV3 {
+
+ private final String operationId;
+ private final Message message;
+
+ private DocumentOperationMessageV3(String operationId, Message message) {
+ this.operationId = operationId;
+ this.message = message;
+ }
+
+ Message getMessage() {
+ return message;
+ }
+
+ String getOperationId() {
+ return operationId;
+ }
+
+ static DocumentOperationMessageV3 newErrorMessage(String operationId, Exception exception) {
+ Message feedErrorMessageV3 = new FeedErrorMessage(operationId);
+ DocumentOperationMessageV3 msg = new DocumentOperationMessageV3(operationId, feedErrorMessageV3);
+ Hop hop = new Hop();
+ hop.addDirective(new ErrorDirective(Exceptions.toMessageString(exception)));
+ Route route = new Route();
+ route.addHop(hop);
+ feedErrorMessageV3.setRoute(route);
+ return msg;
+ }
+
+ static DocumentOperationMessageV3 newUpdateMessage(VespaXMLFeedReader.Operation op, String operationId) {
+ DocumentUpdate update = op.getDocumentUpdate();
+ update.setCondition(op.getCondition());
+ Message msg = new UpdateDocumentMessage(update);
+
+ String id = (operationId == null) ? update.getId().toString() : operationId;
+ return new DocumentOperationMessageV3(id, msg);
+ }
+
+ static DocumentOperationMessageV3 newRemoveMessage(VespaXMLFeedReader.Operation op, String operationId) {
+ DocumentRemove remove = new DocumentRemove(op.getRemove());
+ remove.setCondition(op.getCondition());
+ Message msg = new RemoveDocumentMessage(remove);
+
+ String id = (operationId == null) ? remove.getId().toString() : operationId;
+ return new DocumentOperationMessageV3(id, msg);
+ }
+
+ static DocumentOperationMessageV3 newPutMessage(VespaXMLFeedReader.Operation op, String operationId) {
+ DocumentPut put = new DocumentPut(op.getDocument());
+ put.setCondition(op.getCondition());
+ Message msg = new PutDocumentMessage(put);
+
+ String id = (operationId == null) ? put.getId().toString() : operationId;
+ return new DocumentOperationMessageV3(id, msg);
+ }
+
+ static DocumentOperationMessageV3 create(VespaXMLFeedReader.Operation operation, String operationId, Metric metric) {
+ switch (operation.getType()) {
+ case DOCUMENT:
+ metric.add(MetricNames.NUM_PUTS, 1, null /*metricContext*/);
+ return newPutMessage(operation, operationId);
+ case REMOVE:
+ metric.add(MetricNames.NUM_REMOVES, 1, null /*metricContext*/);
+ return newRemoveMessage(operation, operationId);
+ case UPDATE:
+ metric.add(MetricNames.NUM_UPDATES, 1, null /*metricContext*/);
+ return newUpdateMessage(operation, operationId);
+ default:
+ // typical end of feed
+ return null;
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java
new file mode 100644
index 00000000000..6f7c08148ed
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.text.Utf8;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class ErrorHttpResponse extends HttpResponse {
+
+ private final String msg;
+
+ public ErrorHttpResponse(final int statusCode, final String msg) {
+ super(statusCode);
+ this.msg = msg;
+ }
+
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ outputStream.write(Utf8.toBytes(msg));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java
new file mode 100644
index 00000000000..45336ea911a
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.messagebus.Message;
+import com.yahoo.text.Utf8String;
+
+import java.util.Arrays;
+
+public class FeedErrorMessage extends Message {
+
+ private long sequenceId;
+
+ public FeedErrorMessage(String operationId) {
+ try {
+ DocumentId id = new DocumentId(operationId);
+ sequenceId = Arrays.hashCode(id.getGlobalId());
+ } catch (Exception e) {
+ sequenceId = 0;
+ }
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return new Utf8String("vespa-feed-handler-internal-bogus-protocol");
+ }
+
+ @Override
+ public int getType() {
+ return 1234;
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return sequenceId;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java
new file mode 100644
index 00000000000..3c532d94b24
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java
@@ -0,0 +1,354 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.http.HttpResponse.Status;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.net.LinuxInetAddress;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Accept feeds from outside of the Vespa cluster.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @since 5.1
+ */
+public class FeedHandler extends LoggingRequestHandler {
+
+ private final ExecutorService workers = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("feedhandler"));
+ private final DocumentTypeManager docTypeManager;
+ private final Map<String, ClientState> clients;
+ private final ScheduledThreadPoolExecutor cron;
+ private final SessionCache sessionCache;
+ protected final ReplyHandler feedReplyHandler;
+ private final AtomicLong sessionId;
+ private final Metric metric;
+ private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(2));
+ private final String localHostname;
+ private final FeedHandlerV3 feedHandlerV3;
+
+ public FeedHandler(
+ Executor executor,
+ DocumentmanagerConfig documentManagerConfig,
+ SessionCache sessionCache,
+ Metric metric,
+ AccessLog accessLog,
+ ThreadpoolConfig threadpoolConfig) throws Exception {
+ super(executor, accessLog);
+ feedHandlerV3 = new FeedHandlerV3(executor, documentManagerConfig, sessionCache, metric, accessLog, threadpoolConfig);
+ docTypeManager = createDocumentManager(documentManagerConfig);
+ clients = new HashMap<>();
+ this.sessionCache = sessionCache;
+ sessionId = new AtomicLong(new Random(System.currentTimeMillis()).nextLong());
+ feedReplyHandler = new FeedReplyReader(metric);
+ cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron"));
+ cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES);
+ this.metric = metric;
+ this.localHostname = resolveLocalHostname();
+ }
+
+ /**
+ * Exposed for creating mocks.
+ */
+ protected DocumentTypeManager createDocumentManager(
+ DocumentmanagerConfig documentManagerConfig) {
+ return new DocumentTypeManager(documentManagerConfig);
+ }
+
+ private class CleanClients implements Runnable {
+
+ @Override
+ public void run() {
+ List<ClientState> clientsToShutdown = new ArrayList<>();
+ long now = System.currentTimeMillis();
+
+ synchronized (clients) {
+ for (Iterator<Map.Entry<String, ClientState>> i = clients
+ .entrySet().iterator(); i.hasNext();) {
+ ClientState client = i.next().getValue();
+
+ if (now - client.creationTime > 10 * 60 * 1000) {
+ clientsToShutdown.add(client);
+ i.remove();
+ }
+ }
+ }
+ for (ClientState client : clientsToShutdown) {
+ client.sourceSession.getReference().close();
+ }
+ }
+ }
+
+ private Tuple2<HttpResponse, Integer> checkProtocolVersion(HttpRequest request) {
+ return doCheckProtocolVersion(request.getJDiscRequest().headers().get(Headers.VERSION));
+ }
+
+ static Tuple2<HttpResponse, Integer> doCheckProtocolVersion(List<String> clientSupportedVersions) {
+ List<String> washedClientVersions = splitVersions(clientSupportedVersions);
+
+ if (washedClientVersions == null || washedClientVersions.isEmpty()) {
+ return new Tuple2<>(new ErrorHttpResponse(
+ Headers.HTTP_NOT_ACCEPTABLE,
+ "Request did not contain " + Headers.VERSION
+ + "header. Server supports protocol versions "
+ + serverSupportedVersions), -1);
+ }
+
+ //select the highest version supported by both parties
+ //this could be extended when we support a gazillion versions - but right now: keep it simple.
+ int version;
+ if (washedClientVersions.contains("3")) {
+ version = 3;
+ } else if (washedClientVersions.contains("2")) {
+ version = 2;
+ } else {
+ return new Tuple2<>(new ErrorHttpResponse(
+ Headers.HTTP_NOT_ACCEPTABLE,
+ "Could not parse " + Headers.VERSION
+ + "header of request (values: " + washedClientVersions +
+ "). Server supports protocol versions "
+ + serverSupportedVersions), -1);
+ }
+ return new Tuple2<>(null, version);
+ }
+
+ private static List<String> splitVersions(List<String> clientSupportedVersions) {
+ List<String> splittedVersions = new ArrayList<>();
+ for (String v : clientSupportedVersions) {
+ if (v == null || v.trim().isEmpty()) {
+ continue;
+ }
+ if (!v.contains(",")) {
+ splittedVersions.add(v.trim());
+ continue;
+ }
+ for (String part : v.split(",")) {
+ part = part.trim();
+ if (!part.isEmpty()) {
+ splittedVersions.add(part);
+ }
+ }
+ }
+ return splittedVersions;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ Tuple2<HttpResponse, Integer> protocolVersion = checkProtocolVersion(request);
+
+ if (protocolVersion.first != null) {
+ return protocolVersion.first;
+ }
+ if (3 == protocolVersion.second) {
+ return feedHandlerV3.handle(request);
+ }
+ final BlockingQueue<OperationStatus> operations = new LinkedBlockingQueue<>();
+ Tuple2<String, Boolean> clientId;
+ clientId = sessionId(request);
+
+ if (clientId.second != null && clientId.second) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received initial request from client with session ID " +
+ clientId.first + ", protocol version " + protocolVersion.second);
+ }
+ }
+
+ final Feeder feeder;
+ try {
+ feeder = createFeeder(
+ request, request.getData(), operations, clientId.first, clientId.second, protocolVersion.second);
+ // the synchronous FeedResponse blocks draining the InputStream, letting
+ // the Feeder read it
+ workers.submit(feeder);
+ } catch (UnknownClientException uce) {
+ final String msg = Exceptions.toMessageString(uce);
+ log.log(LogLevel.WARNING, msg);
+ return new ErrorHttpResponse(Status.BAD_REQUEST, msg);
+ } catch (Exception e) {
+ final String msg = "Could not initialize document parsing: "
+ + Exceptions.toMessageString(e);
+ log.log(LogLevel.WARNING, msg);
+ return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, msg);
+ }
+
+ try {
+ feeder.waitForRequestReceived();
+ } catch (InterruptedException e) {
+ return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ }
+
+ return new FeedResponse(200, operations,
+ protocolVersion.second.intValue(),
+ clientId.first);
+ }
+
+ // Protected for testing
+ protected static InputStream unzipStreamIfNeeded(final InputStream inputStream, final HttpRequest httpRequest)
+ throws IOException {
+ final String contentEncodingHeader = httpRequest.getHeader("content-encoding");
+ if ("gzip".equals(contentEncodingHeader)) {
+ return new GZIPInputStream(inputStream);
+ } else {
+ return inputStream;
+ }
+ }
+
+ /**
+ * Exposed for creating mocks.
+ */
+ protected Feeder createFeeder(
+ HttpRequest request,
+ InputStream requestInputStream,
+ final BlockingQueue<OperationStatus> operations,
+ String clientId,
+ boolean sessionIdWasGeneratedJustNow,
+ int protocolVersion) throws Exception {
+ switch (protocolVersion) {
+ case 2:
+ return new Feeder(
+ unzipStreamIfNeeded(requestInputStream, request),
+ new FeedReaderFactory(),
+ docTypeManager,
+ operations,
+ popClient(clientId),
+ new FeederSettings(request),
+ clientId,
+ sessionIdWasGeneratedJustNow,
+ sourceSessionParams(request),
+ sessionCache,
+ this,
+ metric,
+ feedReplyHandler,
+ localHostname);
+ default:
+ throw new IllegalStateException("BUG! Protocol version " + protocolVersion + " not supported.");
+ }
+ }
+
+ private Tuple2<String, Boolean> sessionId(HttpRequest request) {
+ boolean sessionIdWasGeneratedJustNow = false;
+ String sessionId = request.getHeader(Headers.SESSION_ID);
+ if (sessionId == null) {
+ sessionId = Long.toString(this.sessionId.incrementAndGet()) + "-" +
+ remoteHostAddressAndPort(request.getJDiscRequest()) + "#" +
+ localHostname;
+ sessionIdWasGeneratedJustNow = true;
+ }
+ return new Tuple2<>(sessionId, sessionIdWasGeneratedJustNow);
+ }
+
+ private static String remoteHostAddressAndPort(com.yahoo.jdisc.http.HttpRequest httpRequest) {
+ SocketAddress remoteAddress = httpRequest.getRemoteAddress();
+ if (remoteAddress instanceof InetSocketAddress) {
+ InetSocketAddress isa = (InetSocketAddress) remoteAddress;
+ return isa.getAddress().getHostAddress() + "-" + isa.getPort();
+ }
+ return "";
+ }
+
+ private static String resolveLocalHostname() {
+ try {
+ InetAddress inetAddress = LinuxInetAddress.getLocalHost();
+ String hostname = inetAddress.getCanonicalHostName();
+ if (hostname.equals("localhost")) {
+ return "";
+ }
+ return inetAddress.getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ return "";
+ }
+ }
+
+ /**
+ * Exposed for use when creating mocks.
+ */
+ protected SourceSessionParams sourceSessionParams(HttpRequest request) {
+ SourceSessionParams params = new SourceSessionParams();
+ String timeout = request.getHeader(Headers.TIMEOUT);
+
+ if (timeout != null) {
+ try {
+ params.setTimeout(Double.parseDouble(timeout));
+ } catch (NumberFormatException e) {
+ // NOP
+ }
+ }
+ return params;
+ }
+
+ @Override
+ protected void destroy() {
+ feedHandlerV3.destroy();
+ // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction.
+ // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this
+ // and started destructing something that required something only the messenger thread could provide.
+ Thread destroyer = new Thread(() -> {
+ internalDestroy();
+ });
+ destroyer.setDaemon(true);
+ destroyer.start();
+ }
+
+ private void internalDestroy() {
+ super.destroy();
+ workers.shutdown();
+ cron.shutdown();
+ synchronized (clients) {
+ for (ClientState client : clients.values()) {
+ client.sourceSession.getReference().close();
+ }
+ clients.clear();
+ }
+ }
+
+ void putClient(final String sessionId, final ClientState value) {
+ synchronized (clients) {
+ clients.put(sessionId, value);
+ }
+ }
+
+ ClientState popClient(String sessionId) {
+ synchronized (clients) {
+ return clients.remove(sessionId);
+ }
+ }
+
+ /**
+ * Guess what, testing only.
+ */
+ void forceRunCleanClients() {
+ new CleanClients().run();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java
new file mode 100644
index 00000000000..4694563ecbe
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java
@@ -0,0 +1,172 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.yolean.Exceptions;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+/**
+ * This code is based on v2 code, however, in v3, one client has one ClientFeederV3 shared between all client threads.
+ * The new API has more logic for shutting down cleanly as the server is more likely to be upgraded.
+ * The code is restructured a bit.
+ *
+ * @author dybdahl
+ */
+public class FeedHandlerV3 extends LoggingRequestHandler {
+
+ private DocumentTypeManager docTypeManager;
+ private final Map<String, ClientFeederV3> clientFeederByClientId = new HashMap<>();
+ private final ScheduledThreadPoolExecutor cron;
+ private final SessionCache sessionCache;
+ protected final ReplyHandler feedReplyHandler;
+ private final Metric metric;
+ private final Object monitor = new Object();
+ private final AtomicInteger threadsAvailableForFeeding;
+ private static final Logger log = Logger.getLogger(FeedHandlerV3.class.getName());
+
+ public FeedHandlerV3(
+ Executor executor,
+ DocumentmanagerConfig documentManagerConfig,
+ SessionCache sessionCache,
+ Metric metric,
+ AccessLog accessLog,
+ ThreadpoolConfig threadpoolConfig) throws Exception {
+ super(executor, accessLog);
+ docTypeManager = new DocumentTypeManager(documentManagerConfig);
+ this.sessionCache = sessionCache;
+ feedReplyHandler = new FeedReplyReader(metric);
+ cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron"));
+ cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES);
+ this.metric = metric;
+ // Half of the threads can be blocking on feeding before we deny requests.
+ if (threadpoolConfig != null) {
+ threadsAvailableForFeeding = new AtomicInteger(threadpoolConfig.maxthreads() / 2);
+ } else {
+ log.warning("No config for threadpool, using 250 for max blocking threads for feeding.");
+ threadsAvailableForFeeding = new AtomicInteger(250);
+ }
+ }
+
+ public void injectDocumentManangerForTests(DocumentTypeManager docTypeManager) {
+ this.docTypeManager = docTypeManager;
+ }
+
+ // TODO: If this is set up to run without first invoking the old FeedHandler code, we should
+ // verify the version header first. This is done in the old code.
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ final String clientId = clientId(request);
+ final ClientFeederV3 clientFeederV3;
+ synchronized (monitor) {
+ if (! clientFeederByClientId.containsKey(clientId)) {
+ SourceSessionParams sourceSessionParams = sourceSessionParams(request);
+ clientFeederByClientId.put(
+ clientId,
+ new ClientFeederV3(
+ retainSource(sessionCache, sourceSessionParams),
+ new FeedReaderFactory(),
+ docTypeManager,
+ clientId,
+ metric,
+ feedReplyHandler,
+ threadsAvailableForFeeding));
+ }
+ clientFeederV3 = clientFeederByClientId.get(clientId);
+ }
+ try {
+ return clientFeederV3.handleRequest(request);
+ } catch (UnknownClientException uce) {
+ final String msg = Exceptions.toMessageString(uce);
+ log.log(LogLevel.WARNING, msg);
+ return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST, msg);
+ } catch (Exception e) {
+ final String msg = "Could not initialize document parsing: "
+ + Exceptions.toMessageString(e);
+ log.log(LogLevel.WARNING, msg);
+ return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.INTERNAL_SERVER_ERROR, msg);
+ }
+ }
+
+ // SessionCache is final and no easy way to mock it so we need this to be able to do testing.
+ protected ReferencedResource<SharedSourceSession> retainSource(SessionCache sessionCache, SourceSessionParams params) {
+ return sessionCache.retainSource(params);
+ }
+
+ @Override
+ protected void destroy() {
+ // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction.
+ // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this
+ // and started destructing something that required something only the messenger thread could provide.
+ Thread destroyer = new Thread(() -> {
+ super.destroy();
+ cron.shutdown();
+ synchronized (monitor) {
+ for (ClientFeederV3 client : clientFeederByClientId.values()) {
+ client.kill();
+ }
+ clientFeederByClientId.clear();
+ }
+ });
+ destroyer.setDaemon(true);
+ destroyer.start();
+ }
+
+ private String clientId(HttpRequest request) {
+ String clientDictatedId = request.getHeader(Headers.CLIENT_ID);
+ if (clientDictatedId == null || clientDictatedId.isEmpty()) {
+ throw new IllegalArgumentException("Did not get any CLIENT_ID header (" + Headers.CLIENT_ID + ")");
+ }
+ return clientDictatedId;
+ }
+
+ private SourceSessionParams sourceSessionParams(HttpRequest request) {
+ SourceSessionParams params = new SourceSessionParams();
+ String timeout = request.getHeader(Headers.TIMEOUT);
+
+ if (timeout != null) {
+ try {
+ params.setTimeout(Double.parseDouble(timeout));
+ } catch (NumberFormatException e) {
+ // NOP
+ }
+ }
+ return params;
+ }
+
+ private void removeOldClients() {
+ synchronized (monitor) {
+ for (Iterator<Map.Entry<String, ClientFeederV3>> iterator = clientFeederByClientId
+ .entrySet().iterator(); iterator.hasNext();) {
+ ClientFeederV3 client = iterator.next().getValue();
+ if (client.timedOut()) {
+ client.kill();
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java
new file mode 100644
index 00000000000..67d6acd926a
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.json.JsonFeedReader;
+import com.yahoo.vespa.http.client.config.FeedParams;
+import com.yahoo.vespaxmlparser.FeedReader;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.io.InputStream;
+
+/**
+ * Class for creating FeedReader based on dataFormat.
+ * @author dybdahl
+ */
+public class FeedReaderFactory {
+
+ /**
+ * Creates FeedReader
+ * @param inputStream source of feed data
+ * @param docTypeManager handles the parsing of the document
+ * @param dataFormat specifies the format
+ * @return a feedreader
+ */
+ public FeedReader createReader(
+ InputStream inputStream,
+ DocumentTypeManager docTypeManager,
+ FeedParams.DataFormat dataFormat) {
+ switch (dataFormat) {
+ case XML_UTF8:
+ try {
+ return new VespaXMLFeedReader(inputStream, docTypeManager);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not create VespaXMLFeedReader", e);
+ }
+ case JSON_UTF8:
+ return new JsonFeedReader(inputStream, docTypeManager);
+ default:
+ throw new IllegalStateException("Can not create feed reader for format: " + dataFormat);
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java
new file mode 100644
index 00000000000..b7ecf4211a9
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import java.util.logging.Logger;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.Trace;
+import com.yahoo.vespa.http.client.core.ErrorCode;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+/**
+ * Catch message bus replies and make the available to a given session.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class FeedReplyReader implements ReplyHandler {
+
+ private static final Logger log = Logger.getLogger(FeedReplyReader.class.getName());
+ private final Metric metric;
+
+ public FeedReplyReader(Metric metric) {
+ this.metric = metric;
+ }
+
+ @Override
+ public void handleReply(Reply reply) {
+ Object o = reply.getContext();
+ if (!(o instanceof ReplyContext)) {
+ return;
+ }
+ ReplyContext context = (ReplyContext) o;
+ metric.set(
+ MetricNames.LATENCY,
+ Double.valueOf((System.currentTimeMillis() - context.creationTime) / 1000.0d),
+ null);
+ if (reply.hasErrors()) {
+ metric.add(MetricNames.FAILED, 1, null);
+ enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, reply.getTrace());
+ } else {
+ metric.add(MetricNames.SUCCEEDED, 1, null);
+ enqueue(context, "Document processed.", ErrorCode.OK, reply.getTrace());
+ }
+ }
+
+ private void enqueue(ReplyContext context, String message, ErrorCode status, Trace trace) {
+ try {
+ String traceMessage = (trace != null && trace.getLevel() > 0) ? trace.toString() : "";
+ context.feedReplies.put(new OperationStatus(message,
+ context.docId.toString(), status, traceMessage));
+ } catch (InterruptedException e) {
+ log.log(LogLevel.WARNING,
+ "Interrupted while enqueueing result from putting document with id: "
+ + context.docId.toString());
+ Thread.currentThread().interrupt();
+ }
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java
new file mode 100644
index 00000000000..c6ad51242dc
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.ErrorCode;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Reads feed responses from a queue and renders them continuously to the
+ * feeder.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @since 5.1
+ */
+public class FeedResponse extends HttpResponse {
+
+ BlockingQueue<OperationStatus> operations;
+
+ public FeedResponse(
+ int status,
+ BlockingQueue<OperationStatus> operations,
+ int protocolVersion,
+ String sessionId) {
+ super(status);
+ this.operations = operations;
+ headers().add(Headers.SESSION_ID, sessionId);
+ headers().add(Headers.VERSION, Integer.toString(protocolVersion));
+ }
+
+ // This is used by the V3 protocol.
+ public FeedResponse(
+ int status,
+ BlockingQueue<OperationStatus> operations,
+ int protocolVersion,
+ String sessionId,
+ int outstandingClientOperations,
+ String hostName) {
+ super(status);
+ this.operations = operations;
+ headers().add(Headers.SESSION_ID, sessionId);
+ headers().add(Headers.VERSION, Integer.toString(protocolVersion));
+ headers().add(Headers.OUTSTANDING_REQUESTS, Integer.toString(outstandingClientOperations));
+ headers().add(Headers.HOSTNAME, hostName);
+ }
+
+ @Override
+ public void render(OutputStream output) throws IOException {
+ int i = 0;
+ OperationStatus status;
+ try {
+ status = operations.take();
+ while (status.errorCode != ErrorCode.END_OF_FEED) {
+ output.write(toBytes(status.render()));
+ if (++i % 5 == 0) {
+ output.flush();
+ }
+ status = operations.take();
+ }
+ } catch (InterruptedException e) {
+ output.flush();
+ }
+ }
+
+ private byte[] toBytes(String s) {
+ byte[] b = new byte[s.length()];
+ for (int i = 0; i < b.length; ++i) {
+ b[i] = (byte) s.charAt(i); // renderSingleStatus ensures ASCII only
+ }
+ return b;
+ }
+
+ @Override
+ public String getContentType() {
+ return "text/plain";
+ }
+
+ @Override
+ public String getCharacterEncoding() {
+ return StandardCharsets.US_ASCII.name();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java
new file mode 100644
index 00000000000..106afe6ffe9
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java
@@ -0,0 +1,542 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.Result;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.routing.ErrorDirective;
+import com.yahoo.messagebus.routing.Hop;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.text.Utf8String;
+import com.yahoo.vespa.http.client.core.Encoder;
+import com.yahoo.vespa.http.client.core.ErrorCode;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespa.http.server.util.ByteLimitedInputStream;
+import com.yahoo.vespaxmlparser.FeedReader;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL;
+
+/**
+ * Read documents from client, and send them through message bus.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Feeder implements Runnable {
+
+ protected static final Logger log = Logger.getLogger(Feeder.class.getName());
+
+ final InputStream requestInputStream;
+ final DocumentTypeManager docTypeManager;
+ final BlockingQueue<OperationStatus> operations;
+ final BlockingQueue<OperationStatus> feedReplies;
+ int numPending;
+ final FeederSettings settings;
+ final String clientId;
+ final ReferencedResource<SharedSourceSession> sourceSession;
+ final FeedHandler handler;
+ final Metric metric;
+ final Metric.Context metricContext;
+ private long prevOpsPerSecTime; // previous measurement time of OPS
+ private double operationsForOpsPerSec;
+ private final ReplyHandler feedReplyHandler;
+ protected final static String EOF = "End of stream";
+ protected final boolean sessionIdWasGeneratedJustNow;
+ private final CountDownLatch requestReceived = new CountDownLatch(1);
+ private final FeedReaderFactory feedReaderFactory;
+
+ // TODO refactor this perverse pile of constructor arguments
+ public Feeder(InputStream requestInputStream,
+ FeedReaderFactory feedReaderFactory,
+ DocumentTypeManager docTypeManager,
+ BlockingQueue<OperationStatus> operations,
+ ClientState storedState,
+ FeederSettings settings,
+ String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams,
+ SessionCache sessionCache,
+ FeedHandler handler, Metric metric, ReplyHandler feedReplyHandler,
+ String localHostname) throws Exception {
+ super();
+ this.feedReaderFactory = feedReaderFactory;
+ if (storedState == null) {
+ if (!sessionIdWasGeneratedJustNow) {
+ // We do not have a stored state, BUT the session ID came in with the request.
+ // Possible session timeout, server restart, server reconfig, or VIP usage. Examine.
+ examineClientId(clientId, localHostname);
+ }
+ numPending = 0;
+ feedReplies = new LinkedBlockingQueue<>();
+ sourceSession = retainSession(sessionParams, sessionCache);
+ metricContext = createClientMetricContext(metric, clientId);
+ prevOpsPerSecTime = System.currentTimeMillis();
+ operationsForOpsPerSec = 0.0;
+ } else {
+ //we have a stored state, and the session ID was obviously not generated now. All OK.
+ numPending = storedState.pending;
+ feedReplies = storedState.feedReplies;
+ sourceSession = storedState.sourceSession;
+ metricContext = storedState.metricContext;
+ prevOpsPerSecTime = storedState.prevOpsPerSecTime;
+ operationsForOpsPerSec = storedState.operationsForOpsPerSec;
+ }
+ this.clientId = clientId;
+ this.sessionIdWasGeneratedJustNow = sessionIdWasGeneratedJustNow;
+ this.requestInputStream = requestInputStream;
+ this.docTypeManager = docTypeManager;
+ this.operations = operations;
+ this.settings = settings;
+ this.handler = handler;
+ this.metric = metric;
+ this.feedReplyHandler = feedReplyHandler;
+ }
+ protected void examineClientId(String clientId, String localHostname) {
+ if (!clientId.contains("#")) {
+ throw new UnknownClientException("Got request from client with id '" + clientId +
+ "', but found no session for this client. " +
+ "Most probably this server is in VIP rotation, " +
+ "and a client session was rotated from one server to another. " +
+ "This must not happen. Configure VIP with persistence=enabled, " +
+ "or (preferably) do not use a VIP at all.");
+ }
+ int hashPos = clientId.indexOf("#");
+ String supposedHostname = clientId.substring(hashPos + 1, clientId.length());
+ if (supposedHostname.isEmpty()) {
+ throw new UnknownClientException("Got request from client with id '" + clientId +
+ "', but found no session for this client. Possible session " +
+ "timeout due to inactivity, server restart or reconfig, " +
+ "or bad VIP usage.");
+ }
+
+ if (!supposedHostname.equals(localHostname)) {
+ throw new UnknownClientException("Got request from client with id '" + clientId +
+ "', but found no session for this client. " +
+ "Session was originally established towards host " +
+ supposedHostname + ", but our hostname is " +
+ localHostname + ". " +
+ "Most probably this server is in VIP rotation, " +
+ "and a session was rotated from one server to another. " +
+ "This should not happen. Configure VIP with persistence=enabled, " +
+ "or (preferably) do not use a VIP at all.");
+ }
+ log.log(LogLevel.DEBUG, "Client '" + clientId + "' reconnected after session inactivity, or server restart " +
+ "or reconfig. Re-establishing session.");
+ }
+
+
+
+ private static Metric.Context createClientMetricContext(Metric metric, String clientId) {
+ // No real value in separate metric dimensions per client.
+ return null;
+ }
+
+ /**
+ * Exposed for creating mocks.
+ */
+ protected ReferencedResource<SharedSourceSession> retainSession(
+ SourceSessionParams sessionParams, SessionCache sessionCache) {
+ return sessionCache.retainSource(sessionParams);
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (handshake()) {
+ return; //will putClient in finally block below
+ }
+ flushResponseQueue();
+ feed();
+ requestReceived.countDown();
+ drain();
+ } catch (InterruptedException e) {
+ // NOP, just terminate
+ } catch (Exception e) {
+ log.log(LogLevel.WARNING, "Unhandled exception while feeding: "
+ + Exceptions.toMessageString(e), e);
+ } catch (Throwable e) {
+ log.log(LogLevel.WARNING, "Unhandled error while feeding: "
+ + Exceptions.toMessageString(e), e);
+ throw e;
+ } finally {
+ requestReceived.countDown();
+ putClient();
+ try {
+ enqueue("-", "-", ErrorCode.END_OF_FEED, null);
+ } catch (InterruptedException e) {
+ // NOP, we are already exiting the thread
+ }
+ }
+ }
+
+ protected boolean handshake() throws IOException {
+ if (sessionIdWasGeneratedJustNow) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Handshake completed for client " + clientId + ".");
+ }
+ requestInputStream.close();
+ return true;
+ }
+ return false;
+ }
+
+ void feed() throws InterruptedException {
+ while (true) {
+ Result result;
+ String operationId;
+ try {
+ operationId = getNextOperationId();
+ } catch (IOException ioe) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe);
+ }
+ break;
+ }
+
+ //noinspection StringEquality
+ if (operationId == EOF) {
+ break;
+ }
+
+ Tuple2<String, Message> msg;
+ try {
+ msg = getNextMessage(operationId);
+ setRoute(msg);
+ } catch (Exception e) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e);
+ }
+ //noinspection StringEquality
+ if (operationId != null) { //v1 always returns null, all others return something useful, or throw an exception above
+ msg = newErrorMessage(operationId, e);
+ } else {
+ break;
+ }
+ }
+
+ if (msg == null) {
+ break;
+ }
+
+ setMessageParameters(msg);
+
+ while (true) {
+ try {
+ msg.second.pushHandler(feedReplyHandler);
+ if (settings.denyIfBusy) {
+ result = sourceSession.getResource().sendMessage(msg.second);
+ } else {
+ result = sourceSession.getResource().sendMessageBlocking(msg.second);
+ }
+ } catch (RuntimeException e) {
+ enqueue(msg.first, Exceptions.toMessageString(e),
+ ErrorCode.ERROR, msg.second);
+ return;
+ }
+ if (result.isAccepted() || result.getError().getCode() != SEND_QUEUE_FULL) {
+ break;
+ }
+ if (settings.denyIfBusy) {
+ break;
+ } else {
+ //This will never happen
+ Thread.sleep(100);
+ }
+ }
+
+ if (result.isAccepted()) {
+ ++numPending;
+ updateMetrics(msg.second);
+ updateOpsPerSec();
+ log(LogLevel.DEBUG, "Sent message successfully, document id: ",
+ msg.first);
+ } else if (!result.getError().isFatal()) {
+ enqueue(msg.first, result.getError().getMessage(),
+ ErrorCode.TRANSIENT_ERROR, msg.second);
+ break;
+ } else {
+ // should probably not happen, but everybody knows stuff that
+ // shouldn't happen, happens all the time
+ enqueue(msg.first, result.getError().getMessage(),
+ ErrorCode.ERROR, msg.second);
+ break;
+ }
+ }
+ }
+
+ private Tuple2<String, Message> newErrorMessage(String operationId, Exception e) {
+ Message m = new FeedErrorMessage(operationId);
+ Tuple2<String, Message> msg = new Tuple2<>(operationId, m);
+ Hop hop = new Hop();
+ hop.addDirective(new ErrorDirective(Exceptions.toMessageString(e)));
+ Route route = new Route();
+ route.addHop(hop);
+ m.setRoute(route);
+ return msg;
+ }
+
+ private void updateMetrics(Message m) {
+ metric.set(
+ MetricNames.PENDING,
+ Double.valueOf(sourceSession.getResource().session().getPendingCount()),
+ null);
+
+ metric.add(MetricNames.NUM_OPERATIONS, 1, metricContext);
+
+ if (m instanceof PutDocumentMessage) {
+ metric.add(MetricNames.NUM_PUTS, 1, metricContext);
+ } else if (m instanceof RemoveDocumentMessage) {
+ metric.add(MetricNames.NUM_REMOVES, 1, metricContext);
+ } else if (m instanceof UpdateDocumentMessage) {
+ metric.add(MetricNames.NUM_UPDATES, 1, metricContext);
+ }
+ }
+
+ private void updateOpsPerSec() {
+ long now = System.currentTimeMillis();
+ if ((now - prevOpsPerSecTime) >= 1000) { //every second
+ double ms = (double) (now - prevOpsPerSecTime);
+ final double opsPerSec = operationsForOpsPerSec / (ms / 1000);
+ metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, metricContext);
+ operationsForOpsPerSec = 1.0d;
+ prevOpsPerSecTime = now;
+ } else {
+ operationsForOpsPerSec += 1.0d;
+ }
+ }
+
+ private Tuple2<String, Message> getNextMessage(String operationId) throws Exception {
+ VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+ Tuple2<String, Message> msg;
+
+ getNextOperation(op);
+
+ switch (op.getType()) {
+ case DOCUMENT:
+ msg = newPutMessage(op, operationId);
+ break;
+ case REMOVE:
+ msg = newRemoveMessage(op, operationId);
+ break;
+ case UPDATE:
+ msg = newUpdateMessage(op, operationId);
+ break;
+ default:
+ // typical end of feed
+ return null;
+ }
+ log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.first);
+ return msg;
+ }
+
+ private void setMessageParameters(Tuple2<String, Message> msg) {
+ msg.second.setContext(new ReplyContext(msg.first, feedReplies));
+ if (settings.traceLevel != null) {
+ msg.second.getTrace().setLevel(settings.traceLevel);
+ }
+ if (settings.priority != null) {
+ try {
+ DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority);
+ if (msg.second instanceof DocumentMessage) {
+ ((DocumentMessage) msg.second).setPriority(priority);
+ }
+ }
+ catch (IllegalArgumentException i) {
+ log.severe(i.getMessage());
+ }
+ }
+ }
+
+ private void setRoute(Tuple2<String, Message> msg) {
+ if (settings.route != null) {
+ msg.second.setRoute(settings.route);
+ }
+ }
+
+ protected void getNextOperation(VespaXMLFeedReader.Operation op) throws Exception {
+ int length = readByteLength();
+
+ try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){
+ FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat);
+ reader.read(op);
+ }
+ }
+
+ protected String getNextOperationId() throws IOException {
+ return readOperationId();
+ }
+
+ private String readOperationId() throws IOException {
+ StringBuilder idBuf = new StringBuilder(100);
+ int c;
+ while ((c = requestInputStream.read()) != -1) {
+ if (c == 32) {
+ break;
+ }
+ idBuf.append((char) c); //it's ASCII
+ }
+ if (idBuf.length() == 0) {
+ return EOF;
+ }
+ return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString();
+ }
+
+ private int readByteLength() throws IOException {
+ StringBuilder lenBuf = new StringBuilder(8);
+ int c;
+ while ((c = requestInputStream.read()) != -1) {
+ if (c == 10) {
+ break;
+ }
+ lenBuf.append((char) c); //it's ASCII
+ }
+ if (lenBuf.length() == 0) {
+ throw new IllegalStateException("Operation length missing.");
+ }
+ return Integer.valueOf(lenBuf.toString(), 16);
+ }
+
+ protected final void log(LogLevel level, Object... msgParts) {
+ StringBuilder s;
+
+ if (!log.isLoggable(level)) {
+ return;
+ }
+
+ s = new StringBuilder();
+ for (Object part : msgParts) {
+ s.append(part.toString());
+ }
+
+ log.log(level, s.toString());
+ }
+
+ private Tuple2<String, Message> newUpdateMessage(Operation op, String operationId) {
+ DocumentUpdate update = op.getDocumentUpdate();
+ update.setCondition(op.getCondition());
+ Message msg = new UpdateDocumentMessage(update);
+
+ String id = (operationId == null) ? update.getId().toString() : operationId;
+ return new Tuple2<>(id, msg);
+ }
+
+ private Tuple2<String, Message> newRemoveMessage(Operation op, String operationId) {
+ DocumentRemove remove = new DocumentRemove(op.getRemove());
+ remove.setCondition(op.getCondition());
+ Message msg = new RemoveDocumentMessage(remove);
+
+ String id = (operationId == null) ? remove.getId().toString() : operationId;
+ return new Tuple2<>(id, msg);
+ }
+
+ private Tuple2<String, Message> newPutMessage(Operation op, String operationId) {
+ DocumentPut put = new DocumentPut(op.getDocument());
+ put.setCondition(op.getCondition());
+ Message msg = new PutDocumentMessage(put);
+
+ String id = (operationId == null) ? put.getId().toString() : operationId;
+ return new Tuple2<>(id, msg);
+ }
+
+
+ void flushResponseQueue() throws InterruptedException {
+ OperationStatus status = feedReplies.poll();
+ while (status != null) {
+ decreasePending(status);
+ status = feedReplies.poll();
+ }
+ }
+
+ void putClient() {
+ handler.putClient(clientId,
+ new ClientState(numPending,
+ feedReplies, sourceSession, metricContext,
+ prevOpsPerSecTime, operationsForOpsPerSec));
+ }
+
+ void drain() throws InterruptedException {
+ if (settings.drain) {
+ while (numPending > 0) {
+ OperationStatus o = feedReplies.take();
+ decreasePending(o);
+ }
+ }
+ }
+
+ private void decreasePending(OperationStatus o) throws InterruptedException {
+ operations.put(o);
+ --numPending;
+ }
+
+ private void enqueue(String id, String message, ErrorCode code, Message msg)
+ throws InterruptedException {
+ String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0
+ ? msg.getTrace().toString()
+ : "";
+ operations.put(new OperationStatus(message, id, code, traceMessage));
+ }
+
+ public void waitForRequestReceived() throws InterruptedException {
+ requestReceived.await(1, TimeUnit.HOURS);
+ }
+
+ public class FeedErrorMessage extends Message {
+ private long sequenceId;
+
+ private FeedErrorMessage(String operationId) {
+ try {
+ DocumentId id = new DocumentId(operationId);
+ sequenceId = Arrays.hashCode(id.getGlobalId());
+ } catch (Exception e) {
+ sequenceId = 0;
+ }
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return new Utf8String("vespa-feed-handler-internal-bogus-protocol");
+ }
+
+ @Override
+ public int getType() {
+ return 1234;
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return sequenceId;
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java
new file mode 100644
index 00000000000..64bc687d425
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.vespa.http.client.config.FeedParams.DataFormat;
+import com.yahoo.vespa.http.client.core.Headers;
+
+/**
+ * Wrapper for the feed feederSettings read from HTTP request.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class FeederSettings {
+
+ private static final Route DEFAULT_ROUTE = Route.parse("default");
+ public final boolean drain;
+ public final Route route;
+ public final boolean denyIfBusy;
+ public final DataFormat dataFormat;
+ public final String priority;
+ public final Integer traceLevel;
+
+ public FeederSettings(HttpRequest request) {
+ {
+ String tmpDrain = request.getHeader(Headers.DRAIN);
+ if (tmpDrain != null) {
+ drain = Boolean.parseBoolean(tmpDrain);
+ } else {
+ drain = false;
+ }
+ }
+ {
+ String tmpRoute = request.getHeader(Headers.ROUTE);
+ if (tmpRoute != null) {
+ route = Route.parse(tmpRoute);
+ } else {
+ route = DEFAULT_ROUTE;
+ }
+ }
+ {
+ String tmpDenyIfBusy = request.getHeader(Headers.DENY_IF_BUSY);
+ if (tmpDenyIfBusy != null) {
+ denyIfBusy = Boolean.parseBoolean(tmpDenyIfBusy);
+ } else {
+ denyIfBusy = false;
+ }
+ }
+ {
+ String tmpDataFormat = request.getHeader(Headers.DATA_FORMAT);
+ if (tmpDataFormat != null) {
+ dataFormat = DataFormat.valueOf(tmpDataFormat);
+ } else {
+ dataFormat = DataFormat.XML_UTF8;
+ }
+ }
+ {
+ String tmpDataFormat = request.getHeader(Headers.PRIORITY);
+ if (tmpDataFormat != null) {
+ priority = tmpDataFormat;
+ } else {
+ priority = null;
+ }
+ }
+ {
+ String tmpDataFormat = request.getHeader(Headers.TRACE_LEVEL);
+ if (tmpDataFormat != null) {
+ traceLevel = Integer.valueOf(tmpDataFormat);
+ } else {
+ traceLevel = null;
+ }
+ }
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
new file mode 100644
index 00000000000..087cb11c350
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+/**
+ * Place to store the metric names so where the metrics are logged can be found
+ * more easily in an IDE.
+ *
+ * @author steinar
+ */
+public final class MetricNames {
+
+ private static final String PREFIX = "httpapi_";
+
+ public static final String NUM_OPERATIONS = PREFIX + "num_operations";
+ public static final String NUM_PUTS = PREFIX + "num_puts";
+ public static final String NUM_REMOVES = PREFIX + "num_removes";
+ public static final String NUM_UPDATES = PREFIX + "num_updates";
+ public static final String OPERATIONS_PER_SEC = PREFIX + "ops_per_sec";
+ public static final String LATENCY = PREFIX + "latency";
+ public static final String FAILED = PREFIX + "failed";
+ public static final String SUCCEEDED = PREFIX + "succeeded";
+ public static final String PENDING = PREFIX + "pending";
+
+ private MetricNames() {
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java
new file mode 100644
index 00000000000..30d8b04131d
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Mapping between document ID and client session.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ReplyContext {
+
+ public final String docId;
+ public final BlockingQueue<OperationStatus> feedReplies;
+ public final long creationTime;
+
+ public ReplyContext(String docId, BlockingQueue<OperationStatus> feedReplies) {
+ this.docId = docId;
+ this.feedReplies = feedReplies;
+ this.creationTime = System.currentTimeMillis();
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java
new file mode 100644
index 00000000000..0453d41fab8
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.vespa.http.client.core.Encoder;
+import com.yahoo.vespa.http.server.util.ByteLimitedInputStream;
+import com.yahoo.vespaxmlparser.FeedReader;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * This code is based on v2 code, but restructured so stream reading code is in one dedicated class.
+ * @author dybdahl
+ */
+public class StreamReaderV3 {
+
+ protected static final Logger log = Logger.getLogger(StreamReaderV3.class.getName());
+
+ private final FeedReaderFactory feedReaderFactory;
+ private final DocumentTypeManager docTypeManager;
+
+ public StreamReaderV3(FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager) {
+ this.feedReaderFactory = feedReaderFactory;
+ this.docTypeManager = docTypeManager;
+ }
+
+ public VespaXMLFeedReader.Operation getNextOperation(
+ InputStream requestInputStream, FeederSettings settings) throws Exception {
+ VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+
+ int length = readByteLength(requestInputStream);
+
+ try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){
+ FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat);
+ reader.read(op);
+ }
+ return op;
+ }
+
+ public Optional<String> getNextOperationId(InputStream requestInputStream) throws IOException {
+ StringBuilder idBuf = new StringBuilder(100);
+ int c;
+ while ((c = requestInputStream.read()) != -1) {
+ if (c == 32) {
+ break;
+ }
+ idBuf.append((char) c); //it's ASCII
+ }
+ if (idBuf.length() == 0) {
+ return Optional.empty();
+ }
+ return Optional.of(Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString());
+ }
+
+ private int readByteLength(InputStream requestInputStream) throws IOException {
+ StringBuilder lenBuf = new StringBuilder(8);
+ int c;
+ while ((c = requestInputStream.read()) != -1) {
+ if (c == 10) {
+ break;
+ }
+ lenBuf.append((char) c); //it's ASCII
+ }
+ if (lenBuf.length() == 0) {
+ throw new IllegalStateException("Operation length missing.");
+ }
+ return Integer.valueOf(lenBuf.toString(), 16);
+ }
+
+ public static InputStream unzipStreamIfNeeded(final HttpRequest httpRequest)
+ throws IOException {
+ final String contentEncodingHeader = httpRequest.getHeader("content-encoding");
+ if ("gzip".equals(contentEncodingHeader)) {
+ return new GZIPInputStream(httpRequest.getData());
+ } else {
+ return httpRequest.getData();
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java
new file mode 100644
index 00000000000..ce605a82dbc
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.5.0
+ */
+public class UnknownClientException extends RuntimeException {
+
+ public UnknownClientException(String message) {
+ super(message);
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java
new file mode 100644
index 00000000000..1707069871b
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Server side of programmatic API for feeding into Vespa from outside of the
+ * clusters. Not a public API, not meant for direct use.
+ */
+@com.yahoo.api.annotations.PackageMarker
+package com.yahoo.vespa.http.server;
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java
new file mode 100644
index 00000000000..a720611c91e
--- /dev/null
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ *
+ * @since 5.1.23
+ */
+public class ByteLimitedInputStream extends InputStream {
+
+ private final InputStream wrappedStream;
+ private int remaining;
+
+ public ByteLimitedInputStream(InputStream wrappedStream, int limit) {
+ this.wrappedStream = wrappedStream;
+ if (limit < 0) {
+ throw new IllegalArgumentException("limit cannot be 0");
+ }
+ this.remaining = limit;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ int retval = wrappedStream.read();
+ if (retval < 0) {
+ remaining = 0;
+ } else {
+ --remaining;
+ }
+ return retval;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+
+ if (remaining <= 0) {
+ return -1;
+ }
+
+ int bytesToRead = Math.min(remaining, len);
+ int retval = wrappedStream.read(b, off, bytesToRead);
+
+ if (retval < 0) {
+ //end of underlying stream was reached, and nothing was read.
+ remaining = 0;
+ } else {
+ remaining -= retval;
+ }
+ return retval;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return remaining;
+ }
+
+ @Override
+ public void close() throws IOException {
+ //we will never close the underlying stream
+ if (remaining <= 0) {
+ return;
+ }
+ while (remaining > 0) {
+ skip(remaining);
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/application/services.xml b/vespaclient-container-plugin/src/test/application/services.xml
new file mode 100644
index 00000000000..df178e109c3
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/application/services.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<jdisc version="1.0" jetty="true">
+
+ <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test">
+ <binding>http://*/document/v1/*</binding>
+ </handler>
+
+ <component id="injected" class="com.yahoo.document.restapi.resource.MockedOperationHandler" bundle="integration-test">
+ </component>
+
+
+ <http>
+ <!-- This indicates that we want JDisc to allocate a port for us -->
+ <server id="mainServer" port="0" />
+ </http>
+</jdisc>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg
new file mode 100644
index 00000000000..6878ddffb79
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg
@@ -0,0 +1,113 @@
+enablecompression false
+datatype[10]
+datatype[0].id 1002
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1004
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1016
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1001
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[0]
+datatype[5].id 2001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].documenttype[0]
+datatype[6].id 2002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -628990518
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name news.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[6]
+datatype[7].structtype[0].field[0].name url
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 10
+datatype[7].structtype[0].field[1].name title
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name last_downloaded
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 0
+datatype[7].structtype[0].field[3].name value_long
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 4
+datatype[7].structtype[0].field[4].name value_content
+datatype[7].structtype[0].field[4].id[0]
+datatype[7].structtype[0].field[4].datatype 3
+datatype[7].structtype[0].field[5].name stringarr
+datatype[7].structtype[0].field[5].id[0]
+datatype[7].structtype[0].field[5].datatype 1002
+datatype[7].documenttype[0]
+datatype[8].id 538588767
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name news.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[7]
+datatype[8].structtype[0].field[0].name intarr
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 1000
+datatype[8].structtype[0].field[1].name longarr
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype 1004
+datatype[8].structtype[0].field[2].name bytearr
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype 1016
+datatype[8].structtype[0].field[3].name floatarr
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 1001
+datatype[8].structtype[0].field[4].name weightedsetint
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype 2001
+datatype[8].structtype[0].field[5].name weightedsetstring
+datatype[8].structtype[0].field[5].id[0]
+datatype[8].structtype[0].field[5].datatype 2002
+datatype[8].structtype[0].field[6].name content
+datatype[8].structtype[0].field[6].id[0]
+datatype[8].structtype[0].field[6].datatype 3
+datatype[8].documenttype[0]
+datatype[9].id -1048827947
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name news
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -628990518
+datatype[9].documenttype[0].bodystruct 538588767
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml
new file mode 100644
index 00000000000..52739246b1f
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+ Document : test10.xml
+ Created on : July 27, 2007, 11:37 AM
+ Author : alimf
+ Description:
+ this feed contains both documents, updates and removes.
+-->
+
+<vespafeed>
+
+ <document documenttype="news" documentid="doc:news:http://news10a">
+ <url>testUrl</url>
+ <title>testTitle</title>
+ <last_downloaded>1</last_downloaded>
+ <value_long>2</value_long>
+ <value_content>testValueContent</value_content>
+ <stringarr>
+ <item>stringarrItem1</item>
+ <item>stringarrItem2</item>
+ </stringarr>
+ <intarr>
+ <item>3</item>
+ <item>4</item>
+ </intarr>
+ <longarr>
+ <item>5</item>
+ <item>6</item>
+ </longarr>
+ <bytearr>
+ <item>7</item>
+ <item>8</item>
+ </bytearr>
+ <floatarr>
+ <item>9</item>
+ <item>10</item>
+ </floatarr>
+ <weightedsetint>
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </weightedsetint>
+ <weightedsetstring>
+ <item weight="13">string13</item>
+ <item weight="14">string14</item>
+ </weightedsetstring>
+ </document>
+
+ <document documenttype="news" documentid="doc:news:http://news10b">
+ <url>testUrl2</url>
+ </document>
+
+ <update documenttype="news" documentid="doc:news:http://news10c">
+ <add field="stringarr">
+ <item>addString1</item>
+ <item>addString2</item>
+ </add>
+ <add field="longarr">
+ <item>5</item>
+ </add>
+ <add field="longarr">6</add>
+ <add field="weightedsetint">
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </add>
+ <add field="weightedsetstring">
+ <item>add13</item>
+ </add>
+ <add field="weightedsetstring">add14</add>
+ </update>
+
+ <update documenttype="news" documentid="doc:news:http://news10d">
+ <assign field="url">assignUrl</assign>
+ <assign field="value_long">2</assign>
+ <assign field="stringarr">
+ <item>assignString1</item>
+ <item>assignString2</item>
+ </assign>
+ <assign field="intarr">
+ <item>3</item>
+ <item>4</item>
+ </assign>
+ <assign field="weightedsetint">
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </assign>
+ </update>
+
+ <remove documentid="doc:news:http://news10e"/>
+</vespafeed>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml
new file mode 100644
index 00000000000..44762f594e0
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+ Document : test10.xml
+ Created on : July 27, 2007, 11:37 AM
+ Author : alimf
+ Description:
+ this feed contains both documents, updates and removes.
+-->
+
+<vespafeed>
+
+ <document documenttype="news" documentid="doc:news:http://news10a">
+ <url>testUrl</url>
+ <title>testTitle</title>
+ <last_downloaded>1</last_downloaded>
+ <value_long>2</value_long>
+ <value_content>testValueContent</value_content>
+ <stringarr>
+ <item>stringarrItem1</item>
+ <item>stringarrItem2</item>
+ </stringarr>
+ <intarr>
+ <item>3</item>
+ <item>4</item>
+ </intarr>
+ <longarr>
+ <item>5</item>
+ <item>6</item>
+ </longarr>
+ <bytearr>
+ <item>7</item>
+ <item>8</item>
+ </bytearr>
+ <floatarr>
+ <item>9</item>
+ <item>10</item>
+ </floatarr>
+ <weightedsetint>
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </weightedsetint>
+ <weightedsetstring>
+ <item weight="13">string13</item>
+ <item weight="14">string14</item>
+ </weightedsetstring>
+ </document>
+
+ <document documenttype="news" documentid="doc:news:http://news10b">
+ <url>testUrl2</url>
+ </document>
+</vespafeed>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml
new file mode 100644
index 00000000000..a62df0c645a
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+ <document documenttype="news" documentid="doc:news:http://news10a">
+ <url>testUrl</url>
+ <title>testTitle</title>
+ <last_downloaded>1</last_downloaded>
+ <value_long>2</value_long>
+ <value_content>testValueContent</value_content>
+ <stringarr>
+ <item>stringarrItem1</item>
+ <item>stringarrItem2</item>
+ </stringarr>
+ <intarr>
+ <item>3</item>
+ <item>4</item>
+ </intarr>
+ <longarr>
+ <item>5</item>
+ <item>6</item>
+ </longarr>
+ <bytearr>
+ <item>7</item>
+ <item>8</item>
+ </bytearr>
+ <floatarr>
+ <item>9</item>
+ <item>10</item>
+ </floatarr>
+ <weightedsetint>
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </weightedsetint>
+ <weightedsetstring>
+ <item weight="13">string13</item>
+ <item weight="14">string14</item>
+ </weightedsetstring>
+ </document>
+
+ <document documenttype="news" documentid="foobar:news:http://news12">
+ <url>testUrl2</url>
+ </document>
+
+ <document documenttype="news" documentid="doc:news:ok">
+ <url>testUrl2</url>
+ </document>
+</vespafeed>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml
new file mode 100755
index 00000000000..434eaa7789c
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+ <document documenttype="news" documentid="foobar:news:http://news12">
+ <url>testUrl2</url>
+ </document>
+
+ <document documenttype="news" documentid="doc:news:http://news10a">
+ <url>testUrl</url>
+ <title>testTitle</title>
+ <last_downloaded>1</last_downloaded>
+ <value_long>2</value_long>
+ <value_content>testValueContent</value_content>
+ <stringarr>
+ <item>stringarrItem1</item>
+ <item>stringarrItem2</item>
+ </stringarr>
+ <intarr>
+ <item>3</item>
+ <item>4</item>
+ </intarr>
+ <longarr>
+ <item>5</item>
+ <item>6</item>
+ </longarr>
+ <bytearr>
+ <item>7</item>
+ <item>8</item>
+ </bytearr>
+ <floatarr>
+ <item>9</item>
+ <item>10</item>
+ </floatarr>
+ <weightedsetint>
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </weightedsetint>
+ <weightedsetstring>
+ <item weight="13">string13</item>
+ <item weight="14">string14</item>
+ </weightedsetstring>
+ </document>
+</vespafeed>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml
new file mode 100755
index 00000000000..f0b36ea3bc9
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+ <document documenttype="news" documentid="doc:news:http://news10a">
+ <url>testUrl</url>
+ <title>testTitle</title>
+ <last_downloaded>1</last_downloaded>
+ <value_long>2</value_long>
+ <value_content>testValueContent</value_content>
+ <stringarr>
+ <item>stringarrItem1</item>
+ <item>stringarrItem2</item>
+ </stringarr>
+ <intarr>
+ <item>3</item>
+ <item>4</item>
+ </intarr>
+ <longarr>
+ <item>5</item>
+ <item>6</item>
+ </longarr>
+ <bytearr>
+ <item>7</item>
+ <item>8</item>
+ </bytearr>
+ <floatarr>
+ <item>9</item>
+ <item>10</item>
+ </floatarr>
+ <weightedsetint>
+ <item weight="11">11</item>
+ <item weight="12">12</item>
+ </weightedsetint>
+ <weightedsetstring>
+ <item weight="13">string13</item>
+ <item weight="14">string14</item>
+ </weightedsetstring>
+ </document>
+
+ <document documenttype=news documentid=foobar:news:http://news12
+ <url>testUrl2</url>
+ </document>
+</vespafed>
diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_removes b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes
new file mode 100755
index 00000000000..0232970ed4f
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes
@@ -0,0 +1,2 @@
+doc:test:remove1
+doc:test:remove2
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java
new file mode 100644
index 00000000000..87360fcf998
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import com.yahoo.vespaclient.ClusterDef;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.*;
+
+
+public class OperationHandlerImplTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingClusterDef() throws RestApiException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ OperationHandlerImpl.resolveClusterRoute(Optional.empty(), clusterDef);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingClusterDefSpecifiedCluster() throws RestApiException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef);
+ }
+
+ @Test(expected = RestApiException.class)
+ public void oneClusterPresentNotMatching() throws RestApiException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ clusterDef.add(new ClusterDef("foo", "configId"));
+ OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef);
+ }
+
+ @Test()
+ public void oneClusterMatching() throws RestApiException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ clusterDef.add(new ClusterDef("foo", "configId"));
+ assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef),
+ is("[Storage:cluster=foo;clusterconfigid=configId]"));
+ }
+
+ @Test()
+ public void oneClusterMatchingManyAvailable() throws RestApiException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ clusterDef.add(new ClusterDef("foo2", "configId2"));
+ clusterDef.add(new ClusterDef("foo", "configId"));
+ clusterDef.add(new ClusterDef("foo3", "configId2"));
+ assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef),
+ is("[Storage:cluster=foo;clusterconfigid=configId]"));
+ }
+
+ @Test()
+ public void checkErrorMessage() throws RestApiException, IOException {
+ List<ClusterDef> clusterDef = new ArrayList<>();
+ clusterDef.add(new ClusterDef("foo2", "configId2"));
+ clusterDef.add(new ClusterDef("foo", "configId"));
+ clusterDef.add(new ClusterDef("foo3", "configId2"));
+ try {
+ OperationHandlerImpl.resolveClusterRoute(Optional.of("wrong"), clusterDef);
+ } catch(RestApiException e) {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ e.getResponse().render(stream);
+ String errorMsg = new String( stream.toByteArray());
+ assertThat(errorMsg, is("{\"errors\":[\"Your vespa cluster contains the content clusters foo2 " +
+ "(configId2), foo (configId), foo3 (configId2), not wrong. Please select a valid vespa cluster.\"]}"));
+ return;
+ }
+ fail("Expected exception");
+ }
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java
new file mode 100644
index 00000000000..4bb7a264aba
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+public class RestUriTest {
+
+ URI createUri(String path, String query) throws URISyntaxException {
+ return new URIBuilder()
+ .addParameter("foo", "bar")
+ .setHost("host")
+ .setScheme("http")
+ .setPort(666)
+ .setPath(path)
+ .setCustomQuery(query)
+ .setFragment("fargment").build();
+ }
+
+ @Rule
+ public ExpectedException thrown= ExpectedException.none();
+
+ @Test
+ public void testBasic() throws Exception {
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid", "query"));
+ assertThat(restUri.getDocId(), is("myid"));
+ assertThat(restUri.getDocumentType(), is("doctype"));
+ assertThat(restUri.getNamespace(), is("namespace"));
+ assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty()));
+ assertThat(restUri.generateFullId(), is("id:namespace:doctype::myid"));
+ }
+
+ @Test
+ public void encodingSlashes() throws Exception {
+ // Try with slashes encoded.
+ final String id = " !\"øæåp/:;&,.:;'1Q";
+ String encodedId = URLEncoder.encode(id, StandardCharsets.UTF_8.name());
+ RestUri restUri = new RestUri(URI.create("/document/v1/namespace/doctype/docid/" + encodedId));
+ assertThat(restUri.getDocId(), is(id));
+ assertThat(restUri.getDocumentType(), is("doctype"));
+ assertThat(restUri.getNamespace(), is("namespace"));
+ assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty()));
+ assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id));
+ }
+
+ @Test
+ public void encodingSlashes2() throws Exception {
+ // This will decode the slashes.
+ final String id = " !\"øæåp/:;&,.:;'1Q ";
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/" + id, "query"));
+ assertThat(restUri.getDocId(), is(id));
+ assertThat(restUri.getDocumentType(), is("doctype"));
+ assertThat(restUri.getNamespace(), is("namespace"));
+ assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty()));
+ assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id));
+ }
+
+
+ @Test
+ public void testVisit() throws Exception {
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/", "query"));
+ assertThat(restUri.getDocId(), is(""));
+ assertThat(restUri.getDocumentType(), is("doctype"));
+ assertThat(restUri.getNamespace(), is("namespace"));
+ assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty()));
+ assertThat(restUri.generateFullId(), is("id:namespace:doctype::"));
+ }
+
+ @Test
+ public void testOneSlashTooMuchWhichIsFine() throws Exception {
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid:342:23/wrong", ""));
+ assertThat(restUri.getDocId(), is("myid:342:23/wrong"));
+ }
+
+ @Test
+ public void testGroupG() throws Exception {
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group/myid", ""));
+ assertThat(restUri.getDocId(), is("myid"));
+ assertThat(restUri.getDocumentType(), is("doctype"));
+ assertThat(restUri.getGroup().get().name, is('g'));
+ assertThat(restUri.getGroup().get().value, is("group"));
+ assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group:myid"));
+ }
+
+ @Test
+ public void testGroupN() throws Exception {
+ RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", ""));
+ assertThat(restUri.getGroup().get().name, is('n'));
+ assertThat(restUri.getGroup().get().value, is("group"));
+ }
+
+ @Test
+ public void testGroupUnknown() throws Exception {
+ thrown.expect(RestApiException.class);
+ new RestUri(createUri("/document/v1/namespace/doctype/Q/myid", ""));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java
new file mode 100644
index 00000000000..ab5b36c74d1
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi.resource;
+
+import com.yahoo.document.restapi.OperationHandler;
+import com.yahoo.document.restapi.Response;
+import com.yahoo.document.restapi.RestApiException;
+import com.yahoo.document.restapi.RestUri;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.util.Optional;
+
+/**
+ * Mock that collects info about operation and returns them on second delete.
+ */
+public class MockedOperationHandler implements OperationHandler {
+
+ StringBuilder log = new StringBuilder();
+ int deleteCount = 0;
+
+ @Override
+ public VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException {
+ return new VisitResult(Optional.of("token"), "List of json docs, cont token " + continuation.map(a->a).orElse("not set") + ", doc selection: '"
+ + documentSelection + "'");
+ }
+
+ @Override
+ public void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException {
+ log.append("PUT: " + data.getDocument().getId());
+ log.append(data.getDocument().getBody().toString());
+ }
+
+ @Override
+ public void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException {
+ log.append("UPDATE: " + data.getDocumentUpdate().getId());
+ log.append(data.getDocumentUpdate().getFieldUpdates().toString());
+ if (data.getDocumentUpdate().getCreateIfNonExistent()) {
+ log.append("[CREATE IF NON EXISTENT IS TRUE]");
+ }
+ }
+
+ @Override
+ public void delete(RestUri restUri, String condition) throws RestApiException {
+ deleteCount++;
+ if (deleteCount == 2) {
+ String theLog = log.toString();
+ log = new StringBuilder();
+ deleteCount = 0;
+ throw new RestApiException(Response.createErrorResponse(666, theLog));
+ }
+ log.append("DELETE: " + restUri.generateFullId());
+ }
+
+ @Override
+ public Optional<String> get(RestUri restUri) throws RestApiException {
+ log.append("GET: " + restUri.generateFullId());
+ return Optional.empty();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java
new file mode 100644
index 00000000000..17a36c142ea
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi.resource;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.document.restapi.OperationHandler;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+
+public class RestApiMaxThreadTest {
+ final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicInteger requestsInFlight = new AtomicInteger(0);
+ private class RestApiMocked extends RestApi {
+
+ public RestApiMocked() {
+ super(mock(Executor.class), null, (OperationHandler)null);
+ }
+
+ @Override
+ protected HttpResponse handleInternal(HttpRequest request) {
+ requestsInFlight.incrementAndGet();
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+ }
+
+ @Test
+ public void testCallsAreThrottled() throws InterruptedException {
+ RestApiMocked restApiMocked = new RestApiMocked();
+ // Fire lots of requests.
+ for (int x = 0; x < 30; x++) {
+ new Thread(() -> restApiMocked.handle(null)).start();
+ }
+ // Wait for all threads to be used
+ while (requestsInFlight.get() != 19) {
+ Thread.sleep(1);
+ }
+ // A new request should be blocked.
+ final HttpResponse response = restApiMocked.handle(null);
+ assertThat(response.getStatus(), is(429));
+ latch.countDown();
+ }
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java
new file mode 100644
index 00000000000..036dc63ad34
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java
@@ -0,0 +1,298 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi.resource;
+
+import com.yahoo.application.Application;
+import com.yahoo.application.Networking;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.container.Container;
+import com.yahoo.jdisc.http.server.jetty.JettyHttpServer;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.core.StringStartsWith.startsWith;
+import static org.junit.Assert.assertThat;
+
+public class RestApiTest {
+ Application application;
+
+ @Before
+ public void setup() throws Exception {
+ application = Application.fromApplicationPackage(Paths.get("src/test/application"), Networking.enable);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ application.close();
+ }
+
+ String post_test_uri = "/document/v1/namespace/testdocument/docid/c";
+ String post_test_doc = "{\n" +
+ "\"foo\" : \"bar\"," +
+ "\"fields\": {\n" +
+ "\"title\": \"This is the title\",\n" +
+ "\"body\": \"This is the body\"" +
+ "}" +
+ "}";
+ String post_test_response = "{\"id\":\"id:namespace:testdocument::c\"," +
+ "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}";
+
+ // Run this test to manually do request against the REST-API with backend mock.
+ @Ignore
+ @Test
+ public void blockingTest() throws Exception {
+ System.out.println("Running on port " + getFirstListenPort());
+ Thread.sleep(Integer.MAX_VALUE);
+ }
+
+ @Test
+ public void testbasicPost() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri);
+ HttpPost httpPost = new HttpPost(request.getUri());
+ StringEntity entity = new StringEntity(post_test_doc, ContentType.create("application/json"));
+ httpPost.setEntity(entity);
+ String x = doRest(httpPost);
+ assertThat(x, is(post_test_response));
+ }
+
+ String post_test_uri_cond = "/document/v1/namespace/testdocument/docid/c?condition=foo";
+ String post_test_doc_cond = "{\n" +
+ "\"foo\" : \"bar\"," +
+ "\"fields\": {\n" +
+ "\"title\": \"This is the title\",\n" +
+ "\"body\": \"This is the body\"" +
+ "}" +
+ "}";
+ String post_test_response_cond = "{\"id\":\"id:namespace:testdocument::c\"," +
+ "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}";
+
+ @Test
+ public void testConditionalPost() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri_cond);
+ HttpPost httpPost = new HttpPost(request.getUri());
+ StringEntity entity = new StringEntity(post_test_doc_cond, ContentType.create("application/json"));
+ httpPost.setEntity(entity);
+ String x = doRest(httpPost);
+ assertThat(x, is(post_test_response_cond));
+ }
+
+ String post_test_empty_response = "{\"errors\":[\"Could not read document, no document?\"]";
+ @Test
+ public void testEmptyPost() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri);
+ HttpPost httpPost = new HttpPost(request.getUri());
+ StringEntity entity = new StringEntity("", ContentType.create("application/json"));
+ httpPost.setEntity(entity);
+ String x = doRest(httpPost);
+ assertThat(x, startsWith(post_test_empty_response));
+ }
+
+ String update_test_uri = "/document/v1/namespace/testdocument/docid/c";
+ String update_test_doc = "{\n" +
+ "\t\"fields\": {\n" +
+ "\"title\": {\n" +
+ "\"assign\": \"Oh lala\"\n" +
+ "}\n" +
+ "}\n" +
+ "}\n";
+
+ String update_test_response = "{\"id\":\"id:namespace:testdocument::c\"," +
+ "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}";
+
+ @Test
+ public void testbasicUpdate() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri);
+ HttpPut httpPut = new HttpPut(request.getUri());
+ StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json"));
+ httpPut.setEntity(entity);
+ assertThat(doRest(httpPut), is(update_test_response));
+ assertThat(getLog(), not(containsString("CREATE IF NON EXISTING IS TRUE")));
+ }
+
+ @Test
+ public void testbasicUpdateCreateTrue() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri + "?create=true");
+ HttpPut httpPut = new HttpPut(request.getUri());
+ StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json"));
+ httpPut.setEntity(entity);
+ assertThat(doRest(httpPut), is(update_test_response));
+ assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE"));
+ }
+
+ String update_test_create_if_non_existient_uri = "/document/v1/namespace/testdocument/docid/c";
+ String update_test_create_if_non_existient_doc = "{\n" +
+ "\"create\":true," +
+ "\t\"fields\": {\n" +
+ "\"title\": {\n" +
+ "\"assign\": \"Oh lala\"\n" +
+ "}\n" +
+ "}\n" +
+ "}\n";
+
+ String update_test_create_if_non_existing_response = "{\"id\":\"id:namespace:testdocument::c\"," +
+ "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}";
+
+ @Test
+ public void testCreateIfNonExistingUpdateInDocTrue() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri);
+ HttpPut httpPut = new HttpPut(request.getUri());
+ StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json"));
+ httpPut.setEntity(entity);
+ assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response));
+ assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE"));
+
+ }
+
+ @Test
+ public void testCreateIfNonExistingUpdateInDocTrueButQueryParamsFalse() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri + "?create=false");
+ HttpPut httpPut = new HttpPut(request.getUri());
+ StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json"));
+ httpPut.setEntity(entity);
+ assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response));
+ assertThat(getLog(), not(containsString("CREATE IF NON EXISTENT IS TRUE")));
+
+ }
+
+ // Get logs through some hackish fetch method. Logs is something the mocked backend write.
+ String getLog() throws IOException {
+ // The mocked backend will throw a runtime exception wtih a log if delete is called three times..
+ Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri);
+ HttpDelete delete = new HttpDelete(request.getUri());
+ doRest(delete);
+ return doRest(delete);
+ }
+
+
+ String remove_test_uri = "/document/v1/namespace/testdocument/docid/c";
+ String remove_test_response = "{\"id\":\"id:namespace:testdocument::c\"," +
+ "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}";
+
+ @Test
+ public void testbasicRemove() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri);
+ HttpDelete delete = new HttpDelete(request.getUri());
+ assertThat(doRest(delete), is(remove_test_response));
+ }
+
+ String get_test_uri = "/document/v1/namespace/document-type/docid/c";
+ String get_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/c\"";
+ String get_response_part2 = "\"id\":\"id:namespace:document-type::c\"";
+
+
+ @Test
+ public void testbasicGet() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + get_test_uri);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(get_response_part1));
+ assertThat(rest, containsString(get_response_part2));
+ }
+
+ String id_test_uri = "/document/v1/namespace/document-type/docid/f/u/n/n/y/!";
+ String id_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/f/u/n/n/y/!\"";
+ String id_response_part2 = "\"id\":\"id:namespace:document-type::f/u/n/n/y/!\"";
+
+ @Test
+ public void testSlashesInId() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + id_test_uri);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(id_response_part1));
+ assertThat(rest, containsString(id_response_part2));
+ }
+
+
+ String get_enc_id = "!\":æøå@/& Q1+";
+ // Space encoded as %20, not encoding !
+ String get_enc_id_encoded_v1 = "!%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26%20Q1%2B";
+ // Space encoded as +
+ String get_enc_id_encoded_v2 = "%21%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26+Q1%2B";
+ String get_enc_test_uri_v1 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1;
+ String get_enc_test_uri_v2 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2;
+ String get_enc_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1 + "\"";
+ String get_enc_response_part1_v2 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2 + "\"";
+
+ // JSON encode " as \"
+ String get_enc_response_part2 = "\"id\":\"id:namespace:document-type::" + get_enc_id.replace("\"", "\\\"") + "\"";
+
+
+ @Test
+ public void testbasicEncodingV1() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v1);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(get_enc_response_part1));
+ assertThat(rest, containsString(get_enc_response_part2));
+ }
+
+ @Test
+ public void testbasicEncodingV2() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v2);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(get_enc_response_part1_v2));
+ assertThat(rest, containsString(get_enc_response_part2));
+ }
+
+ String visit_test_uri = "/document/v1/namespace/document-type/docid/?continuation=abc";
+ String visit_response_part1 = "\"documents\":[List of json docs, cont token abc, doc selection: '']";
+ String visit_response_part2 = "\"continuation\":\"token\"";
+ String visit_response_part3 = "\"pathId\":\"/document/v1/namespace/document-type/docid/\"";
+
+
+ @Test
+ public void testbasicVisit() throws Exception {
+ Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_uri);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(visit_response_part1));
+ assertThat(rest, containsString(visit_response_part2));
+ assertThat(rest, containsString(visit_response_part3));
+ }
+
+ String visit_test_bad_uri = "/document/v1/namespace/document-type/group/abc?continuation=abc";
+ String visit_test_bad_response = "Visiting does not support setting value for group/value,";
+
+
+ @Test
+ public void testBadVisit() throws Exception {
+ final Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_bad_uri);
+ HttpGet get = new HttpGet(request.getUri());
+ final String rest = doRest(get);
+ assertThat(rest, containsString(visit_test_bad_response));
+ }
+
+ private String doRest(HttpRequestBase request) throws IOException {
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpResponse response = client.execute(request);
+ HttpEntity entity = response.getEntity();
+ return EntityUtils.toString(entity);
+ }
+
+ private String getFirstListenPort() {
+ JettyHttpServer serverProvider =
+ (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0);
+ return Integer.toString(serverProvider.getListenPort());
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java
new file mode 100644
index 00000000000..cfb120d9891
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.restapi.resource;
+
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.restapi.OperationHandler;
+
+import java.util.concurrent.Executor;
+
+/**
+ * For setting up RestApi with a simple document type manager.
+ *
+ * @author dybdahl
+ */
+public class RestApiWithTestDocumentHandler extends RestApi{
+
+ private DocumentTypeManager docTypeManager = new DocumentTypeManager();
+
+ public RestApiWithTestDocumentHandler(
+ Executor executor,
+ AccessLog accessLog,
+ OperationHandler operationHandler) {
+ super(executor, accessLog, operationHandler);
+
+ DocumentType documentType = new DocumentType("testdocument");
+
+ documentType.addField("title", DataType.STRING);
+ documentType.addField("body", DataType.STRING);
+ docTypeManager.registerDocumentType(documentType);
+
+ setDocTypeManagerForTests(docTypeManager);
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java
new file mode 100644
index 00000000000..6d0ee59c68d
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.jdisc.HeaderFields;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespa.http.server.FeedHandler;
+import com.yahoo.vespa.http.server.Feeder;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit test for FeedHandler class.
+ *
+ * @author dybdahl
+ */
+public class FeedHandlerTest {
+
+ /**
+ * This class extends FeedHandler and allows to create a custom Feeder.
+ */
+ static class TestFeedHandler extends FeedHandler {
+ private final CountDownLatch countDownLatch = new CountDownLatch(1);
+
+ public TestFeedHandler() throws Exception {
+ super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null);
+ }
+
+ /**
+ * Builds a feeder that blocks until countDownLatch is stepped down.
+ */
+ @Override
+ protected Feeder createFeeder(
+ com.yahoo.container.jdisc.HttpRequest request,
+ InputStream requestInputStream,
+ final BlockingQueue<OperationStatus> operations,
+ String clientId,
+ boolean sessionIdWasGeneratedJustNow,
+ int protocolVersion) throws Exception {
+ Feeder feeder = mock(Feeder.class);
+ doAnswer(invocation -> {
+ try {
+ countDownLatch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }).when(feeder).waitForRequestReceived();
+ return feeder;
+ }
+ }
+
+ /**
+ * nginx require that a post is finished before the server ack with a response. This behaviour is verified
+ * in this test
+ */
+ @Test
+ public void testResponseIsSentAfterWaitForRequestReceivedReturns() throws Exception {
+ HttpRequest request = mock(HttpRequest.class);
+
+ // Create a request with valid version.
+ com.yahoo.jdisc.http.HttpRequest jdiscRequest = mock(com.yahoo.jdisc.http.HttpRequest.class);
+ HeaderFields headerFields = mock(HeaderFields.class);
+ List<String> version = new ArrayList<>();
+ version.add("2");
+ when(headerFields.get(Headers.VERSION)).thenReturn(version);
+ when(jdiscRequest.headers()).thenReturn(headerFields);
+ when(request.getJDiscRequest()).thenReturn(jdiscRequest);
+
+ TestFeedHandler feedHandler = new TestFeedHandler();
+ // After a short period, make the feed finish.
+ new Thread(() -> {
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ feedHandler.countDownLatch.countDown();
+ }).start();
+ // This should not return before countdown latch is stepped down.
+ feedHandler.handle(request);
+ // This should always returns after the countDownLatch has become zero. This might cause false positive,
+ // but not false negatives. This is fine.
+ assertThat(feedHandler.countDownLatch.getCount(), is(0L));
+
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java
new file mode 100755
index 00000000000..646bcb805f6
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java
@@ -0,0 +1,1015 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.docproc.CallStack;
+import com.yahoo.jdisc.HeaderFields;
+import com.yahoo.messagebus.*;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.container.Container;
+import com.yahoo.docproc.*;
+import com.yahoo.docproc.jdisc.DocumentProcessingHandler;
+import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters;
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.jdisc.handler.RequestHandler;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import com.yahoo.vespaclient.config.FeederConfig;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.logging.Logger;
+import java.util.zip.GZIPOutputStream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class VespaFeedHandlerTestCase {
+
+ private VespaFeedHandler feedHandler;
+ private VespaFeedHandlerRemove removeHandler;
+ private VespaFeedHandlerStatus statusHandler;
+ private VespaFeedHandlerRemoveLocation removeLocationHandler;
+ private FeedContext context;
+
+ private DummySessionFactory factory;
+ private final String xmlFilesPath = "src/test/files/feedhandler/";
+
+ public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg,
+ boolean autoReply,
+ DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception {
+ DocumentTypeManager docMan = new DocumentTypeManager();
+ DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg");
+
+ if (autoReply) {
+ if (autoReplyFactory != null) {
+ factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory);
+ } else {
+ factory = DummySessionFactory.createWithErrorAutoReply(e);
+ }
+ } else {
+ factory = DummySessionFactory.createDefault();
+ }
+
+ context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric());
+
+ Executor threadPool = Executors.newCachedThreadPool();
+ feedHandler = new VespaFeedHandler(context, threadPool);
+ removeHandler = new VespaFeedHandlerRemove(context, threadPool);
+ statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool);
+ removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool);
+
+ CallStack dpCallstack = new CallStack("bar");
+ dpCallstack.addLast(new TestDocProc());
+ dpCallstack.addLast(new TestLaterDocProc());
+
+ DocprocService myservice = new DocprocService("bar");
+ myservice.setCallStack(dpCallstack);
+ myservice.setInService(true);
+
+ ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>();
+ registry.register(new ComponentId(myservice.getName()), myservice);
+
+ DocumentProcessingHandler handler = new DocumentProcessingHandler(registry,
+ new ComponentRegistry<>(), new ComponentRegistry<>(),
+ new DocumentProcessingHandlerParameters());
+
+ Container container = Container.get();
+ ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>();
+ requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler);
+ container.setRequestHandlerRegistry(requestHandlerComponentRegistry);
+ }
+
+ public void setup(com.yahoo.messagebus.Error e) throws Exception {
+ setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null);
+ }
+
+ public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception {
+ setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory);
+ }
+
+ public void setup() throws Exception {
+ setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null);
+ }
+
+ @After
+ public void resetContainer() {
+ Container.resetInstance();
+ }
+
+
+ @Test
+ public void testLoadTypes() throws Exception {
+ List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>();
+ typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW"));
+ typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3"));
+
+ setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null);
+
+ {
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:removeme", d.toString());
+ assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType());
+ assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority());
+ }
+ }
+
+ @Test
+ public void testPostXML() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10b", d.toString());
+ }
+
+ assertTrue(res.output.contains("count=\"2\""));
+ assertTrue(res.error == null);
+ }
+
+ @Test
+ public void testPostXMLAsync() throws Exception {
+ setup();
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10b", d.toString());
+ }
+
+ // Should not have metrics at this point.
+ assertTrue(!res.output.contains("count=\"2\""));
+ assertTrue(res.error == null);
+ }
+
+
+ @Test
+ public void testPostGZIPedXML() throws Exception {
+ setup(null);
+ Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10b", d.toString());
+ }
+
+ assertTrue(res.error == null);
+ }
+
+ @Test
+ public void testDocProc() throws Exception {
+ setup(null);
+
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument();
+
+ assertEquals("doc:news:http://news10a", d.getId().toString());
+ assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded"));
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument();
+
+ assertEquals("doc:news:http://news10b", d.getId().toString());
+ assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded"));
+ }
+ }
+
+ @Test
+ public void testPostXMLVariousTypes() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test10.xml", "feed?");
+
+ assertEquals(5, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10b", d.toString());
+ }
+
+ {
+ Message m = res.messages.get(2);
+ assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType());
+ DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId();
+ assertEquals("doc:news:http://news10c", d.toString());
+ }
+ {
+ Message m = res.messages.get(3);
+ assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType());
+ DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId();
+ assertEquals("doc:news:http://news10d", d.toString());
+ }
+ {
+ Message m = res.messages.get(4);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:news:http://news10e", d.toString());
+ }
+
+ String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
+
+ assertEquals("<result>\n" +
+ "\n" +
+ " <route name=\"default\">\n" +
+ " <total>\n" +
+ " <latency count=\"5\"/>\n" +
+ " <count count=\"5\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"1\"/>\n" +
+ " <count count=\"1\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ "\n" +
+ "</result>\n", val);
+ }
+
+ @Test
+ public void testStatusPage() throws Exception {
+ setup(null);
+
+ testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
+ testFeed(xmlFilesPath + "test10.xml", "feed?");
+ testFeed(xmlFilesPath + "test10.xml", "feed?route=storage");
+ testFeed(xmlFilesPath + "test_removes", "remove?");
+
+ assertEquals(2, factory.sessionsCreated());
+ Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+
+ String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
+ val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\"");
+
+ assertEquals("<status>\n" +
+ "\n" +
+ " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" +
+ " <routes>\n" +
+ " <route name=\"total\">\n" +
+ " <total>\n" +
+ " <latency count=\"14\"/>\n" +
+ " <count count=\"14\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"6\"/>\n" +
+ " <count count=\"6\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " <route name=\"default\">\n" +
+ " <total>\n" +
+ " <latency count=\"9\"/>\n" +
+ " <count count=\"9\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"3\"/>\n" +
+ " <count count=\"3\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " <route name=\"storage\">\n" +
+ " <total>\n" +
+ " <latency count=\"5\"/>\n" +
+ " <count count=\"5\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"1\"/>\n" +
+ " <count count=\"1\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " </routes>\n" +
+ " </snapshot>\n" +
+ "\n" +
+ "</status>\n", val);
+ }
+
+ @Test
+ public void testStatusPage2() throws Exception {
+ setup(null);
+
+ testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
+ testFeed(xmlFilesPath + "test10.xml", "feed?");
+ testFeed(xmlFilesPath + "test10.xml", "feed?route=storage");
+ testFeed(xmlFilesPath + "test_removes", "remove?");
+
+ assertEquals(2, factory.sessionsCreated());
+ Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+
+ String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
+ val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\"");
+
+ assertEquals("<status>\n" +
+ "\n" +
+ " <routes>\n" +
+ " <route name=\"total\" description=\"Messages sent to all routes\">\n" +
+ " <total description=\"All kinds of messages sent to the given route\">\n" +
+ " <latency count=\"14\"/>\n" +
+ " <count count=\"14\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"6\"/>\n" +
+ " <count count=\"6\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " <route name=\"default\" description=\"Messages sent to the named route\">\n" +
+ " <total description=\"All kinds of messages sent to the given route\">\n" +
+ " <latency count=\"9\"/>\n" +
+ " <count count=\"9\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"4\"/>\n" +
+ " <count count=\"4\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"3\"/>\n" +
+ " <count count=\"3\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " <route name=\"storage\" description=\"Messages sent to the named route\">\n" +
+ " <total description=\"All kinds of messages sent to the given route\">\n" +
+ " <latency count=\"5\"/>\n" +
+ " <count count=\"5\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </putdocument>\n" +
+ " <updatedocument>\n" +
+ " <latency count=\"2\"/>\n" +
+ " <count count=\"2\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </updatedocument>\n" +
+ " <removedocument>\n" +
+ " <latency count=\"1\"/>\n" +
+ " <count count=\"1\"/>\n" +
+ " <ignored count=\"0\"/>\n" +
+ " </removedocument>\n" +
+ " </route>\n" +
+ " </routes>\n" +
+ "\n" +
+ "</status>\n", val);
+ }
+
+ @Test
+ public void testMetricForIgnoredDocumentsIsIncreased() throws Exception {
+ DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() {
+ @Override
+ public Reply createReply(Message m) {
+ return new DocumentIgnoredReply();
+ }
+ };
+ setupWithReplyFactory(replyFactory);
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
+ assertEquals(2, res.messages.size());
+
+ String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
+
+ assertEquals("<result>\n" +
+ "\n" +
+ " <route name=\"default\">\n" +
+ " <total>\n" +
+ " <ignored count=\"2\"/>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <ignored count=\"2\"/>\n" +
+ " </putdocument>\n" +
+ " </route>\n" +
+ "\n" +
+ "</result>\n", val);
+ }
+
+ @Test
+ public void testPostXMLWithMBusFailureAllowed() throws Exception {
+ setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>"));
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10b", d.toString());
+ }
+
+ String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", "");
+ System.out.println(val);
+
+ assertEquals("<result>\n" +
+ "\n" +
+ " <route name=\"default\">\n" +
+ " <total>\n" +
+ " <errors>\n" +
+ " <error name=\"total\" count=\"2\"/>\n" +
+ " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" +
+ " </errors>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <errors>\n" +
+ " <error name=\"total\" count=\"2\"/>\n" +
+ " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" +
+ " </errors>\n" +
+ " </putdocument>\n" +
+ " </route>\n\n" +
+ " <errors count=\"2\">\n" +
+ " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in &lt;document&gt;\"/>\n" +
+ " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in &lt;document&gt;\"/>\n" +
+ " </errors>\n" +
+ "\n" +
+ "</result>\n", val);
+
+ assertTrue(res.error != null);
+ assertTrue(res.errorCount > 0);
+ }
+
+ @Test
+ public void testPostXMLWithMBusFailure() throws Exception {
+ setup(new com.yahoo.messagebus.Error(32, "Hello world"));
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
+
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+
+ String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", "");
+ assertEquals("<result>\n" +
+ "\n" +
+ " <route name=\"default\">\n" +
+ " <total>\n" +
+ " <errors>\n" +
+ " <error name=\"total\" count=\"1\"/>\n" +
+ " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" +
+ " </errors>\n" +
+ " </total>\n" +
+ " <putdocument>\n" +
+ " <errors>\n" +
+ " <error name=\"total\" count=\"1\"/>\n" +
+ " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" +
+ " </errors>\n" +
+ " </putdocument>\n" +
+ " </route>\n\n" +
+ " <errors count=\"1\">\n" +
+ " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" +
+ " </errors>\n" +
+ "\n" +
+ "</result>\n", val);
+
+ assertTrue(res.error != null);
+ assertTrue(res.errorCount > 0);
+ }
+
+ @Test
+ public void testPostXMLWithIllegalDocId() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?");
+
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ }
+
+ @Test
+ public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false");
+
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:ok", d.toString());
+ }
+ }
+
+ @Test
+ public void testPostUnparseableXML() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?");
+
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ }
+
+ @Test
+ public void testOverrides() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2");
+
+ assertEquals(2, res.messages.size());
+
+ for (Message m : res.messages) {
+ assertEquals(2222, m.getTimeRemaining());
+ assertEquals(Route.parse("storage"), m.getRoute());
+ assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority());
+ }
+ }
+
+ @Test
+ public void testBogusPriority() throws Exception {
+ try {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS");
+ assertTrue(false);
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ @Test
+ public void testPostXMLWithIllegalDocIdFirst() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?");
+
+ assertEquals(0, res.messages.size());
+ }
+
+ @Test
+ public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false");
+
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
+ DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
+ assertEquals("doc:news:http://news10a", d.toString());
+ }
+ }
+
+ @Test
+ public void testSimpleRemove() throws Exception {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:removeme", d.toString());
+ }
+ }
+
+ @Test
+ public void testRemoveUser() throws Exception {
+ setup(null);
+
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
+ String selection = ((RemoveLocationMessage)m).getDocumentSelection();
+ assertEquals("storage", m.getRoute().toString());
+ assertEquals("id.user=1234", selection);
+ }
+ }
+
+ @Test
+ public void testRemoveGroup() throws Exception {
+ setup(null);
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
+ String selection = ((RemoveLocationMessage)m).getDocumentSelection();
+ assertEquals("storage", m.getRoute().toString());
+ assertEquals("id.group=\"foo\"", selection);
+ }
+ }
+
+ @Test
+ public void testRemoveBadSyntax() throws Exception {
+ setup(null);
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(0, res.messages.size());
+ assertTrue(res.error.toString().contains("Exactly one of"));
+ }
+
+ @Test
+ public void testRemoveGroupMultipleClusters() throws Exception {
+ setup(null);
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1"));
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2"));
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(0, res.messages.size());
+ assertTrue(res.error.toString().contains("More than one"));
+ }
+
+ @Test
+ public void testRemoveGroupNoClusters() throws Exception {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(0, res.messages.size());
+ assertTrue(res.error.toString().contains("No storage clusters"));
+ }
+
+ @Test
+ public void testRemoveSelection() throws Exception {
+ setup(null);
+ context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
+ Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
+ String selection = ((RemoveLocationMessage)m).getDocumentSelection();
+ assertEquals("id.user=1234", selection);
+ }
+ }
+
+ @Test
+ public void testSimpleRemoveIndex() throws Exception {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(1, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:removeme", d.toString());
+ }
+ }
+
+ @Test
+ public void testPostRemove() throws Exception {
+ setup(null);
+ Result res = testFeed(xmlFilesPath + "test_removes", "remove?");
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:remove1", d.toString());
+ }
+
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:remove2", d.toString());
+ }
+ }
+
+ @Test
+ public void testRemoveBogusId() throws Exception {
+ try {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertTrue(false);
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testMultiRemove() throws Exception {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(3, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:removeme", d.toString());
+ }
+
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:remove2", d.toString());
+ }
+
+ {
+ Message m = res.messages.get(2);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
+ assertEquals("doc:test:remove3", d.toString());
+ }
+ }
+
+ @Test
+ public void testMultiRemoveSameDoc() throws Exception {
+ setup(null);
+ Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
+ assertEquals(2, res.messages.size());
+
+ {
+ Message m = res.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ }
+
+ {
+ Message m = res.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
+ }
+ }
+
+ @Test
+ public void testFeedHandlerStatusCreation() throws Exception {
+ VespaFeedHandlerStatus status = new VespaFeedHandlerStatus(
+ new FeedContext(new MessagePropertyProcessor(
+ new FeederConfig(new FeederConfig.Builder()),
+ new LoadTypeConfig(new LoadTypeConfig.Builder())),
+ factory, null, new ClusterList(), new NullFeedMetric()),
+ true, true,
+ Executors.newCachedThreadPool());
+ }
+
+ private class TestDocProc extends DocumentProcessor {
+ @Override
+ public Progress process(Processing processing) {
+ for (DocumentOperation op : processing.getDocumentOperations()) {
+ if (op instanceof DocumentPut) {
+ Document document = ((DocumentPut)op).getDocument();
+ document.setFieldValue("last_downloaded", new IntegerFieldValue(1234));
+ }
+ }
+ return Progress.DONE;
+ }
+ }
+
+ private class TestLaterDocProc extends DocumentProcessor {
+ private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName());
+
+ private int counter = 0;
+ @Override
+ public Progress process(Processing processing) {
+ synchronized (this) {
+ counter++;
+ if (counter % 2 == 1) {
+ log.info("Returning LATER.");
+ return Progress.LATER;
+ }
+ log.info("Returning DONE.");
+ return Progress.DONE;
+ }
+ }
+ }
+
+ private Result testRequest(HttpRequest req) throws Exception {
+ HttpResponse response = null;
+ String feedPrefix = "feed";
+ String removePrefix = "remove";
+ String feedStatusPrefix = "feedstatus";
+ String removeLocationPrefix = "removelocation";
+
+ if (req.getUri().getPath().startsWith(feedPrefix)) {
+ response = feedHandler.handle(req);
+ }
+ if (req.getUri().getPath().startsWith(removePrefix)) {
+ response = removeHandler.handle(req);
+ }
+ if (req.getUri().getPath().startsWith(feedStatusPrefix)) {
+ response = statusHandler.handle(req);
+ }
+ if (req.getUri().getPath().startsWith(removeLocationPrefix)) {
+ response = removeLocationHandler.handle(req);
+ }
+
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ response.render(output);
+
+ Result res = new Result();
+ res.messages = factory.messages;
+ res.output = new String(output.toByteArray());
+
+ if (response instanceof FeedResponse) {
+ FeedResponse feedResponse = (FeedResponse)response;
+ res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0);
+ res.errorCount = feedResponse.getErrorMessageList().size();
+ assertTrue(feedResponse.isSuccess() == (res.errorCount == 0));
+ }
+ return res;
+ }
+
+ private Result testFeed(String xmlFile, String request) throws Exception {
+ return testRequest(new FileRequest(new File(xmlFile), request).toRequest());
+ }
+
+ private Result testFeedGZIP(String xmlFile, String request) throws Exception {
+ return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest());
+ }
+
+ private class FileRequest {
+
+ private final String req;
+ private final File f;
+ private boolean gzip = false;
+
+ FileRequest(File f, String req) {
+ this.req = req;
+ this.f = f;
+ }
+
+ FileRequest(File f, String req, boolean gzip) {
+ this.f = f;
+ this.req = req;
+ this.gzip = gzip;
+ }
+
+ public InputStream getData() {
+ try {
+ InputStream fileStream = new FileInputStream(f);
+ if (gzip) {
+ // Not exactly pretty, but in lack of an elegant way of transcoding
+ ByteArrayOutputStream rawOut = new ByteArrayOutputStream();
+ GZIPOutputStream compressed = new GZIPOutputStream(rawOut);
+ byte[] buffer = new byte[1024];
+ int read = -1;
+ while (true) {
+ read = fileStream.read(buffer);
+ if (read == -1) break;
+ compressed.write(buffer, 0, read);
+ }
+ compressed.finish();
+ compressed.flush();
+ rawOut.flush();
+ return new ByteArrayInputStream(rawOut.toByteArray());
+ }
+ return fileStream;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ public void addHeaders(HeaderFields headers) {
+ headers.add("Content-Type", "image/jpeg");
+ if (gzip)
+ headers.add("Content-Encoding", "gzip");
+ }
+
+ public HttpRequest toRequest() {
+ HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData());
+ addHeaders(request.getJDiscRequest().headers());
+ return request;
+ }
+
+ }
+
+ private class Result {
+ private List<Message> messages;
+ private String output;
+ private com.yahoo.processing.request.ErrorMessage error;
+ private int errorCount;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java
new file mode 100644
index 00000000000..708a9ed7a6b
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.feedhandler.v3;
+
+import com.google.common.base.Splitter;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.http.client.config.FeedParams;
+import com.yahoo.vespa.http.client.core.ErrorCode;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespa.http.server.ReplyContext;
+import com.yahoo.vespa.http.server.FeedHandlerV3;
+import org.junit.Test;
+import com.yahoo.container.jdisc.HttpRequest;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import com.yahoo.messagebus.Result;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class FeedTesterV3 {
+
+ @Test
+ public void feedOneDocument() throws Exception {
+ final FeedHandlerV3 feedHandlerV3 = setupFeederHandler();
+ HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1));
+ ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ httpResponse.render(outStream);
+ assertThat(httpResponse.getContentType(), is("text/plain"));
+ assertThat(Utf8.toString(outStream.toByteArray()), is("1230 OK message trace\n"));
+
+ }
+
+ @Test
+ public void feedManyDocument() throws Exception {
+ final FeedHandlerV3 feedHandlerV3 = setupFeederHandler();
+ HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100));
+ ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ httpResponse.render(outStream);
+ assertThat(httpResponse.getContentType(), is("text/plain"));
+ String result = Utf8.toString(outStream.toByteArray());
+ assertThat(Splitter.on("\n").splitToList(result).size(), is(101));
+ }
+
+ DocumentTypeManager createDoctypeManager() {
+ DocumentTypeManager docTypeManager = new DocumentTypeManager();
+ DocumentType documentType = new DocumentType("testdocument");
+ documentType.addField("title", DataType.STRING);
+ documentType.addField("body", DataType.STRING);
+ docTypeManager.registerDocumentType(documentType);
+ return docTypeManager;
+ }
+
+ HttpRequest createRequest(int numberOfDocs) {
+ String clientId = "client123";
+ StringBuilder wireData = new StringBuilder();
+ for (int x = 0; x < numberOfDocs; x++) {
+ String docData = "[{\"put\": \"id:testdocument:testdocument::c\", \"fields\": { \"title\": \"fooKey\", \"body\": \"value\"}}]";
+ String operationId = "123" + x;
+ wireData.append(operationId + " " + Integer.toHexString(docData.length()) + "\n" + docData);
+ }
+ InputStream inputStream = new ByteArrayInputStream(wireData.toString().getBytes());
+ HttpRequest request = HttpRequest.createTestRequest(
+ "http://dummyhostname:19020/reserved-for-internal-use/feedapi",
+ com.yahoo.jdisc.http.HttpRequest.Method.POST,
+ inputStream);
+ request.getJDiscRequest().headers().add(Headers.VERSION, "3");
+ request.getJDiscRequest().headers().add(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name());
+ request.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ request.getJDiscRequest().headers().add(Headers.CLIENT_ID, clientId);
+ request.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ request.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ request.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ return request;
+ }
+
+ FeedHandlerV3 setupFeederHandler() throws Exception {
+ Executor threadPool = Executors.newCachedThreadPool();
+ DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true));
+ FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3(
+ threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null) {
+ @Override
+ protected ReferencedResource<SharedSourceSession> retainSource(
+ SessionCache sessionCache, SourceSessionParams sessionParams) {
+ SharedSourceSession sharedSourceSession = mock(SharedSourceSession.class);
+
+ try {
+ Mockito.stub(sharedSourceSession.sendMessageBlocking(anyObject())).toAnswer((Answer) invocation -> {
+ Object[] args = invocation.getArguments();
+ PutDocumentMessage putDocumentMessage = (PutDocumentMessage) args[0];
+ ReplyContext replyContext = (ReplyContext)putDocumentMessage.getContext();
+ replyContext.feedReplies.add(new OperationStatus("message", replyContext.docId, ErrorCode.OK, "trace"));
+ Result result = mock(Result.class);
+ when(result.isAccepted()).thenReturn(true);
+ return result;
+ });
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ Result result = mock(Result.class);
+ when(result.isAccepted()).thenReturn(true);
+ ReferencedResource<SharedSourceSession> refSharedSessopn =
+ new ReferencedResource<>(sharedSourceSession, () -> {});
+ return refSharedSessopn;
+ }
+ };
+ feedHandlerV3.injectDocumentManangerForTests(createDoctypeManager());
+ return feedHandlerV3;
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java
new file mode 100644
index 00000000000..8b991909cd8
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorIterator;
+import org.junit.Test;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.junit.Assert.*;
+
+public class ContinuationHitTest {
+
+ private static final String SINGLE_BUCKET_URL_SAFE_BASE64
+ = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA";
+ private static final String MULTI_BUCKET_URL_SAFE_BASE64
+ = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" +
+ "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA=";
+
+ @Test
+ public void continuationTokensAreUrlSafeBase64Encoded() throws Exception {
+ ContinuationHit hit = new ContinuationHit(createSingleBucketProgress());
+ // We want -_ instead of +/
+ assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue());
+ }
+
+ @Test
+ public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception {
+ ContinuationHit hit = new ContinuationHit(createMultiBucketProgress());
+ assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits.
+ assertFalse(hit.getValue().contains("\n"));
+ }
+
+ @Test
+ public void decodingAcceptsUrlSafeTokens() throws Exception {
+ final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64);
+ // Roundtrip should yield identical results.
+ assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64,
+ new ContinuationHit(token).getValue());
+ }
+
+ /**
+ * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era.
+ */
+ @Test
+ public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception {
+ final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64);
+ final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64);
+
+ assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64,
+ new ContinuationHit(legacyToken).getValue());
+ }
+
+ /**
+ * Legacy Base64 encoder would happily output line breaks after each MIME line
+ * boundary. Ensure we handle these gracefully.
+ */
+ @Test
+ public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception {
+ final String multiBucketLegacyToken =
+ "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" +
+ "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA=";
+ final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken);
+
+ assertEquals(MULTI_BUCKET_URL_SAFE_BASE64,
+ new ContinuationHit(legacyToken).getValue());
+ }
+
+ /**
+ * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit)
+ */
+ private ProgressToken createSingleBucketProgress() {
+ ProgressToken token = new ProgressToken(16);
+ // Use explicit bucket set so we can better control the binary representation
+ // of the buckets, and thus the values written as base 64.
+ Set<BucketId> buckets = new TreeSet<>();
+ // This particular bucket ID will contain +/ chars when output as non-URL safe base 64.
+ buckets.add(new BucketId(58, 0x123456789abcfeffL));
+ VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token.
+ return token;
+ }
+
+ /**
+ * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit)
+ */
+ private ProgressToken createMultiBucketProgress() {
+ ProgressToken token = new ProgressToken(16);
+ Set<BucketId> buckets = new TreeSet<>();
+ buckets.add(new BucketId(58, 0x123456789abcfeffL));
+ buckets.add(new BucketId(58, 0x123456789abcfefaL));
+ buckets.add(new BucketId(58, 0x123456789abcfefbL));
+ VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token.
+ return token;
+ }
+
+ private String convertedToMimeBase64Chars(String token) {
+ // Doesn't split on MIME line boundaries, so not fully MIME compliant.
+ return token.replace('-', '+').replace('_', '/');
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java
new file mode 100755
index 00000000000..ea88128aaa9
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.documentapi.VisitorParameters;
+import com.yahoo.documentapi.VisitorSession;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedapi.SendSession;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+
+/**
+ * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage
+ */
+public class DocumentSessionFactory extends DummySessionFactory {
+
+ private DocumentType docType;
+ private Error error;
+ // Reply instances are shared between the two collections.
+ List<GetDocumentReply> autoReplies;
+ Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>();
+ boolean autoReply = true;
+ boolean nullReply = false;
+ private int sessionsCreated = 0;
+
+ private class DocumentReplySession extends SendSession {
+
+ ReplyHandler handler;
+ Error e;
+ DummySessionFactory owner;
+
+ public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) {
+ this.handler = handler;
+ this.e = e;
+ this.owner = owner;
+ }
+
+ protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException {
+ if (!(m instanceof GetDocumentMessage)) {
+ throw new IllegalArgumentException("Expected GetDocumentMessage");
+ }
+ GetDocumentMessage gm = (GetDocumentMessage)m;
+ owner.messages.add(m);
+ if (autoReply) {
+ Document replyDoc;
+ if (!nullReply) {
+ replyDoc = new Document(docType, gm.getDocumentId());
+ } else {
+ replyDoc = null;
+ }
+ Reply r = new GetDocumentReply(replyDoc);
+ r.setMessage(m);
+ r.setContext(m.getContext());
+ if (e != null) {
+ r.addError(e);
+ }
+ handler.handleReply(r);
+ } else if (owner.messages.size() == autoReplies.size()) {
+ // Pair up all replies with their messages
+ for (Message msg : owner.messages) {
+ GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId());
+ reply.setMessage(msg);
+ reply.setContext(msg.getContext());
+ if (e != null) {
+ reply.addError(e);
+ }
+ }
+ // Now send them in the correct order. Instances are shared, so source
+ // messages and contexts are properly set
+ for (Reply reply : autoReplies) {
+ handler.handleReply(reply);
+ }
+ }
+
+ return Result.ACCEPTED;
+ }
+
+ public void close() {
+ }
+ }
+
+ public DocumentSessionFactory(DocumentType docType) {
+ this.docType = docType;
+ this.error = null;
+ }
+
+ public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) {
+ this.docType = docType;
+ this.error = error;
+ this.autoReplies = Arrays.asList(autoReplies);
+ for (GetDocumentReply reply : autoReplies) {
+ this.autoReplyLookup.put(reply.getDocument().getId(), reply);
+ }
+ this.autoReply = autoReply;
+ }
+
+ public boolean isNullReply() {
+ return nullReply;
+ }
+
+ public void setNullReply(boolean nullReply) {
+ this.nullReply = nullReply;
+ }
+
+ public int getSessionsCreated() {
+ return sessionsCreated;
+ }
+
+ public SendSession createSendSession(ReplyHandler r, Metric metric) {
+ ++sessionsCreated;
+ return new DocumentReplySession(r, error, this);
+ }
+
+ @Override
+ public VisitorSession createVisitorSession(VisitorParameters p) {
+ return new DummyVisitorSession(p, docType);
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java
new file mode 100644
index 00000000000..d1c55c0968f
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentType;
+import com.yahoo.documentapi.*;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.Trace;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Stub to test visitors.
+ */
+public class DummyVisitorSession implements VisitorSession {
+
+ final VisitorParameters parameters;
+ final DocumentType documentType;
+ final List<Message> autoReplyMessages = new ArrayList<>();
+
+ DummyVisitorSession(VisitorParameters p, DocumentType documentType) {
+ parameters = p;
+ this.documentType = documentType;
+ p.getLocalDataHandler().setSession(this);
+ addDefaultReplyMessages();
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+
+ @Override
+ public ProgressToken getProgress() {
+ return new ProgressToken(12);
+ }
+
+ @Override
+ public Trace getTrace() {
+ return null;
+ }
+
+ public void addDocumentReply(String docId) {
+ Document replyDoc = new Document(documentType, docId);
+ autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc)));
+ }
+
+ public void addRemoveReply(String docId) {
+ autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId)));
+ }
+
+ public void addDefaultReplyMessages() {
+ addDocumentReply("userdoc:foo:1234:bar");
+ if (parameters.visitRemoves()) {
+ addRemoveReply("userdoc:foo:1234:removed");
+ }
+ }
+
+ public void clearAutoReplyMessages() {
+ autoReplyMessages.clear();
+ }
+
+ @Override
+ public boolean waitUntilDone(long l) throws InterruptedException {
+ for (Message msg : autoReplyMessages) {
+ parameters.getLocalDataHandler().onMessage(msg, new AckToken(this));
+ }
+ return true;
+ }
+
+ @Override
+ public void ack(AckToken ackToken) {
+ }
+
+ @Override
+ public void abort() {
+ }
+
+ @Override
+ public VisitorResponse getNext() {
+ return null;
+ }
+
+ @Override
+ public VisitorResponse getNext(int i) throws InterruptedException {
+ return null;
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java
new file mode 100755
index 00000000000..1e39b9766b1
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java
@@ -0,0 +1,1090 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.jdisc.HeaderFields;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.java7compat.Util;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.prelude.templates.SearchRendererAdaptor;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespaclient.ClusterList;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.zip.GZIPOutputStream;
+
+import static org.junit.Assert.*;
+
+public class GetSearcherTestCase {
+
+ private DocumentTypeManager docMan = null;
+ private DocumentType docType;
+ private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder());
+ private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder());
+
+ @org.junit.Before
+ public void setUp() {
+ docMan = new DocumentTypeManager();
+ docType = new DocumentType("kittens");
+ docType.addHeaderField("name", DataType.STRING);
+ docType.addField("description", DataType.STRING);
+ docType.addField("image", DataType.STRING);
+ docType.addField("fluffiness", DataType.INT);
+ docType.addField("foo", DataType.RAW);
+ docMan.registerDocumentType(docType);
+ }
+
+ @org.junit.After
+ public void tearDown() {
+ docMan = null;
+ docType = null;
+ }
+
+ private void assertHits(HitGroup hits, String... wantedHits) {
+ assertEquals(wantedHits.length, hits.size());
+ for (int i = 0; i < wantedHits.length; ++i) {
+ assertTrue(hits.get(i) instanceof DocumentHit);
+ DocumentHit hit = (DocumentHit)hits.get(i);
+ assertEquals(wantedHits[i], hit.getDocument().getId().toString());
+ }
+ }
+
+ @Test
+ public void testGetSingleDocumentQuery() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2"));
+ System.out.println("HTTP request is " + result.getQuery().getHttpRequest());
+
+ assertEquals(1, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:1:2", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ }
+ assertEquals(1, result.hits().size());
+ assertHits(result.hits(), "userdoc:kittens:1:2");
+ // By default, document hit should not have its hit fields set
+ DocumentHit hit = (DocumentHit)result.hits().get(0);
+ assertEquals(0, hit.fieldKeys().size());
+ }
+
+ @Test
+ public void testGetMultipleDocumentsQuery() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4");
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+
+ assertEquals(2, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:1:2", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ }
+
+ {
+ Message m = factory.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:3:4", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ }
+ assertEquals(2, result.hits().size());
+ assertNull(result.hits().getErrorHit());
+ assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4");
+ assertEquals(2, query.getHits());
+ }
+
+ // Test that you can use both query string and POSTed IDs
+ @Test
+ public void testGetMultipleDocumentsQueryAndPOST() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10";
+ MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4");
+ Query query = new Query(request.toRequest());
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+
+ assertEquals(5, factory.messages.size());
+ assertEquals(5, result.hits().size());
+ assertNull(result.hits().getErrorHit());
+ assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4",
+ "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10");
+ }
+
+ @Test
+ public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10";
+
+ // Create with automatic GZIP encoding
+ MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true);
+ Query query = new Query(request.toRequest());
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+
+ assertEquals(5, factory.messages.size());
+ assertEquals(5, result.hits().size());
+ assertNull(result.hits().getErrorHit());
+ assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4",
+ "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10");
+ }
+
+ /* Test that a query without any ids is passed through to the next chain */
+ @Test
+ public void testQueryPassThrough() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ HitGroup hits = new HitGroup("mock");
+ hits.add(new Hit("blernsball"));
+ Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits));
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern"));
+
+ assertEquals(0, factory.messages.size());
+ assertEquals(1, result.hits().size());
+ assertNotNull(result.hits().get("blernsball"));
+ }
+
+ /* Test that a query will contain both document hits and hits from a searcher
+ * further down the chain, iff the searcher returns a DocumentHit.
+ */
+ @Test
+ public void testQueryPassThroughAndGet() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo"));
+ doc1.setFieldValue("name", new StringFieldValue("megacat"));
+ doc1.setFieldValue("description", new StringFieldValue("supercat"));
+ doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1)
+ };
+
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5);
+ Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit));
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo"));
+
+ assertEquals(1, factory.messages.size());
+ assertEquals(2, result.hits().size());
+ assertNotNull(result.hits().get("userdoc:kittens:5678:bar"));
+ assertNotNull(result.hits().get("userdoc:kittens:1234:foo"));
+ }
+
+ @Test
+ public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ HitGroup hits = new HitGroup("mock");
+ hits.add(new Hit("blernsball"));
+ Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits));
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa"));
+
+ assertEquals(0, factory.messages.size());
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " +
+ "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " +
+ "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " +
+ "Only DocumentHit instances are supported in the backend hit result set " +
+ "when doing queries that contain document identifier sets recognised by the " +
+ "Get Searcher.\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("route66").retryenabled(false)), defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah"));
+
+ assertEquals(1, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("doc:batman:dahnahnahnah", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(Route.parse("route66"), gdm.getRoute());
+ assertFalse(gdm.getRetryEnabled());
+ assertEquals(458000, gdm.getTimeRemaining());
+ }
+ }
+
+ @Test
+ public void testConfigChanges() throws Exception {
+ String config = "raw:timeout 458\nroute \"riksveg18\"\nretryenabled true";
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)),
+ defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah"));
+
+ assertEquals(1, factory.messages.size());
+ assertEquals(1, factory.getSessionsCreated());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("doc:batman:dahnahnahnah", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(Route.parse("riksveg18"), gdm.getRoute());
+ assertTrue(gdm.getRetryEnabled());
+ assertEquals(458000, gdm.getTimeRemaining());
+ }
+
+ factory.messages.clear();
+
+ FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder()
+ .timeout(123)
+ .route("e6")
+ .retryenabled(false)
+ );
+ searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg);
+
+ new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=doc:spiderman:does_whatever_a_spider_can"));
+
+ // riksveg18 is created again, and e6 is created as well.
+ assertEquals(3, factory.getSessionsCreated());
+
+ assertEquals(1, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(Route.parse("e6"), gdm.getRoute());
+ assertFalse(gdm.getRetryEnabled());
+ assertEquals(123000, gdm.getTimeRemaining());
+ }
+ }
+
+ @Test
+ public void testQueryOverridesDefaults() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=458"));
+
+ assertEquals(2, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:1:2", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
+ assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
+ assertEquals(458000, gdm.getTimeRemaining());
+ }
+
+ {
+ Message m = factory.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:3:4", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
+ assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
+ assertEquals(458000, gdm.getTimeRemaining());
+ }
+ }
+
+ @Test
+ public void testQueryOverridesConfig() throws Exception {
+ String config = "raw:timeout 458\nroute \"route66\"";
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123"));
+
+ assertEquals(2, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:1:2", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
+ assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
+ assertEquals(123000, gdm.getTimeRemaining());
+ }
+
+ {
+ Message m = factory.messages.get(1);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:3:4", d.toString());
+ assertEquals("[all]", gdm.getFieldSet());
+ assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
+ assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
+ assertEquals(123000, gdm.getTimeRemaining());
+ }
+ }
+
+ @Test
+ public void testBadPriorityValue() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon"));
+
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: No enum const" +
+ (Util.isJava7Compatible() ? "ant " : " class ") +
+ "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" +
+ (Util.isJava7Compatible() ? "." : "$") +
+ "Priority.onkel_jubalon\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testMultiIdBadArrayIndex() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ {
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[1]=userdoc:kittens:1:2"));
+
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: query contains document ID " +
+ "array that is not zero-based and/or linearly increasing\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ {
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3"));
+
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: query contains document ID " +
+ "array that is not zero-based and/or linearly increasing\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ {
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:1:2&id[1]=userdoc:kittens:2:3"));
+
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: query contains document ID " +
+ "array that is not zero-based and/or linearly increasing\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ {
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0=userdoc:kittens:1:2"));
+
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+ }
+
+ @Test
+ public void testLegacyHeadersOnly() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:1:2&headersonly=true"));
+
+ assertEquals(1, factory.messages.size());
+ {
+ Message m = factory.messages.get(0);
+ assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
+ GetDocumentMessage gdm = (GetDocumentMessage)m;
+ DocumentId d = gdm.getDocumentId();
+ assertEquals("userdoc:kittens:1:2", d.toString());
+ assertEquals("[header]", gdm.getFieldSet());
+ }
+ assertEquals(1, result.hits().size());
+ assertHits(result.hits(), "userdoc:kittens:1:2");
+ }
+
+ @Test
+ public void testFieldSet() throws Exception {
+ }
+
+ @Test
+ public void testConsistentResultOrdering() throws Exception {
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))),
+ new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))),
+ new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123")))
+ };
+
+ // Use a predefined reply list to ensure messages are answered out of order
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8"));
+
+ assertEquals(3, factory.messages.size());
+ assertEquals(3, result.hits().size());
+ // Hits must be in the same order as their document IDs in the query
+ assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8");
+
+ assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex());
+ assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex());
+ assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex());
+ }
+
+ @Test
+ public void testResultWithSingleError() throws Exception {
+ com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly");
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"));
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testResultWithMultipleErrors() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
+ Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111"));
+ GetDocumentReply errorReply1 = new GetDocumentReply(doc1);
+ errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas."));
+ GetDocumentReply errorReply2 = new GetDocumentReply(doc2);
+ errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains."));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ errorReply1,
+ errorReply2
+ };
+
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111"));
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" +
+ "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testResultWithNullDocument() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true);
+ factory.setNullReply(true);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:55:bad_document_id"));
+ // Document not found does not produce any hit at all, error or otherwise
+ assertNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testDefaultDocumentHitRendering() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4"));
+ doc1.setFieldValue("name", new StringFieldValue("mittens"));
+ doc1.setFieldValue("description", new StringFieldValue("it's a cat"));
+ doc1.setFieldValue("fluffiness", new IntegerFieldValue(8));
+ Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2"));
+ doc2.setFieldValue("name", new StringFieldValue("garfield"));
+ doc2.setFieldValue("description",
+ new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised"));
+ doc2.setFieldValue("fluffiness", new IntegerFieldValue(2));
+ Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
+ GetDocumentReply errorReply = new GetDocumentReply(doc3);
+ errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas."));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ new GetDocumentReply(doc2),
+ errorReply
+ };
+
+ // Use a predefined reply list to ensure messages are answered out of order
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4"));
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" +
+ "</errors>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" +
+ " <name>garfield</name>\n" +
+ " <description>preliminary research indicates &lt;em&gt;hatred&lt;/em&gt; of mondays. caution advised</description>\n" +
+ " <fluffiness>2</fluffiness>\n" +
+ "</document>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" +
+ " <name>mittens</name>\n" +
+ " <description>it's a cat</description>\n" +
+ " <fluffiness>8</fluffiness>\n" +
+ "</document>\n" +
+ "</result>\n", xmlResult);
+ }
+
+ @Test
+ public void testDocumentFieldNoContentType() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
+ doc1.setFieldValue("name", "derrick");
+ doc1.setFieldValue("description", "kommisar katze");
+ doc1.setFieldValue("fluffiness", 0);
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ };
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:5:1&field=description"));
+
+ assertNull(result.hits().getErrorHit());
+ assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType());
+ assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>kommisar katze</result>\n", result);
+ }
+
+ @Test
+ public void testDocumentFieldEscapeXML() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
+ doc1.setFieldValue("name", "asfd");
+ doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>");
+ doc1.setFieldValue("fluffiness", 0);
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ };
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:5:1&field=description"));
+
+ assertNull(result.hits().getErrorHit());
+ assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType());
+ assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>&lt;script type=\"evil/madness\"&gt;horror &amp; screams&lt;/script&gt;</result>\n", result);
+ }
+
+ @Test
+ public void testDocumentFieldRawContent() throws Exception {
+ byte[] contentBytes = new byte[] { 0, -128, 127 };
+
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456"));
+ doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes)));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1)
+ };
+
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:123:456&field=foo"));
+
+ assertNull(result.hits().getErrorHit());
+ assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType());
+
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ SearchRendererAdaptor.callRender(stream, result);
+ stream.flush();
+
+ byte[] resultBytes = stream.toByteArray();
+ assertEquals(contentBytes.length, resultBytes.length);
+ for (int i = 0; i < resultBytes.length; ++i) {
+ assertEquals(contentBytes[i], resultBytes[i]);
+ }
+ }
+
+ @Test
+ public void testDocumentFieldRawWithContentOverride() throws Exception {
+ byte[] contentBytes = new byte[] { 0, -128, 127 };
+
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456"));
+ doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes)));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1)
+ };
+
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy"));
+
+ assertNull(result.hits().getErrorHit());
+ assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType());
+
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ SearchRendererAdaptor.callRender(stream, result);
+ stream.flush();
+
+ byte[] resultBytes = stream.toByteArray();
+ assertEquals(contentBytes.length, resultBytes.length);
+ for (int i = 0; i < resultBytes.length; ++i) {
+ assertEquals(contentBytes[i], resultBytes[i]);
+ }
+ }
+
+ @Test
+ public void testDocumentFieldWithMultipleIDs() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name"));
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
+ "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testDocumentFieldNotSet() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
+ doc1.setFieldValue("name", "asdf");
+ doc1.setFieldValue("description", "fdsafsdf");
+ doc1.setFieldValue("fluffiness", 10);
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ };
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:5:1&field=image"));
+
+ assertNotNull(result.hits().getErrorHit());
+ assertEquals(1, result.hits().size());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " +
+ "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+
+ @Test
+ public void testDocumentFieldWithDocumentNotFound() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true);
+ factory.setNullReply(true);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:1:2&field=name"));
+ assertNotNull(result.hits().getErrorHit());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " +
+ "Document not found, could not return field 'name'\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
+ doc1.setFieldValue("name", "asdf");
+ // don't set body fields
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ };
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true"));
+
+ assertNotNull(result.hits().getErrorHit());
+ assertEquals(1, result.hits().size());
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " +
+ "Field 'description' is located in document body, but headersonly " +
+ "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" +
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testVespaXMLTemplate() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4"));
+ doc1.setFieldValue("name", "mittens");
+ doc1.setFieldValue("description", "it's a cat");
+ doc1.setFieldValue("fluffiness", 8);
+ Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2"));
+ doc2.setFieldValue("name", "garfield");
+ doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised");
+ doc2.setFieldValue("fluffiness", 2);
+ Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
+ GetDocumentReply errorReply = new GetDocumentReply(doc3);
+ errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>"));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1),
+ new GetDocumentReply(doc2),
+ errorReply
+ };
+
+ // Use a predefined reply list to ensure messages are answered out of order
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO!
+
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a &lt;ni!&gt;&quot;shrubbery&quot;&lt;/ni!&gt;\"/>\n"+
+ "</errors>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" +
+ " <name>garfield</name>\n" +
+ " <description>preliminary research indicates &lt;em&gt;hatred&lt;/em&gt; of mondays. caution advised</description>\n" +
+ " <fluffiness>2</fluffiness>\n" +
+ "</document>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" +
+ " <name>mittens</name>\n" +
+ " <description>it's a cat</description>\n" +
+ " <fluffiness>8</fluffiness>\n" +
+ "</document>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testDocumentHitWithPopulatedHitFields() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo"));
+ doc1.setFieldValue("name", new StringFieldValue("megacat"));
+ doc1.setFieldValue("description", new StringFieldValue("supercat"));
+ doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new GetDocumentReply(doc1)
+ };
+
+ // Use a predefined reply list to ensure messages are answered out of order
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true"));
+ assertEquals(1, result.hits().size());
+ assertHits(result.hits(), "userdoc:kittens:1234:foo");
+
+ DocumentHit hit = (DocumentHit)result.hits().get(0);
+ Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator();
+ Set<String> fieldSet = new TreeSet<>();
+ while (iter.hasNext()) {
+ Map.Entry<String, Object> kv = iter.next();
+ StringBuilder field = new StringBuilder();
+ field.append(kv.getKey()).append(" -> ").append(kv.getValue());
+ fieldSet.add(field.toString());
+ }
+ StringBuilder fields = new StringBuilder();
+ for (String s : fieldSet) {
+ fields.append(s).append("\n");
+ }
+ assertEquals(
+ "description -> supercat\n" +
+ "documentid -> userdoc:kittens:1234:foo\n" +
+ "fluffiness -> 10000\n" +
+ "name -> megacat\n",
+ fields.toString());
+ }
+
+ @Test
+ public void deserializationExceptionsAreHandledGracefully() throws Exception {
+ Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
+ GetDocumentReply[] replies = new GetDocumentReply[] {
+ new MockFailingGetDocumentReply(doc1),
+ };
+ Chain<Searcher> searchChain = createSearcherChain(replies);
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1"));
+ assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<errors>\n" +
+ "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " +
+ "Got exception of type java.lang.RuntimeException during document " +
+ "deserialization: epic dragon attack\"/>\n"+
+ "</errors>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void testJsonRendererSetting() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ Chain<Searcher> searchChain = new Chain<>(searcher);
+
+ Query query = newQuery("?id=userdoc:kittens:1:2&format=json");
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+ assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate);
+ }
+
+ private Query newQuery(String queryString) {
+ return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+ }
+
+ private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
+ GetSearcher searcher = new GetSearcher(new FeedContext(
+ new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
+ factory, docMan, new ClusterList(), new NullFeedMetric()));
+ return new Chain<>(searcher);
+ }
+
+ private static class MockFailingGetDocumentReply extends GetDocumentReply {
+ private int countdown = 2;
+
+ private MockFailingGetDocumentReply(Document doc) {
+ super(doc);
+ }
+
+ @Override
+ public Document getDocument() {
+ // Reason for countdown is that the test DocumentSessionFactory calls
+ // getDocument once internally before code can ever reach handleReply.
+ if (--countdown == 0) {
+ throw new RuntimeException("epic dragon attack");
+ }
+ return super.getDocument();
+ }
+ }
+
+ private static class MockBackend extends Searcher {
+ private Hit hitToReturn;
+
+ public MockBackend(Hit hitToReturn) {
+ this.hitToReturn = hitToReturn;
+ }
+
+ public @Override Result search(Query query, Execution execution) {
+ Result result = new Result(query);
+ result.hits().add(hitToReturn);
+ return result;
+ }
+ }
+
+ private class MockHttpRequest {
+
+ private final String req;
+ private byte[] data;
+ private boolean gzip = false;
+
+ MockHttpRequest(byte[] data, String req) {
+ this.req = req;
+ this.data = data;
+ }
+
+ MockHttpRequest(byte[] data, String req, boolean gzip) {
+ this.data = data;
+ this.req = req;
+ this.gzip = gzip;
+ }
+
+ public InputStream getData() {
+ if (gzip) {
+ try {
+ ByteArrayOutputStream rawOut = new ByteArrayOutputStream();
+ GZIPOutputStream compressed = new GZIPOutputStream(rawOut);
+ compressed.write(data, 0, data.length);
+ compressed.finish();
+ compressed.flush();
+ rawOut.flush();
+ return new ByteArrayInputStream(rawOut.toByteArray());
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ return new ByteArrayInputStream(data);
+ }
+
+ public void addHeaders(HeaderFields headers) {
+ headers.add("Content-Type", "text/plain;encoding=UTF-8");
+ if (gzip)
+ headers.add("Content-Encoding", "gzip");
+ }
+
+ public com.yahoo.container.jdisc.HttpRequest toRequest() {
+ com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData());
+ addHeaders(request.getJDiscRequest().headers());
+ return request;
+ }
+
+ }
+
+ public static void assertRendered(String expected,Result result) throws Exception {
+ assertRendered(expected,result,true);
+ }
+
+ public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception {
+ if (checkFullEquality)
+ assertEquals(expected, ResultRenderingUtil.getRendered(result));
+ else
+ assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java
new file mode 100644
index 00000000000..21d39089d2a
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.prelude.templates.SearchRendererAdaptor;
+import com.yahoo.processing.rendering.Renderer;
+import com.yahoo.search.Result;
+import com.yahoo.search.rendering.DefaultRenderer;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+
+public class ResultRenderingUtil {
+
+ public static String getRendered(Result result) throws Exception {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ Charset cs = Charset.forName("utf-8");
+ CharsetDecoder decoder = cs.newDecoder();
+ SearchRendererAdaptor.callRender(stream, result);
+ stream.flush();
+ return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java
new file mode 100644
index 00000000000..820f7f56e2f
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java
@@ -0,0 +1,248 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.storage.searcher;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.cloud.config.ClusterListConfig;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.protect.Error;
+import com.yahoo.documentapi.VisitorSession;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.VisitorParameters;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.prelude.templates.DefaultTemplateSet;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vdslib.VisitorOrdering;
+import com.yahoo.vespaclient.ClusterList;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class VisitorSearcherTestCase {
+
+ private DocumentTypeManager docMan = null;
+ private DocumentType docType;
+ DocumentSessionFactory factory;
+
+ @org.junit.Before
+ public void setUp() {
+ docMan = new DocumentTypeManager();
+ docType = new DocumentType("kittens");
+ docType.addHeaderField("name", DataType.STRING);
+ docType.addField("description", DataType.STRING);
+ docType.addField("image", DataType.RAW);
+ docType.addField("fluffiness", DataType.INT);
+ docType.addField("foo", DataType.RAW);
+ docMan.registerDocumentType(docType);
+ factory = new DocumentSessionFactory(docType);
+ }
+
+ public VisitSearcher create() throws Exception {
+ ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar");
+ ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster));
+ ClusterList clusterList = new ClusterList();
+ clusterList.configure(clusterListCfg);
+ return new VisitSearcher(new FeedContext(
+ new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)),
+ new LoadTypeConfig(new LoadTypeConfig.Builder())),
+ factory, docMan, clusterList, new NullFeedMetric()));
+ }
+
+ @Test
+ public void testQueryParameters() throws Exception {
+ VisitSearcher searcher = create();
+ VisitorParameters params = searcher.getVisitorParameters(
+ newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" +
+ "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" +
+ "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" +
+ "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null);
+
+ assertEquals("id.user=1234", params.getDocumentSelection());
+ assertEquals(7, params.getMaxPending());
+ assertEquals(2, params.getMaxBucketsPerVisitor());
+ assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
+ assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString());
+ assertEquals("othercluster", params.getRemoteDataHandler());
+ assertEquals("[header]", params.fieldSet());
+ assertEquals(112, params.getFromTimestamp());
+ assertEquals(224, params.getToTimestamp());
+ assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering());
+ assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority());
+ assertEquals(7, params.getTraceLevel());
+ assertEquals(true, params.visitInconsistentBuckets());
+ assertEquals(true, params.visitRemoves());
+ }
+
+ @Test
+ public void timestampQueryParametersAreParsedAsLongs() throws Exception {
+ VisitorParameters params = create().getVisitorParameters(
+ newQuery("visit?visit.selection=id.user=1234&" +
+ "visit.fromTimestamp=1419021596000000&" +
+ "visit.toTimestamp=1419021597000000"), null);
+ assertEquals(1419021596000000L, params.getFromTimestamp());
+ assertEquals(1419021597000000L, params.getToTimestamp());
+ }
+
+ @Test
+ public void testQueryParametersDefaults() throws Exception {
+ VisitSearcher searcher = create();
+ VisitorParameters params = searcher.getVisitorParameters(
+ newQuery("visit?visit.selection=id.user=1234&hits=100"), null);
+
+ assertEquals("id.user=1234", params.getDocumentSelection());
+ assertEquals(1, params.getMaxBucketsPerVisitor());
+ assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
+ assertEquals(1, params.getMaxFirstPassHits());
+ assertEquals(1, params.getMaxTotalHits());
+ assertEquals(32, params.getMaxPending());
+ assertEquals(false, params.visitInconsistentBuckets());
+ }
+
+ @Test
+ public void testWrongCluster() throws Exception {
+ VisitSearcher searcher = create();
+
+ try {
+ searcher.getVisitorParameters(
+ newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null);
+
+ assertTrue(false);
+ } catch (Exception e) {
+ // e.printStackTrace();
+ }
+ }
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNoClusterParamWhenSeveralClusters() throws Exception {
+ DocumentSessionFactory factory = new DocumentSessionFactory(docType);
+ ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo");
+ ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar");
+ ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2)));
+ ClusterList clusterList = new ClusterList();
+ clusterList.configure(clusterListCfg);
+ VisitSearcher searcher = new VisitSearcher(new FeedContext(
+ new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)),
+ new LoadTypeConfig(new LoadTypeConfig.Builder())),
+ factory, docMan, clusterList, new NullFeedMetric()));
+
+ searcher.getVisitorParameters(
+ newQuery("visit?visit.selection=id.user=1234"), null);
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ Chain<Searcher> searchChain = new Chain<>(create());
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100"));
+ assertEquals(1, result.hits().size());
+ assertRendered(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" +
+ "</result>\n", result);
+ }
+
+ private Result invokeVisitRemovesSearchChain() throws Exception {
+ Chain<Searcher> searchChain = new Chain<>(create());
+ return new Execution(searchChain, Execution.Context.createContextStub()).search(
+ newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true"));
+ }
+
+ @Test
+ public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception {
+ Result result = invokeVisitRemovesSearchChain();
+ assertEquals(2, result.hits().size());
+ assertRendered(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" +
+ "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" +
+ "</result>\n", result);
+ }
+
+ @Test
+ public void removedDocumentIdsAreXmlEscaped() throws Exception {
+ factory = mock(DocumentSessionFactory.class);
+ when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> {
+ VisitorParameters params = (VisitorParameters)p.getArguments()[0];
+ DummyVisitorSession session = new DummyVisitorSession(params, docType);
+ session.clearAutoReplyMessages();
+ session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>");
+ return session;
+ });
+ Result result = invokeVisitRemovesSearchChain();
+ assertEquals(1, result.hits().size());
+ assertRendered(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<result>\n" +
+ "<remove documentid=\"userdoc:foo:1234:&lt;rem&quot;o&quot;ved&amp;stuff&gt;\"/>\n" +
+ "</result>\n", result);
+ }
+
+ private Result invokeSearcherWithUserQuery() throws Exception {
+ Chain<Searcher> searchChain = new Chain<>(create());
+ return new Execution(searchChain, Execution.Context.createContextStub())
+ .search(new Query("visit?visit.selection=id.user=1234&hits=100"));
+ }
+
+ @Test
+ public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception {
+ VisitorSession session = mock(VisitorSession.class);
+ when(session.waitUntilDone(anyLong())).thenReturn(false);
+ factory = mock(DocumentSessionFactory.class);
+ when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session);
+
+ Result result = invokeSearcherWithUserQuery();
+ assertNotNull(result.hits().getErrorHit());
+ assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode());
+ }
+
+ @Test
+ public void testRendererWiring() throws Exception {
+ Chain<Searcher> searchChain = new Chain<>(create());
+ {
+ Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json");
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+ assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass());
+ }
+ {
+ Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer");
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+ assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass());
+ }
+ {
+ Query query = newQuery("visit?visit.selection=id.user=1234&hits=100");
+ Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
+ assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass());
+ }
+ }
+
+ public static void assertRendered(String expected, Result result) throws Exception {
+ assertEquals(expected, ResultRenderingUtil.getRendered(result));
+ }
+
+ private Query newQuery(String queryString) {
+ return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java
new file mode 100644
index 00000000000..d19560ee553
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.jdisc.Metric;
+
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.20
+ */
+class DummyMetric implements Metric {
+
+ @Override
+ public void set(String key, Number val, Context ctx) {
+ }
+
+ @Override
+ public void add(String key, Number val, Context ctx) {
+ }
+
+ @Override
+ public Context createContext(Map<String, ?> properties) {
+ return DummyContext.INSTANCE;
+ }
+
+ private static class DummyContext implements Context {
+ private static final DummyContext INSTANCE = new DummyContext();
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java
new file mode 100644
index 00000000000..2cabed13a7b
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FeedHandlerCompressionTest {
+
+ public static byte[] compress(final String dataToBrCompressed) throws IOException {
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(dataToBrCompressed.length());
+ final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);
+ gzipOutputStream.write(dataToBrCompressed.getBytes());
+ gzipOutputStream.close();
+ byte[] compressedBytes = byteArrayOutputStream.toByteArray();
+ byteArrayOutputStream.close();
+ return compressedBytes;
+ }
+
+ @Test
+ public void testUnzipStreamIfNeeded() throws Exception {
+ final String testData = "foo bar";
+ InputStream inputStream = new ByteArrayInputStream(compress(testData));
+ HttpRequest httpRequest = mock(HttpRequest.class);
+ when(httpRequest.getHeader("content-encoding")).thenReturn("gzip");
+ InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest);
+ final StringBuilder processedInput = new StringBuilder();
+ while (true) {
+ int readValue = decompressedStream.read();
+ if (readValue < 0) {
+ break;
+ }
+ processedInput.append((char)readValue);
+ }
+ assertThat(processedInput.toString(), is(testData));
+ }
+
+ /**
+ * Test by setting encoding, but not compressing data.
+ * @throws Exception
+ */
+ @Test
+ public void testUnzipFails() throws Exception {
+ final String testData = "foo bar";
+ InputStream inputStream = new ByteArrayInputStream(testData.getBytes());
+ HttpRequest httpRequest = mock(HttpRequest.class);
+ when(httpRequest.getHeader("Content-Encoding")).thenReturn("gzip");
+ InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest);
+ final StringBuilder processedInput = new StringBuilder();
+ while (true) {
+ int readValue = decompressedStream.read();
+ if (readValue < 0) {
+ break;
+ }
+ processedInput.append((char)readValue);
+ }
+ assertThat(processedInput.toString(), is(testData));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java
new file mode 100644
index 00000000000..ae96a25fad3
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.text.Utf8;
+
+import java.io.ByteArrayInputStream;
+
+/**
+ * Stream with extra data outside the actual input stream.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public final class MetaStream extends ByteArrayInputStream {
+
+ private byte[] operations;
+ int i;
+
+ public MetaStream(byte[] buf) {
+ super(createPayload(buf));
+ this.operations = buf;
+ i = 0;
+ }
+
+ private static final byte[] createPayload(byte[] buf) {
+ StringBuilder bu = new StringBuilder();
+ for (byte b : buf) {
+ bu.append("id:banana:banana::doc1 0\n");
+ }
+ return Utf8.toBytes(bu.toString());
+ }
+
+ public byte getNextOperation() {
+ if (i >= operations.length) {
+ return 0;
+ }
+ return operations[i++];
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java
new file mode 100644
index 00000000000..1208ce334d9
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import java.util.List;
+
+import com.yahoo.jrt.slobrok.api.IMirror;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.network.Network;
+import com.yahoo.messagebus.network.NetworkOwner;
+import com.yahoo.messagebus.routing.RoutingNode;
+
+/**
+ * NOP-network.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class MockNetwork implements Network {
+
+ @Override
+ public boolean waitUntilReady(double seconds) {
+ return true;
+ }
+
+ @Override
+ public void attach(NetworkOwner owner) {
+ }
+
+ @Override
+ public void registerSession(String session) {
+ }
+
+ @Override
+ public void unregisterSession(String session) {
+
+ }
+
+ @Override
+ public boolean allocServiceAddress(RoutingNode recipient) {
+ return true;
+ }
+
+ @Override
+ public void freeServiceAddress(RoutingNode recipient) {
+
+ }
+
+ @Override
+ public void send(Message msg, List<RoutingNode> recipients) {
+ }
+
+ @Override
+ public void sync() {
+ }
+
+ @Override
+ public void shutdown() {
+ }
+
+ @Override
+ public String getConnectionSpec() {
+ return null;
+ }
+
+ @Override
+ public IMirror getMirror() {
+ return null;
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java
new file mode 100644
index 00000000000..298c925d032
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.messagebus.Reply;
+import com.yahoo.text.Utf8String;
+
+/**
+ * Minimal reply simulator.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class MockReply extends Reply {
+
+ Object context;
+
+ public MockReply(Object context) {
+ this.context = context;
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return null;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public Object getContext() {
+ return context;
+ }
+
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java
new file mode 100644
index 00000000000..2e88c440b12
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java
@@ -0,0 +1,236 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.jdisc.References;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.shared.SharedMessageBus;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8String;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespaxmlparser.MockFeedReaderFactory;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Check FeedHandler APIs.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class V2ErrorsInResultTestCase {
+
+ LessConfiguredHandler handler;
+ ExecutorService workers;
+
+ @Before
+ public void setUp() throws Exception {
+ workers = Executors.newCachedThreadPool();
+ handler = new LessConfiguredHandler(workers);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ handler.destroy();
+ workers.shutdown();
+ }
+
+ private static class LessConfiguredHandler extends FeedHandler {
+
+ public LessConfiguredHandler(Executor executor) throws Exception {
+ super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null);
+ }
+
+
+ @Override
+ protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream,
+ BlockingQueue<OperationStatus> operations, String clientId,
+ boolean sessionIdWasGeneratedJustNow, int protocolVersion)
+ throws Exception {
+ return new LessConfiguredFeeder(requestInputStream, operations,
+ popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow,
+ sourceSessionParams(request), null, this, this.feedReplyHandler, "");
+ }
+
+ @Override
+ protected DocumentTypeManager createDocumentManager(
+ DocumentmanagerConfig documentManagerConfig) {
+ return null;
+ }
+ }
+
+ private static class MockSharedSession extends SharedSourceSession {
+ int count;
+
+ public MockSharedSession(SourceSessionParams params) {
+ super(new SharedMessageBus(new MessageBus(new MockNetwork(),
+ new MessageBusParams())), params);
+ count = 0;
+ }
+
+ @Override
+ public Result sendMessageBlocking(Message msg) throws InterruptedException {
+ return sendMessage(msg);
+ }
+
+ @Override
+ public Result sendMessage(Message msg) {
+ Result r;
+ ReplyHandler handler = msg.popHandler();
+
+ switch (count++) {
+ case 0:
+ r = new Result(ErrorCode.FATAL_ERROR,
+ "boom");
+ break;
+ case 1:
+ r = new Result(ErrorCode.TRANSIENT_ERROR,
+ "transient boom");
+ break;
+ case 2:
+ final FailedReply reply = new FailedReply(msg.getContext());
+ reply.addError(new Error(
+ ErrorCode.FATAL_ERROR,
+ "bad mojo, dude"));
+ handler.handleReply(reply);
+ r = Result.ACCEPTED;
+ break;
+ default:
+ handler.handleReply(new MockReply(msg.getContext()));
+ r = Result.ACCEPTED;
+ }
+ return r;
+ }
+
+ }
+
+ private static class FailedReply extends Reply {
+ Object context;
+
+ public FailedReply(Object context) {
+ this.context = context;
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return null;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public Object getContext() {
+ return context;
+ }
+ }
+
+ private static class LessConfiguredFeeder extends Feeder {
+
+ public LessConfiguredFeeder(InputStream stream,
+ BlockingQueue<OperationStatus> operations,
+ ClientState storedState, FeederSettings settings,
+ String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams,
+ SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler,
+ String localHostname) throws Exception {
+ super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow,
+ sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname);
+ }
+
+ protected ReferencedResource<SharedSourceSession> retainSession(
+ SourceSessionParams sessionParams, SessionCache sessionCache) {
+ final SharedSourceSession session = new MockSharedSession(sessionParams);
+ return new ReferencedResource<>(session, References.fromResource(session));
+ }
+ }
+
+ @Test
+ public final void test() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 ERROR boom \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR transient{20}boom \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 ERROR bad{20}mojo,{20}dude \n",
+ Utf8.toString(out.toByteArray()));
+ }
+
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java
new file mode 100644
index 00000000000..74ed844d69f
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java
@@ -0,0 +1,535 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.jdisc.References;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.MessageBus;
+import com.yahoo.messagebus.MessageBusParams;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.Result;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.network.Network;
+import com.yahoo.messagebus.shared.SharedMessageBus;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.http.client.config.FeedParams.DataFormat;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespaxmlparser.MockFeedReaderFactory;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Check FeedHandler APIs.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class V2ExternalFeedTestCase {
+
+ LessConfiguredHandler handler;
+ ExecutorService workers;
+ Level logLevel;
+ Logger logger;
+ boolean initUseParentHandlers;
+ LogBuffer logChecker;
+
+ @Before
+ public void setUp() throws Exception {
+ workers = Executors.newCachedThreadPool();
+ handler = new LessConfiguredHandler(workers);
+ logger = Logger.getLogger(Feeder.class.getName());
+ logLevel = logger.getLevel();
+ logger.setLevel(Level.ALL);
+ initUseParentHandlers = logger.getUseParentHandlers();
+ logChecker = new LogBuffer();
+ logger.setUseParentHandlers(false);
+ logger.addHandler(logChecker);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ handler.destroy();
+ workers.shutdown();
+ logger.setLevel(logLevel);
+ logger.removeHandler(logChecker);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ private static class LogBuffer extends Handler {
+ public final BlockingQueue<LogRecord> records = new LinkedBlockingQueue<>();
+
+ @Override
+ public void publish(LogRecord record) {
+ try {
+ records.put(record);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws SecurityException {
+ }
+ }
+
+ private static class LessConfiguredHandler extends FeedHandler {
+ volatile DataFormat lastFormatSeen;
+
+ public LessConfiguredHandler(Executor executor) throws Exception {
+ super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null);
+ }
+
+ @Override
+ protected Feeder createFeeder(HttpRequest request,
+ InputStream requestInputStream,
+ BlockingQueue<OperationStatus> operations,
+ String clientId,
+ boolean sessionIdWasGeneratedJustNow, int protocolVersion)
+ throws Exception {
+ LessConfiguredFeeder f = new LessConfiguredFeeder(requestInputStream, operations,
+ popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow,
+ sourceSessionParams(request), null, this, this.feedReplyHandler, "ourHostname");
+ lastFormatSeen = f.settings.dataFormat;
+ return f;
+ }
+
+ @Override
+ protected DocumentTypeManager createDocumentManager(
+ DocumentmanagerConfig documentManagerConfig) {
+ return null;
+ }
+ }
+
+ private static class MockSharedSession extends SharedSourceSession {
+
+ public MockSharedSession(SourceSessionParams params) {
+ super(new SharedMessageBus(new MessageBus(new MockNetwork(),
+ new MessageBusParams())), params);
+ }
+
+ @Override
+ public Result sendMessageBlocking(Message msg) throws InterruptedException {
+ return sendMessage(msg);
+ }
+
+ @Override
+ public Result sendMessage(Message msg) {
+ ReplyHandler handler = msg.popHandler();
+ MockReply mockReply = new MockReply(msg.getContext());
+ if (msg instanceof Feeder.FeedErrorMessage) {
+ mockReply.addError(new com.yahoo.messagebus.Error(123, "Could not feed this"));
+ }
+ if (msg instanceof PutDocumentMessage) {
+ assert(msg.getTrace().getLevel() == 4);
+ assert(((PutDocumentMessage) msg).getPriority().name().equals("LOWEST"));
+ }
+ handler.handleReply(mockReply);
+ return Result.ACCEPTED;
+ }
+
+ }
+
+ private static class LessConfiguredFeeder extends Feeder {
+ public LessConfiguredFeeder(InputStream stream,
+ BlockingQueue<OperationStatus> operations,
+ ClientState storedState, FeederSettings settings,
+ String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams,
+ SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler,
+ String localHostname) throws Exception {
+ super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow,
+ sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname);
+ }
+
+ protected ReferencedResource<SharedSourceSession> retainSession(
+ SourceSessionParams sessionParams, SessionCache sessionCache) {
+ final SharedSourceSession session = new MockSharedSession(sessionParams);
+ return new ReferencedResource<>(session, References.fromResource(session));
+ }
+ }
+
+ @Test
+ public final void test() throws IOException, InterruptedException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n",
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding());
+ assertEquals(7, logChecker.records.size());
+ String actualHandshake = logChecker.records.take().getMessage();
+ assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true));
+ assertEquals("Successfully deserialized document id: id:banana:banana::doc1",
+ logChecker.records.take().getMessage());
+ assertEquals("Sent message successfully, document id: id:banana:banana::doc1",
+ logChecker.records.take().getMessage());
+ }
+
+ //test session ID without #, i.e. something fishy related to VIPs is going on
+ sessionId = "something";
+
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "2");
+
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ String expectedErrorMsg = "Got request from client with id 'something', but found no session for this client. " +
+ "Most probably this server is in VIP rotation, and a client session was rotated from one " +
+ "server to another. This must not happen. Configure VIP with persistence=enabled, or " +
+ "(preferably) do not use a VIP at all.";
+ assertEquals(expectedErrorMsg,
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding());
+ }
+
+ //test session ID with trailing # but no hostname
+ sessionId = "something#";
+
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ String expectedErrorMsg = "Got request from client with id 'something#', but found no session for this " +
+ "client. Possible session timeout due to inactivity, server restart or " +
+ "reconfig, or bad VIP usage.";
+ assertEquals(expectedErrorMsg,
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding());
+ }
+
+ //test session ID with trailing # and some unknown hostname at the end
+ sessionId = "something#thisHostnameDoesNotExistAnywhere";
+
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ String expectedErrorMsg = "Got request from client with id 'something#thisHostnameDoesNotExistAnywhere', " +
+ "but found no session for this client. Session was originally established " +
+ "towards host thisHostnameDoesNotExistAnywhere, but our hostname is " +
+ "ourHostname. Most probably this server is in VIP rotation, and a session " +
+ "was rotated from one server to another. This should not happen. Configure VIP " +
+ "with persistence=enabled, or (preferably) do not use a VIP at all.";
+ assertEquals(expectedErrorMsg,
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding());
+ }
+
+ //test session ID with trailing # and some unknown hostname at the end
+ sessionId = "something#ourHostname";
+
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" +
+ "id:banana:banana::doc1 OK Document{20}processed. \n" +
+ "id:banana:banana::doc1 OK Document{20}processed. \n",
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding());
+ Thread.sleep(1000);
+ }
+ }
+
+ @Test
+ public final void testFailedReading() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 4 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ }
+
+ @Test
+ public final void testCleaningDoesNotBlowUp() throws IOException {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ handler.forceRunCleanClients();
+ }
+
+ @Test
+ public final void testMockNetworkDoesNotBlowUp() {
+ Network n = new MockNetwork();
+ n.registerSession(null);
+ n.unregisterSession(null);
+ assertTrue(n.allocServiceAddress(null));
+ n.freeServiceAddress(null);
+ n.send(null, null);
+ assertNull(n.getConnectionSpec());
+ assertNull(n.getMirror());
+ }
+
+ @Test
+ public final void testMockReplyDoesNotBlowUp() {
+ MockReply r = new MockReply(null);
+ assertNull(r.getProtocol());
+ assertEquals(0, r.getType());
+ assertFalse(r.hasFatalErrors());
+ }
+
+ @Test
+ public final void testFlush() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1, 1, 1, 1, 1, 1, 1});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ }
+
+ @Test
+ public final void testIllegalVersion() throws IOException {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers()
+ .add(Headers.VERSION, Integer.toString(Integer.MAX_VALUE));
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals(Headers.HTTP_NOT_ACCEPTABLE, r.getStatus());
+ }
+
+ @Test
+ public final void testSettings() {
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ nalle.getJDiscRequest().headers().add(Headers.ROUTE, "bamse brakar");
+ nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true");
+ FeederSettings settings = new FeederSettings(nalle);
+ assertEquals(false, settings.drain);
+ assertEquals(2, settings.route.getNumHops());
+ assertEquals(true, settings.denyIfBusy);
+ }
+
+ @Test
+ public final void testJsonInputFormat() throws IOException, InterruptedException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[]{1, 3, 2});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DATA_FORMAT, DataFormat.JSON_UTF8.name());
+ nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST");
+ nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n"
+ + "id:banana:banana::doc1 OK Document{20}processed. \n",
+ Utf8.toString(out.toByteArray()));
+ assertEquals("text/plain", r.getContentType());
+ assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding());
+ assertEquals(7, logChecker.records.size());
+ String actualHandshake = logChecker.records.take().getMessage();
+ assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true));
+ assertEquals("Successfully deserialized document id: id:banana:banana::doc1",
+ logChecker.records.take().getMessage());
+ assertEquals("Sent message successfully, document id: id:banana:banana::doc1",
+ logChecker.records.take().getMessage());
+ assertSame(DataFormat.JSON_UTF8, handler.lastFormatSeen);
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java
new file mode 100644
index 00000000000..4e6ca17abef
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java
@@ -0,0 +1,224 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.jdisc.References;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.shared.SharedMessageBus;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+import com.yahoo.vespaxmlparser.MockFeedReaderFactory;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Check FeedHandler APIs.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class V2FailingMessagebusTestCase {
+
+ LessConfiguredHandler handler;
+ ExecutorService workers;
+ int mbus;
+
+ @Before
+ public void setUp() throws Exception {
+ workers = Executors.newCachedThreadPool();
+ handler = new LessConfiguredHandler(workers);
+ mbus = 0;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ handler.destroy();
+ workers.shutdown();
+ mbus = 0;
+ }
+
+ private class LessConfiguredHandler extends FeedHandler {
+
+ public LessConfiguredHandler(Executor executor) throws Exception {
+ super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null);
+ }
+
+ @Override
+ protected Feeder createFeeder(HttpRequest request,
+ InputStream requestInputStream,
+ BlockingQueue<OperationStatus> operations,
+ String clientId,
+ boolean sessionIdWasGeneratedJustNow, int protocolVersion) throws Exception {
+ return new LessConfiguredFeeder(requestInputStream, operations,
+ popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow,
+ sourceSessionParams(request), null, this, this.feedReplyHandler, "");
+ }
+
+ @Override
+ protected DocumentTypeManager createDocumentManager(
+ DocumentmanagerConfig documentManagerConfig) {
+ return null;
+ }
+ }
+
+ private class MockSharedSession extends SharedSourceSession {
+
+ public MockSharedSession(SourceSessionParams params) {
+ super(new SharedMessageBus(new MessageBus(new MockNetwork(),
+ new MessageBusParams())), params);
+ }
+
+ @Override
+ public Result sendMessageBlocking(Message msg) throws InterruptedException {
+ return sendMessage(msg);
+ }
+
+ @Override
+ public Result sendMessage(Message msg) {
+ ReplyHandler handler = msg.popHandler();
+
+ switch (mbus) {
+ case 0:
+ throw new RuntimeException("boom");
+ case 1:
+ Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "tralala");
+ mbus = 2;
+ return r;
+ case 2:
+ handler.handleReply(new MockReply(msg.getContext()));
+ return Result.ACCEPTED;
+ default:
+ throw new IllegalStateException("WTF?!");
+ }
+ }
+ }
+
+ private class LessConfiguredFeeder extends Feeder {
+
+ public LessConfiguredFeeder(InputStream inputStream,
+ BlockingQueue<OperationStatus> operations,
+ ClientState storedState, FeederSettings settings,
+ String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams,
+ SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler,
+ String localHostname) throws Exception {
+ super(inputStream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow,
+ sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname);
+ }
+
+ protected ReferencedResource<SharedSourceSession> retainSession(
+ SourceSessionParams sessionParams, SessionCache sessionCache) {
+ final SharedSourceSession session = new MockSharedSession(sessionParams);
+ return new ReferencedResource<>(session, References.fromResource(session));
+ }
+ }
+
+ @Test
+ public final void testFailingMbus() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[]{1});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[]{1});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 ERROR boom \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ }
+
+ @Test
+ public final void testBusyMbus() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[]{1});
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ mbus = 2;
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ mbus = 1;
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ nalle.getJDiscRequest().headers()
+ .add(Headers.DENY_IF_BUSY, "false");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ mbus = 1;
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR tralala \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java
new file mode 100644
index 00000000000..1b2d855bdf2
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java
@@ -0,0 +1,162 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.jdisc.References;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.shared.SharedMessageBus;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespa.http.client.core.OperationStatus;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Check FeedHandler APIs.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class V2NoXmlReaderTestCase {
+
+ LessConfiguredHandler handler;
+ ExecutorService workers;
+
+ @Before
+ public void setUp() throws Exception {
+ workers = Executors.newCachedThreadPool();
+ handler = new LessConfiguredHandler(workers);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ handler.destroy();
+ workers.shutdown();
+ }
+
+ private static class LessConfiguredHandler extends FeedHandler {
+
+ public LessConfiguredHandler(Executor executor) throws Exception {
+ super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null);
+ }
+
+
+ @Override
+ protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream,
+ BlockingQueue<OperationStatus> operations, String clientId,
+ boolean sessionIdWasGeneratedJustNow, int protocolVersion)
+ throws Exception {
+ return new LessConfiguredFeeder(requestInputStream, operations,
+ popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow,
+ sourceSessionParams(request), null, this, this.feedReplyHandler, "");
+ }
+
+ @Override
+ protected DocumentTypeManager createDocumentManager(
+ DocumentmanagerConfig documentManagerConfig) {
+ return null;
+ }
+ }
+
+ private static class MockSharedSession extends SharedSourceSession {
+
+ public MockSharedSession(SourceSessionParams params) {
+ super(new SharedMessageBus(new MessageBus(new MockNetwork(),
+ new MessageBusParams())), params);
+ }
+
+ @Override
+ public Result sendMessageBlocking(Message msg) throws InterruptedException {
+ return sendMessage(msg);
+ }
+
+ @Override
+ public Result sendMessage(Message msg) {
+ ReplyHandler handler = msg.popHandler();
+ MockReply mockReply = new MockReply(msg.getContext());
+ if (msg instanceof Feeder.FeedErrorMessage) {
+ mockReply.addError(new Error(123, "Could not feed this"));
+ }
+ handler.handleReply(mockReply);
+ return Result.ACCEPTED;
+ }
+
+ }
+
+ private static class LessConfiguredFeeder extends Feeder {
+
+ public LessConfiguredFeeder(InputStream inputStream,
+ BlockingQueue<OperationStatus> operations,
+ ClientState storedState, FeederSettings settings,
+ String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams,
+ SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler,
+ String localHostname) throws Exception {
+ super(inputStream, null, null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow,
+ sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname);
+ }
+
+ protected ReferencedResource<SharedSourceSession> retainSession(
+ SourceSessionParams sessionParams, SessionCache sessionCache) {
+ final SharedSourceSession session = new MockSharedSession(sessionParams);
+ return new ReferencedResource<>(session, References.fromResource(session));
+ }
+ }
+
+ @Test
+ public final void test() throws IOException {
+ String sessionId;
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest
+ .createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false");
+ HttpResponse r = handler.handle(nalle);
+ sessionId = r.headers().getFirst(Headers.SESSION_ID);
+ r.render(out);
+ assertEquals("",
+ Utf8.toString(out.toByteArray()));
+ }
+ {
+ InputStream in = new MetaStream(new byte[] { 1 });
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ HttpRequest nalle = HttpRequest.createTestRequest(
+ "http://test4-steinar:19020/reserved-for-internal-use/feedapi",
+ Method.POST, in);
+ nalle.getJDiscRequest().headers().add(Headers.VERSION, "2");
+ nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId);
+ nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true");
+ HttpResponse r = handler.handle(nalle);
+ r.render(out);
+ //This is different from v1. If we cannot parse XML, we will still get response code 200, but with a sensible
+ //error message in the response.
+ assertEquals(200, r.getStatus());
+ assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n",
+ Utf8.toString(out.toByteArray()));
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java
new file mode 100644
index 00000000000..73095a3efe9
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.ReferencedResource;
+import com.yahoo.jdisc.References;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.MessageBus;
+import com.yahoo.messagebus.MessageBusParams;
+import com.yahoo.messagebus.ReplyHandler;
+import com.yahoo.messagebus.Result;
+import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.shared.SharedMessageBus;
+import com.yahoo.messagebus.shared.SharedSourceSession;
+import com.yahoo.vespa.http.client.core.Headers;
+import com.yahoo.vespaxmlparser.MockFeedReaderFactory;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertTrue;
+
+
+public class V3CongestionTestCase {
+ AtomicInteger threadsAvail = new AtomicInteger(10);
+ AtomicInteger requests = new AtomicInteger(0);
+
+
+ static class ClientFeederWithMocks extends ClientFeederV3 {
+
+ private final DocumentOperationMessageV3 docOp;
+
+ ClientFeederWithMocks(ReferencedResource<SharedSourceSession> sourceSession, FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager, String clientId, Metric metric, ReplyHandler feedReplyHandler, AtomicInteger threadsAvailableForFeeding) {
+ super(sourceSession, feedReaderFactory, docTypeManager, clientId, metric, feedReplyHandler, threadsAvailableForFeeding);
+ // The operation to return from the client feeder.
+ VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+ docOp = DocumentOperationMessageV3.newRemoveMessage(op, "operation id");
+
+ }
+
+ @Override
+ protected DocumentOperationMessageV3 getNextMessage(
+ String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception {
+ while (true) {
+ int data = requestInputStream.read();
+ if (data == -1 || data == (char)'\n') {
+ break;
+ }
+ }
+ return docOp;
+ }
+ }
+
+ final static int NUMBER_OF_QUEUE_FULL_RESPONSES = 5;
+
+ ClientFeederV3 clientFeederV3;
+ HttpRequest request;
+
+ @Before
+ public void setup() {
+ // Set up a request to be used from the tests.
+ InputStream in = new MetaStream(new byte[] { 1 });
+ request = HttpRequest
+ .createTestRequest(
+ "http://foo.bar:19020/reserved-for-internal-use/feedapi",
+ com.yahoo.jdisc.http.HttpRequest.Method.POST, in);
+ request.getJDiscRequest().headers().add(Headers.VERSION, "3");
+ request.getJDiscRequest().headers().add(Headers.CLIENT_ID, "clientId");
+
+
+ // Create a mock that does not parse the message, only reads the rest of the line. Makes it easier
+ // to write tests. It uses a mock for message bus.
+ clientFeederV3 = new ClientFeederWithMocks(
+ retainMockSession(new SourceSessionParams(), requests),
+ new MockFeedReaderFactory(),
+ null /*DocTypeManager*/,
+ "clientID",
+ null/*metric*/,
+ new FeedReplyReader(null/*metric*/),
+ threadsAvail);
+ }
+
+ // A mock for message bus that can simulate blocking requests.
+ private static class MockSharedSession extends SharedSourceSession {
+ boolean queuFull = true;
+ AtomicInteger requests;
+
+ public MockSharedSession(SourceSessionParams params, AtomicInteger requests) {
+ super(new SharedMessageBus(new MessageBus(new MockNetwork(),
+ new MessageBusParams())), params);
+ this.requests = requests;
+ }
+
+ @Override
+ public Result sendMessageBlocking(Message msg) throws InterruptedException {
+ return sendMessage(msg);
+ }
+
+ @Override
+ public Result sendMessage(Message msg) {
+ ReplyHandler handler = msg.popHandler();
+ if (queuFull) {
+ requests.incrementAndGet();
+ // Disable queue full after some attempts
+ if (requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES) {
+ queuFull = false;
+ }
+ Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "queue full");
+ return r;
+ }
+
+ handler.handleReply(new MockReply(msg.getContext()));
+ return Result.ACCEPTED;
+ }
+ }
+
+ ReferencedResource<SharedSourceSession> retainMockSession(
+ SourceSessionParams sessionParams,
+ AtomicInteger requests) {
+ final SharedSourceSession session = new MockSharedSession(sessionParams, requests);
+ return new ReferencedResource<>(session, References.fromResource(session));
+ }
+
+ @Test
+ public void testRetriesWhenThreadsAvailable() throws IOException {
+ request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true");
+ threadsAvail.set(10);
+
+ clientFeederV3.handleRequest(request);
+ assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES);
+ }
+
+ @Test
+ public void testNoRetriesWhenNoThreadsAvailable() throws IOException {
+ request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true");
+ threadsAvail.set(0);
+
+ clientFeederV3.handleRequest(request);
+ assertTrue(requests.get() == 1);
+ }
+
+ @Test
+ public void testRetriesWhenNoThreadsAvailableButNoDenyIfBusy() throws IOException {
+ request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "false");
+ threadsAvail.set(0);
+
+ clientFeederV3.handleRequest(request);
+ assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES);
+ }
+} \ No newline at end of file
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java
new file mode 100644
index 00000000000..df1de5d75ed
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.container.jdisc.HttpResponse;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.7.0
+ */
+public class VersionsTestCase {
+
+ private static final List<String> EMPTY = Collections.emptyList();
+ private static final List<String> ONE_TWO = Arrays.asList("1", "2");
+ private static final List<String> TWO_THREE = Arrays.asList("3", "2");
+ private static final List<String> ONE_NULL_TWO = Arrays.asList("1", null, "2");
+ private static final List<String> ONE_COMMA_TWO = Collections.singletonList("1, 2");
+ private static final List<String> ONE_EMPTY_TWO = Arrays.asList("1", "", "2");
+ private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000");
+ private static final List<String> TWO_TOO_LARGE_NUMBER = Arrays.asList("2", "1000000000");
+ private static final List<String> TWO_COMMA_TOO_LARGE_NUMBER = Arrays.asList("2,1000000000");
+ private static final List<String> GARBAGE = Collections.singletonList("garbage");
+
+ @Test
+ public void testEmpty() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(EMPTY);
+ assertThat(v.first, instanceOf(ErrorHttpResponse.class));
+ assertThat(v.second, is(-1));
+ }
+
+ @Test
+ public void testOneTwo() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testTwoThree() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_THREE);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(3));
+ }
+
+ @Test
+ public void testOneNullTwo() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_TWO);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testOneCommaTwo() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_TWO);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testOneEmptyTwo() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_TWO);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testTooLarge() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TOO_LARGE_NUMBER);
+ assertThat(v.first, instanceOf(ErrorHttpResponse.class));
+ assertThat(v.second, is(-1));
+ }
+
+ @Test
+ public void testTwoTooLarge() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_TOO_LARGE_NUMBER);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testTwoCommaTooLarge() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_COMMA_TOO_LARGE_NUMBER);
+ assertThat(v.first, nullValue());
+ assertThat(v.second, is(2));
+ }
+
+ @Test
+ public void testGarbage() throws Exception {
+ Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(GARBAGE);
+ assertThat(v.first, instanceOf(ErrorHttpResponse.class));
+ assertThat(v.second, is(-1));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java
new file mode 100644
index 00000000000..b134b5c34cd
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.http.server.util;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.23
+ */
+public class ByteLimitedInputStreamTestCase {
+
+ private static ByteLimitedInputStream create(byte[] source, int limit) {
+ if (limit > source.length) {
+ throw new IllegalArgumentException("Limit is greater than length of source buffer.");
+ }
+ InputStream wrappedStream = new ByteArrayInputStream(source);
+ return new ByteLimitedInputStream(wrappedStream, limit);
+ }
+
+ @Test
+ public void requireThatBasicsWork() throws IOException {
+ ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9);
+
+ assertThat(stream.available(), is(9));
+ assertThat(stream.read(), is(97));
+ assertThat(stream.available(), is(8));
+ assertThat(stream.read(), is(98));
+ assertThat(stream.available(), is(7));
+ assertThat(stream.read(), is(99));
+ assertThat(stream.available(), is(6));
+ assertThat(stream.read(), is(100));
+ assertThat(stream.available(), is(5));
+ assertThat(stream.read(), is(101));
+ assertThat(stream.available(), is(4));
+ assertThat(stream.read(), is(102));
+ assertThat(stream.available(), is(3));
+ assertThat(stream.read(), is(103));
+ assertThat(stream.available(), is(2));
+ assertThat(stream.read(), is(104));
+ assertThat(stream.available(), is(1));
+ assertThat(stream.read(), is(105));
+ assertThat(stream.available(), is(0));
+ assertThat(stream.read(), is(-1));
+ assertThat(stream.available(), is(0));
+ assertThat(stream.read(), is(-1));
+ assertThat(stream.available(), is(0));
+ assertThat(stream.read(), is(-1));
+ assertThat(stream.available(), is(0));
+ assertThat(stream.read(), is(-1));
+ assertThat(stream.available(), is(0));
+ assertThat(stream.read(), is(-1));
+ assertThat(stream.available(), is(0));
+ }
+
+ @Test
+ public void requireThatChunkedReadWorks() throws IOException {
+ ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9);
+
+ assertThat(stream.available(), is(9));
+ byte[] toBuf = new byte[4];
+ assertThat(stream.read(toBuf), is(4));
+ assertThat(toBuf[0], is((byte) 97));
+ assertThat(toBuf[1], is((byte) 98));
+ assertThat(toBuf[2], is((byte) 99));
+ assertThat(toBuf[3], is((byte) 100));
+ assertThat(stream.available(), is(5));
+
+ assertThat(stream.read(toBuf), is(4));
+ assertThat(toBuf[0], is((byte) 101));
+ assertThat(toBuf[1], is((byte) 102));
+ assertThat(toBuf[2], is((byte) 103));
+ assertThat(toBuf[3], is((byte) 104));
+ assertThat(stream.available(), is(1));
+
+ assertThat(stream.read(toBuf), is(1));
+ assertThat(toBuf[0], is((byte) 105));
+ assertThat(stream.available(), is(0));
+
+ assertThat(stream.read(toBuf), is(-1));
+ assertThat(stream.available(), is(0));
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java
new file mode 100644
index 00000000000..c245490b1d7
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.vespa.http.client.config.FeedParams;
+import com.yahoo.vespa.http.server.FeedReaderFactory;
+
+import java.io.InputStream;
+
+/**
+ * For creating MockReader of innput stream.
+ * @author dybdahl
+ */
+public class MockFeedReaderFactory extends FeedReaderFactory {
+
+ @Override
+ public FeedReader createReader(
+ InputStream inputStream,
+ DocumentTypeManager docTypeManager,
+ FeedParams.DataFormat dataFormat) {
+ try {
+ return new MockReader(inputStream);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java
new file mode 100644
index 00000000000..a4ac0f4fdaf
--- /dev/null
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.vespa.http.server.MetaStream;
+import com.yahoo.vespa.http.server.util.ByteLimitedInputStream;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation;
+
+import java.io.InputStream;
+import java.lang.reflect.Field;
+
+/**
+ * Mock for ExternalFeedTestCase which had to override package private methods.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class MockReader implements FeedReader {
+
+ MetaStream stream;
+ boolean finished = false;
+
+ public MockReader(InputStream stream) throws Exception {
+ this.stream = getMetaStream(stream);
+ }
+
+ private static MetaStream getMetaStream(InputStream stream) {
+ if (stream instanceof MetaStream) {
+ return (MetaStream) stream;
+ }
+ if (!(stream instanceof ByteLimitedInputStream)) {
+ throw new IllegalStateException("Given unknown stream type.");
+ }
+ //Ooooooo this is so ugly
+ try {
+ ByteLimitedInputStream byteLimitedInputStream = (ByteLimitedInputStream) stream;
+ Field f = byteLimitedInputStream.getClass().getDeclaredField("wrappedStream"); //NoSuchFieldException
+ f.setAccessible(true);
+ return (MetaStream) f.get(byteLimitedInputStream);
+ } catch (Exception e) {
+ throw new IllegalStateException("Implementation of ByteLimitedInputStream has changed.", e);
+ }
+ }
+
+ @Override
+ public void read(Operation operation) throws Exception {
+ if (finished) {
+ return;
+ }
+
+ byte whatToDo = stream.getNextOperation();
+ DocumentId id = new DocumentId("id:banana:banana::doc1");
+ DocumentType docType = new DocumentType("banana");
+ switch (whatToDo) {
+ case 0:
+ finished = true;
+ break;
+ case 1:
+ Document doc = new Document(docType, id);
+ operation.setDocument(doc);
+ break;
+ case 2:
+ operation.setRemove(id);
+ break;
+ case 3:
+ operation.setDocumentUpdate(new DocumentUpdate(docType, id));
+ break;
+ case 4:
+ throw new RuntimeException("boom");
+ }
+ }
+
+} \ No newline at end of file