diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespaclient-container-plugin |
Publish
Diffstat (limited to 'vespaclient-container-plugin')
84 files changed, 10259 insertions, 0 deletions
diff --git a/vespaclient-container-plugin/.gitignore b/vespaclient-container-plugin/.gitignore new file mode 100644 index 00000000000..016c6f704f0 --- /dev/null +++ b/vespaclient-container-plugin/.gitignore @@ -0,0 +1,2 @@ +pom.xml.build +/target diff --git a/vespaclient-container-plugin/OWNERS b/vespaclient-container-plugin/OWNERS new file mode 100644 index 00000000000..0e39145d8c3 --- /dev/null +++ b/vespaclient-container-plugin/OWNERS @@ -0,0 +1 @@ +dybdahl diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml new file mode 100644 index 00000000000..017216c3cbd --- /dev/null +++ b/vespaclient-container-plugin/pom.xml @@ -0,0 +1,109 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespaclient-container-plugin</artifactId> + <version>6-SNAPSHOT</version> + <packaging>container-plugin</packaging> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespaclient-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-http-client</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>1.8.4</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:rawtypes</arg> + <arg>-Xlint:deprecation</arg> + <arg>-Xlint:unchecked</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + </plugins> + </build> +</project> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java new file mode 100644 index 00000000000..4dc47f20889 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.documentapi.DumpVisitorDataHandler; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.nio.charset.StandardCharsets; + +/** + * Handling data from visit. + * + * @author dybdahl + */ +class LocalDataVisitorHandler extends DumpVisitorDataHandler { + + StringBuilder commaSeparatedJsonDocuments = new StringBuilder(); + final StringBuilder errors = new StringBuilder(); + + private boolean isFirst = true; + private final Object monitor = new Object(); + + String getErrors() { + return errors.toString(); + } + + String getCommaSeparatedJsonDocuments() { + return commaSeparatedJsonDocuments.toString(); + } + + @Override + public void onDocument(Document document, long l) { + try { + final String docJson = new String(JsonWriter.toByteArray(document), StandardCharsets.UTF_8.name()); + synchronized (monitor) { + if (!isFirst) { + commaSeparatedJsonDocuments.append(","); + } + isFirst = false; + commaSeparatedJsonDocuments.append(docJson); + } + } catch (Exception e) { + synchronized (monitor) { + errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + } + } + } + + // TODO: Not sure if we should support removeal or not. Do nothing here maybe? + @Override + public void onRemove(DocumentId documentId) { + try { + final String removeJson = new String(JsonWriter.documentRemove(documentId), StandardCharsets.UTF_8.name()); + synchronized (monitor) { + if (!isFirst) { + commaSeparatedJsonDocuments.append(","); + } + isFirst = false; + commaSeparatedJsonDocuments.append(removeJson); + } + } catch (Exception e) { + synchronized (monitor) { + errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + } + } + } +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java new file mode 100644 index 00000000000..240cc3c3c61 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.util.Optional; + +/** + * Abstract the backend stuff for the REST API, such as retrieving or updating documents. + * + * @author dybdahl + */ +public interface OperationHandler { + + class VisitResult{ + + public final Optional<String> token; + public final String documentsAsJsonList; + + public VisitResult(Optional<String> token, String documentsAsJsonList) { + this.token = token; + this.documentsAsJsonList = documentsAsJsonList; + } + } + + VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException; + + void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException; + + void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException; + + void delete(RestUri restUri, String condition) throws RestApiException; + + Optional<String> get(RestUri restUri) throws RestApiException; + + /** Called just before this is disposed of */ + default void shutdown() {} + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java new file mode 100644 index 00000000000..21fd930e9d5 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -0,0 +1,260 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.document.DocumentPut; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.storage.searcher.ContinuationHit; +import com.yahoo.vdslib.VisitorOrdering; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Sends operations to messagebus via document api. + * + * @author dybdahl + */ +public class OperationHandlerImpl implements OperationHandler { + + public static final int VISIT_TIMEOUT_MS = 120000; + private final DocumentAccess documentAccess; + + public OperationHandlerImpl(DocumentAccess documentAccess) { + this.documentAccess = documentAccess; + } + + @Override + public void shutdown() { + documentAccess.shutdown(); + } + + private static final int HTTP_STATUS_BAD_REQUEST = 400; + private static final int HTTP_STATUS_INSUFFICIENT_STORAGE = 507; + + private static int getHTTPStatusCode(Set<Integer> errorCodes) { + if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) { + return HTTP_STATUS_INSUFFICIENT_STORAGE; + } + return HTTP_STATUS_BAD_REQUEST; + } + + private static Response createErrorResponse(DocumentAccessException documentException, RestUri restUri) { + return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), documentException.getMessage(), restUri); + } + + @Override + public VisitResult visit( + RestUri restUri, + String documentSelection, + Optional<String> cluster, + Optional<String> continuation) throws RestApiException { + + VisitorParameters visitorParameters = createVisitorParameters(restUri, documentSelection, cluster, continuation); + + VisitorControlHandler visitorControlHandler = new VisitorControlHandler(); + visitorParameters.setControlHandler(visitorControlHandler); + LocalDataVisitorHandler localDataVisitorHandler = new LocalDataVisitorHandler(); + visitorParameters.setLocalDataHandler(localDataVisitorHandler); + + final VisitorSession visitorSession; + try { + visitorSession = documentAccess.createVisitorSession(visitorParameters); + // Not sure if this line is required + visitorControlHandler.setSession(visitorSession); + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse( + 500, + "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTrace(e), + restUri)); + } + try { + if (! visitorControlHandler.waitUntilDone(VISIT_TIMEOUT_MS)) { + throw new RestApiException(Response.createErrorResponse(500, "Timed out", restUri)); + } + if (visitorControlHandler.getResult().code != VisitorControlHandler.CompletionCode.SUCCESS) { + throw new RestApiException(Response.createErrorResponse(400, visitorControlHandler.getResult().toString())); + } + } catch (InterruptedException e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + if (localDataVisitorHandler.getErrors().isEmpty()) { + final Optional<String> continuationToken; + if (! visitorControlHandler.getProgress().isFinished()) { + final ContinuationHit continuationHit = new ContinuationHit(visitorControlHandler.getProgress()); + continuationToken = Optional.of(continuationHit.getValue()); + } else { + continuationToken = Optional.empty(); + } + return new VisitResult(continuationToken, localDataVisitorHandler.getCommaSeparatedJsonDocuments()); + } + throw new RestApiException(Response.createErrorResponse(500, localDataVisitorHandler.getErrors(), restUri)); + } + + @Override + public void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + try { + SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters()); + DocumentPut put = new DocumentPut(data.getDocument()); + put.setCondition(data.getCondition()); + syncSession.put(put); + } catch (DocumentAccessException documentException) { + throw new RestApiException(createErrorResponse(documentException, restUri)); + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + } + + @Override + public void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + try { + SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters()); + syncSession.update(data.getDocumentUpdate()); + } catch (DocumentAccessException documentException) { + throw new RestApiException(createErrorResponse(documentException, restUri)); + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + } + + @Override + public void delete(RestUri restUri, String condition) throws RestApiException { + try { + DocumentId id = new DocumentId(restUri.generateFullId()); + SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters()); + DocumentRemove documentRemove = new DocumentRemove(id); + if (condition != null && ! condition.isEmpty()) { + documentRemove.setCondition(new TestAndSetCondition(condition)); + } + syncSession.remove(documentRemove); + } catch (DocumentAccessException documentException) { + throw new RestApiException(Response.createErrorResponse(400, documentException.getMessage(), restUri)); + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + } + + @Override + public Optional<String> get(RestUri restUri) throws RestApiException { + try { + DocumentId id = new DocumentId(restUri.generateFullId()); + SyncSession syncSession = documentAccess.createSyncSession(new SyncParameters()); + final Document document = syncSession.get(id); + if (document == null) { + return Optional.empty(); + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + JsonWriter jsonWriter = new JsonWriter(outputStream); + jsonWriter.write(document); + return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name())); + + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + } + + private static String resolveClusterRoute(Optional<String> wantedCluster) throws RestApiException { + List<ClusterDef> clusters = new ClusterList("client").getStorageClusters(); + return resolveClusterRoute(wantedCluster, clusters); + } + + // Based on resolveClusterRoute in VdsVisit, protected for testability + protected static String resolveClusterRoute(Optional<String> wantedCluster, List<ClusterDef> clusters) throws RestApiException { + if (clusters.size() == 0) { + throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " + + "declared. Visiting feature is not available."); + } + if (! wantedCluster.isPresent()) { + if (clusters.size() != 1) { + new RestApiException(Response.createErrorResponse(400, "Several clusters exist: " + + clusterListToString(clusters) + " you must specify one.. ")); + } + return clusterDefToRoute(clusters.get(0)); + } + + for (ClusterDef clusterDef : clusters) { + if (clusterDef.getName().equals(wantedCluster.get())) { + return clusterDefToRoute(clusterDef); + } + } + throw new RestApiException(Response.createErrorResponse(400, "Your vespa cluster contains the content clusters " + + clusterListToString(clusters) + " not " + wantedCluster.get() + ". Please select a valid vespa cluster.")); + + } + + private static String clusterDefToRoute(ClusterDef clusterDef) { + return "[Storage:cluster=" + clusterDef.getName() + ";clusterconfigid=" + clusterDef.getConfigId() + "]"; + } + + private static String clusterListToString(List<ClusterDef> clusters) { + StringBuilder clusterListString = new StringBuilder(); + clusters.forEach(x -> clusterListString.append(x.getName()).append(" (").append(x.getConfigId()).append("), ")); + return clusterListString.toString(); + } + + private VisitorParameters createVisitorParameters( + RestUri restUri, + String documentSelection, + Optional<String> clusterName, + Optional<String> continuation) + throws RestApiException { + + StringBuilder selection = new StringBuilder(); + + if (! documentSelection.isEmpty()) { + selection.append("(").append(documentSelection).append(" and "); + } + selection.append(restUri.getDocumentType()).append(" and (id.namespace=='").append(restUri.getNamespace()).append("')"); + if (! documentSelection.isEmpty()) { + selection.append(")"); + } + + VisitorParameters params = new VisitorParameters(selection.toString()); + + params.setMaxBucketsPerVisitor(1); + params.setMaxPending(32); + params.setMaxFirstPassHits(1); + params.setMaxTotalHits(10); + params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(1)); + params.setToTimestamp(0L); + params.setFromTimestamp(0L); + + params.visitInconsistentBuckets(true); + params.setVisitorOrdering(VisitorOrdering.ASCENDING); + + params.setRoute(resolveClusterRoute(clusterName)); + + params.setTraceLevel(0); + params.setPriority(DocumentProtocol.Priority.NORMAL_4); + params.setVisitRemoves(false); + + if (continuation.isPresent()) { + try { + params.setResumeToken(ContinuationHit.getToken(continuation.get())); + } catch (Exception e) { + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri)); + } + } + return params; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java new file mode 100644 index 00000000000..9c846e9ce38 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +public class Response extends HttpResponse { + + private final static ObjectMapper objectMapper = new ObjectMapper(); + private final String jsonMessage; + + public Response(int code, Optional<ObjectNode> element, Optional<RestUri> restPath) { + super(code); + ObjectNode objectNode = element.orElse(objectMapper.createObjectNode()); + if (restPath.isPresent()) { + objectNode.put("id", restPath.get().generateFullId()); + objectNode.put("pathId", restPath.get().getRawPath()); + } + jsonMessage = objectNode.toString(); + } + + public static Response createErrorResponse(int code, String errorMessage) { + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.putArray("errors").add(errorMessage); + return new Response(code, Optional.of(objectNode), Optional.<RestUri>empty()); + } + + public static Response createErrorResponse(int code, String errorMessage, RestUri restUri) { + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.putArray("errors").add(errorMessage); + return new Response(code, Optional.of(objectNode), Optional.of(restUri)); + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(jsonMessage.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { return "application/json"; } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java new file mode 100644 index 00000000000..b553d83d848 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +/** + * Exceptions for Rest API + * + * @author dybdahl + */ +public class RestApiException extends Exception { + + private final Response response; + + public RestApiException(Response response) { + this.response = response; + } + + public Response getResponse() { + return response; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java new file mode 100644 index 00000000000..98293508168 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import static com.yahoo.jdisc.Response.Status.*; + +/** + * Represents the request URI with its values. + * + * @author dybdahl + */ +public class RestUri { + + public static final char NUMBER_STREAMING = 'n'; + public static final char GROUP_STREAMING = 'g'; + public static final String DOCUMENT = "document"; + public static final String V_1 = "v1"; + public static final String ID = "id:"; + + /** + * Represents the "grouping" part of document id which can be used with streaming model. + */ + public static class Group { + public final char name; + public final String value; + Group(char name, String value) { + this.name = name; + this.value = value; + } + } + private final String namespace; + private final String documentType; + private final String docId; + private Optional<Group> group = Optional.empty(); + private final String rawPath; + + public String getRawPath() { + return rawPath; + } + + public String getNamespace() { + return namespace; + } + + public String getDocumentType() { + return documentType; + } + + public String getDocId() { + return docId; + } + + public Optional<Group> getGroup() { + return group; + } + + public String generateFullId() { + return ID + namespace + ":" + documentType + ":" + + (getGroup().isPresent() ? group.get().name + "=" + group.get().value : "") + + ":" + docId; + } + + static class PathParser { + final List<String> rawParts; + final String originalPath; + int readPos = 0; + public PathParser(String path) { + this.originalPath = path; + this.rawParts = Splitter.on('/').splitToList(path); + } + String nextTokenOrException() throws RestApiException { + if (readPos >= rawParts.size()) { + throwUsage(originalPath); + } + return rawParts.get(readPos++); + } + + String restOfPath() throws RestApiException { + String rawId = Joiner.on("/").join(rawParts.listIterator(readPos)); + try { + return URLDecoder.decode(rawId, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,"Problems decoding the URI: " + e.getMessage())); + } + } + } + + public RestUri(URI uri) throws RestApiException { + rawPath = uri.getRawPath(); + PathParser pathParser = new PathParser(rawPath); + if (! pathParser.nextTokenOrException().equals("") || + ! pathParser.nextTokenOrException().equals(DOCUMENT) || + ! pathParser.nextTokenOrException().equals(V_1)) { + throwUsage(uri.getRawPath()); + } + namespace = pathParser.nextTokenOrException(); + documentType = pathParser.nextTokenOrException(); + switch (pathParser.nextTokenOrException()) { + case "number": + group = Optional.of(new Group(NUMBER_STREAMING, pathParser.nextTokenOrException())); + break; + case "docid": group = Optional.empty(); + break; + case "group": + group = Optional.of(new Group(GROUP_STREAMING, pathParser.nextTokenOrException())); + break; + default: throwUsage(uri.getRawPath()); + } + docId = pathParser.restOfPath(); + } + + private static void throwUsage(String inputPath) throws RestApiException { + throw new RestApiException(Response.createErrorResponse(BAD_REQUEST, + "Expected:\n" + + ".../{namespace}/{document-type}/group/{name}/[{user-specified}]\n" + + ".../{namespace}/{document-type}/docid/[{user-specified}]\n: but got " + inputPath)); + } + +} + diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java new file mode 100644 index 00000000000..8b863a69f5c --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.config.DocumentmanagerConfig; + +import com.yahoo.document.json.SingleDocumentParser; +import com.yahoo.document.restapi.OperationHandler; +import com.yahoo.document.restapi.OperationHandlerImpl; +import com.yahoo.document.restapi.Response; +import com.yahoo.document.restapi.RestApiException; +import com.yahoo.document.restapi.RestUri; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * API for handling single operation on a document and visiting. + * + * @author dybdahl + */ +public class RestApi extends LoggingRequestHandler { + + private static final String CREATE_PARAMETER_NAME = "create"; + private static final String CONDITION_PARAMETER_NAME = "condition"; + private static final String DOCUMENTS = "documents"; + private static final String FIELDS = "fields"; + private static final String DOC_ID_NAME = "id"; + private static final String PATH_NAME = "pathId"; + private static final String SELECTION = "selection"; + private static final String CLUSTER = "cluster"; + private static final String CONTINUATION = "continuation"; + private final OperationHandler operationHandler; + private SingleDocumentParser singleDocumentParser; + private ObjectMapper mapper = new ObjectMapper(); + private AtomicInteger threadsAvailableForApi = new AtomicInteger(20 /*max concurrent requests */); + + @Inject + public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig) { + super(executor, accessLog); + final LoadTypeSet loadTypes = new LoadTypeSet("client"); + this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(new MessageBusParams(loadTypes))); + this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); + } + + // For testing and development + public RestApi( + Executor executor, + AccessLog accessLog, + OperationHandler operationHandler) { + super(executor, accessLog); + this.operationHandler = operationHandler; + } + + @Override + public void destroy() { + operationHandler.shutdown(); + } + + // For testing and development + protected void setDocTypeManagerForTests(DocumentTypeManager docTypeManager) { + this.singleDocumentParser = new SingleDocumentParser(docTypeManager); + } + + // Returns null if invalid value. + private Optional<Boolean> parseBoolean(String parameter, HttpRequest request) { + final String property = request.getProperty(parameter); + if (property != null && ! property.isEmpty()) { + switch (property) { + case "true" : return Optional.of(true); + case "false": return Optional.of(false); + default : return null; + } + } + return Optional.empty(); + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + if (threadsAvailableForApi.decrementAndGet() < 1) { + return Response.createErrorResponse( + 429 /* Too Many Requests */, + "Too many parallel requests, consider using http-vespa-java-client. Please try again later."); + } + return handleInternal(request); + } finally { + threadsAvailableForApi.incrementAndGet(); + } + } + + // protected for testing + protected HttpResponse handleInternal(HttpRequest request) { + final RestUri restUri; + try { + restUri = new RestUri(request.getUri()); + } catch (RestApiException e) { + return e.getResponse(); + } catch (Exception e2) { + return Response.createErrorResponse(500, "Exception while parsing URI: " + e2.getMessage()); + } + + Optional<Boolean> create = parseBoolean(CREATE_PARAMETER_NAME, request); + if (create == null) { + return Response.createErrorResponse(403, "Non valid value for 'create' parameter, must be empty, true, or " + + "false: " + request.getProperty(CREATE_PARAMETER_NAME)); + } + String condition = request.getProperty(CONDITION_PARAMETER_NAME); + Optional<ObjectNode> resultJson = Optional.empty(); + try { + switch (request.getMethod()) { + case GET: // Vespa Visit/Get + return restUri.getDocId().isEmpty() ? handleVisit(restUri, request) : handleGet(restUri); + case POST: // Vespa Put + operationHandler.put(restUri, createPutOperation(request, restUri.generateFullId(), condition)); + break; + case PUT: // Vespa Update + operationHandler.update(restUri, createUpdateOperation(request, restUri.generateFullId(), condition, create)); + break; + case DELETE: // Vespa Delete + operationHandler.delete(restUri, condition); + break; + default: + return new Response(405, Optional.empty(), Optional.of(restUri)); + } + } catch (RestApiException e) { + return e.getResponse(); + } catch (Exception e2) { + // We always blame the user. This might be a bit nasty, but the parser throws various kind of exception + // types, but with nice descriptions. + return Response.createErrorResponse(400, e2.getMessage(), restUri); + } + return new Response(200, resultJson, Optional.of(restUri)); + } + + private VespaXMLFeedReader.Operation createPutOperation(HttpRequest request, String id, String condition) { + final VespaXMLFeedReader.Operation operationPut = + singleDocumentParser.parsePut(request.getData(), id); + if (condition != null && ! condition.isEmpty()) { + operationPut.setCondition(new TestAndSetCondition(condition)); + } + return operationPut; + } + + private VespaXMLFeedReader.Operation createUpdateOperation(HttpRequest request, String id, String condition, Optional<Boolean> create) { + final VespaXMLFeedReader.Operation operationUpdate = + singleDocumentParser.parseUpdate(request.getData(), id); + if (condition != null && ! condition.isEmpty()) { + operationUpdate.getDocumentUpdate().setCondition(new TestAndSetCondition(condition)); + } + if (create.isPresent()) { + operationUpdate.getDocumentUpdate().setCreateIfNonExistent(create.get()); + } + return operationUpdate; + } + + private HttpResponse handleGet(RestUri restUri) throws RestApiException { + final Optional<String> getDocument = operationHandler.get(restUri); + final ObjectNode resultNode = mapper.createObjectNode(); + if (getDocument.isPresent()) { + final JsonNode parseNode; + try { + parseNode = mapper.readTree(getDocument.get()); + } catch (IOException e) { + throw new RuntimeException("Failed while parsing my own results", e); + } + resultNode.putPOJO(FIELDS, parseNode.get(FIELDS)); + } + resultNode.put(DOC_ID_NAME, restUri.generateFullId()); + resultNode.put(PATH_NAME, restUri.getRawPath()); + + return new HttpResponse(getDocument.isPresent() ? 200 : 404) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8.name())); + } + }; + } + + private HttpResponse handleVisit(RestUri restUri, HttpRequest request) throws RestApiException { + if (restUri.getGroup().isPresent() && ! restUri.getGroup().get().value.isEmpty()) { + return Response.createErrorResponse( + 400, + "Visiting does not support setting value for group/value, try using expression parameter instead.", + restUri); + + } + String documentSelection = Optional.ofNullable(request.getProperty(SELECTION)).orElse(""); + Optional<String> cluster = Optional.ofNullable(request.getProperty(CLUSTER)); + Optional<String> continuation = Optional.ofNullable(request.getProperty(CONTINUATION)); + final OperationHandler.VisitResult visit = operationHandler.visit(restUri, documentSelection, cluster, continuation); + final ObjectNode resultNode = mapper.createObjectNode(); + if (visit.token.isPresent()) { + resultNode.put(CONTINUATION, visit.token.get()); + } + resultNode.putArray(DOCUMENTS).addPOJO(visit.documentsAsJsonList); + resultNode.put(PATH_NAME, restUri.getRawPath()); + + HttpResponse httpResponse = new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + try { + outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + return httpResponse; + } +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java new file mode 100644 index 00000000000..52e32009e00 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.document.restapi.resource; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java new file mode 100755 index 00000000000..d8989e347f6 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.metrics.MetricManager; +import com.yahoo.metrics.MetricSnapshot; +import com.yahoo.text.Utf8String; +import com.yahoo.text.XMLWriter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; + +public class StatusResponse extends HttpResponse { + + MetricManager manager; + int verbosity; + int snapshotTime; + + StatusResponse(MetricManager manager, int verbosity, int snapshotTime) { + super(com.yahoo.jdisc.http.HttpResponse.Status.OK); + this.manager = manager; + this.snapshotTime = snapshotTime; + this.verbosity = verbosity; + } + + @Override + public void render(OutputStream stream) throws IOException { + XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream)); + writer.openTag("status"); + if (verbosity >= 2) { + writer.attribute(new Utf8String("description"), "Metrics since start"); + } + + if (snapshotTime == 0) { + MetricSnapshot snapshot = (new MetricSnapshot( + "Total metrics from start until current time", 0, + manager.getActiveMetrics().getMetrics(), false)); + manager.getTotalMetricSnapshot().addToSnapshot(snapshot, (int)(System.currentTimeMillis() / 1000), false); + snapshot.printXml(manager, "", verbosity, writer); + } else { + try { + manager.getMetricSnapshotSet(snapshotTime).getSnapshot().printXml(manager, "", verbosity, writer); + } catch (Exception e) { + writer.openTag("error"); + writer.attribute(new Utf8String("details"), "No metric snapshot with period " + snapshotTime + + " was found. Legal snapshot periods are: " + manager.getSnapshotPeriods()); + writer.closeTag(); + } + } + writer.closeTag(); + writer.flush(); + } + + @Override + public java.lang.String getContentType() { + return "application/xml"; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java new file mode 100755 index 00000000000..a4e0ddd1748 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import java.util.concurrent.Executor; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; + +public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler { + + private final VespaFeedHandlerGet getHandler; + private final VespaFeedHandler feedHandler; + + public VespaFeedHandlerCompatibility(Executor executor, VespaFeedHandlerGet getHandler, + VespaFeedHandler feedHandler) { + super(executor); + this.getHandler = getHandler; + this.feedHandler = feedHandler; + } + + @Override + public HttpResponse handle(HttpRequest request) { + boolean hasType = request.hasProperty("type"); + // If we have an ID and no document type, redirect to Get + if (request.hasProperty("id") && !hasType) { + return getHandler.handle(request); + } else { + return feedHandler.handle(request); + } + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java new file mode 100755 index 00000000000..aadfe5852ce --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import java.util.Collections; +import java.util.concurrent.Executor; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.search.handler.SearchHandler; + +public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public VespaFeedHandlerGet(SearchHandler searchHandler, Executor executor) { + super(executor, null, true); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest request) { + return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget"))); + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java new file mode 100755 index 00000000000..14b2d86ae75 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.google.inject.Inject; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.DocumentId; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.feedapi.SingleSender; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.concurrent.Executor; + +public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { + + @Inject + public VespaFeedHandlerRemove(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, Executor executor, Metric metric) throws Exception { + super(feederConfig, loadTypeConfig, executor, metric); + } + + VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception { + super(context, executor); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getProperty("status") != null) { + return new MetricResponse(context.getMetrics().getMetricSet()); + } + + MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); + String route = properties.getRoute().toString(); + FeedResponse response = new FeedResponse(new RouteMetricSet(route, null)); + SingleSender sender = new SingleSender(response, getSharedSender(route)); + sender.addMessageProcessor(properties); + + response.setAbortOnFeedError(properties.getAbortOnFeedError()); + + if (request.hasProperty("id")) { + sender.remove(new DocumentId(request.getProperty("id"))); + } else if (request.hasProperty("id[0]")) { + int index = 0; + while (request.hasProperty("id[" + index + "]")) { + sender.remove(new DocumentId(request.getProperty("id[" + index + "]"))); + ++index; + } + } + + if (request.getData() != null) { + try { + String line; + BufferedReader reader = new BufferedReader( + new InputStreamReader(getRequestInputStream(request), "UTF-8")); + while ((line = reader.readLine()) != null) { + sender.remove(new DocumentId(line)); + } + } catch (Exception e) { + response.addError(e.getClass() + ": " + e.getCause()); + } + } + + sender.done(); + long millis = getTimeoutMillis(request); + boolean completed = sender.waitForPending(millis); + if ( ! completed) + response.addError("Timed out after "+millis+" ms waiting for responses"); + return response; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java new file mode 100644 index 00000000000..3b2f82c865e --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.google.inject.Inject; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.feedapi.SingleSender; +import com.yahoo.jdisc.Metric; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.util.concurrent.Executor; + +public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { + + @Inject + public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor, + Metric metric) throws Exception { + super(feederConfig, loadTypeConfig, executor, metric); + } + + VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception { + super(context, executor); + } + + @Override + public HttpResponse handle(HttpRequest request) { + MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); + FeedResponse response; + + if (request.getProperty("route") == null) { + if (context.getClusterList().getStorageClusters().size() == 0) { + return new FeedResponse(null).addError("No storage clusters configured and no alternate route specified."); + } else if (context.getClusterList().getStorageClusters().size() > 1) { + return new FeedResponse(null).addError("More than one storage cluster configured and no route specified."); + } else { + properties.setRoute(Route.parse(context.getClusterList().getStorageClusters().get(0).getName())); + } + } + + response = new FeedResponse(new RouteMetricSet(properties.getRoute().toString(), null)); + + SingleSender sender = new SingleSender(response, getSharedSender(properties.getRoute().toString())); + sender.addMessageProcessor(properties); + + String user = request.getProperty("user"); + String group = request.getProperty("group"); + String selection = request.getProperty("selection"); + + boolean oneFound = (user != null) ^ (group != null) ^ (selection != null); + + if (!oneFound) { + response.addError("Exactly one of \"user\", \"group\" or \"selection\" must be specified for removelocation"); + return response; + } + + if (user != null) { + selection = "id.user=" + user; + } + if (group != null) { + selection = "id.group=\"" + group + "\""; + } + + sender.send(new RemoveLocationMessage(selection)); + sender.done(); + long millis = getTimeoutMillis(request); + boolean completed = sender.waitForPending(millis); + if ( ! completed) + response.addError("Timed out after "+millis+" ms waiting for responses"); + return response; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java new file mode 100755 index 00000000000..77930ae5a94 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import java.util.concurrent.Executor; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.metrics.MetricManager; +import com.yahoo.metrics.MetricSet; +import com.yahoo.vespaclient.config.FeederConfig; + +public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler { + + private MetricManager manager; + + public VespaFeedHandlerStatus(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor) { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()), true, true, executor); + } + + VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) { + super(executor); + manager = new MetricManager(); + final MetricSet metricSet = context.getMetrics().getMetricSet(); + metricSet.unregister(); + manager.registerMetric(metricSet); + if (doLog) { + manager.addMetricToConsumer("log", "routes.total.putdocument.count"); + manager.addMetricToConsumer("log", "routes.total.removedocument.count"); + manager.addMetricToConsumer("log", "routes.total.updatedocument.count"); + manager.addMetricToConsumer("log", "routes.total.getdocument.count"); + + manager.addMetricToConsumer("log", "routes.total.putdocument.errors.total"); + manager.addMetricToConsumer("log", "routes.total.removedocument.errors.total"); + manager.addMetricToConsumer("log", "routes.total.updatedocument.errors.total"); + manager.addMetricToConsumer("log", "routes.total.getdocument.errors.total"); + + manager.addMetricToConsumer("log", "routes.total.putdocument.latency"); + manager.addMetricToConsumer("log", "routes.total.removedocument.latency"); + manager.addMetricToConsumer("log", "routes.total.updatedocument.latency"); + manager.addMetricToConsumer("log", "routes.total.getdocument.latency"); + } + + if (doLog || makeSnapshots) { + new Thread(manager).start(); + } + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + return new StatusResponse(manager, asInt(request.getProperty("verbosity"), 0), asInt(request.getProperty("snapshotperiod"), 0)); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private int asInt(String value, int defaultValue) { + if (value == null) return defaultValue; + return Integer.parseInt(value); + } + + @Override + public void destroy() { + manager.stop(); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java new file mode 100644 index 00000000000..dba6bb78be1 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import java.util.Collections; +import java.util.concurrent.Executor; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.search.handler.SearchHandler; + +/** + * @author thomasg + */ +public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public VespaFeedHandlerVisit(SearchHandler searchHandler, Executor executor) { + super(executor, null, true); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest request) { + return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespavisit"))); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java new file mode 100755 index 00000000000..cae35669c02 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.search.result.Hit; +import java.io.IOException; +import java.util.Base64; + +public class ContinuationHit extends Hit { + + private final String value; + + public ContinuationHit(ProgressToken token) { + super("continuation"); + + final byte[] serialized = token.serialize(); + value = Base64.getUrlEncoder().encodeToString(serialized); + } + + public static ProgressToken getToken(String continuation) throws IOException { + byte[] serialized; + try { + serialized = Base64.getUrlDecoder().decode(continuation); + } catch (IllegalArgumentException e) { + // Legacy visitor tokens were encoded with MIME Base64 which may fail decoding as URL-safe. + // Try again with MIME decoder to avoid breaking upgrade scenarios. + // TODO(vekterli): remove once this is no longer a risk. + serialized = Base64.getMimeDecoder().decode(continuation); + } + return new ProgressToken(serialized); + } + + public String getValue() { + return value; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java new file mode 100755 index 00000000000..901ea44102b --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.io.ByteWriter; +import com.yahoo.prelude.templates.Context; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.text.XML; + +import java.io.IOException; +import java.io.Writer; + +/** + * Template used to render a single field for a single Document. Fields + * that are either of type CONTENT or RAW are written directly, while + * all other fields are wrapped in Vespa XML and escaped. + */ +public class DocumentFieldTemplate extends UserTemplate<Writer> { + + Field field; + String contentType; + String encoding; + boolean wrapXml; + + public DocumentFieldTemplate(Field field, String contentType, + String encoding, boolean wrapXml) { + // Defaults here are chosen as to give the same expected results as + // HTTPGateway's GetRequest + super("documentfield", contentType, + encoding); + + this.field = field; + this.contentType = contentType; + this.encoding = encoding; + this.wrapXml = wrapXml; + } + + @Override + public void error(Context context, Writer writer) throws IOException { + // Error shouldn't be handled by this template, but rather + // delegated to the searcher + } + + @Override + public Writer wrapWriter(Writer writer) { + /* TODO: uncomment + if (!(writer instanceof ByteWriter)) { + throw new IllegalArgumentException("ByteWriter required, but got " + writer.getClass().getName()); + } + */ + + return writer; + } + + @Override + public void header(Context context, Writer writer) throws IOException { + if (wrapXml) { + // XML wrapping should only be used for default field rendering + writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); + writer.write("<result>"); + } + } + + @Override + public void footer(Context context, Writer writer) throws IOException { + if (wrapXml) { + writer.write("</result>\n"); + } + } + + @Override + public void hit(Context context, Writer writer) throws IOException { + DocumentHit hit = (DocumentHit)context.get("hit"); + Document doc = hit.getDocument(); + // Assume field existence has been checked before we ever get here. + // Also assume that relevant encoding/content type is set + // appropriately according to the request and the field's content + // type, as this is immutable in the template set. + FieldValue value = doc.getFieldValue(field); + if (field.getDataType() == DataType.RAW) { + ByteWriter bw = (ByteWriter)writer; + bw.append(((Raw) value).getByteBuffer().array()); + } else { + writer.write(XML.xmlEscape(value.toString(), false)); + } + } + + @Override + public void hitFooter(Context context, Writer writer) throws IOException { + } + + @Override + public void noHits(Context context, Writer writer) throws IOException { + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java new file mode 100755 index 00000000000..8520b5352e3 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.Document; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.search.result.Hit; + +import java.util.Iterator; +import java.util.Map; + +public class DocumentHit extends Hit { + + private Document document; + private int index; + + public DocumentHit(Document document, int index) { + super(document.getId().toString()); + this.document = document; + this.index = index; + } + + public void populateHitFields() { + // Create hit fields for all document fields + Iterator<Map.Entry<Field, FieldValue>> fieldIter = document.iterator(); + while (fieldIter.hasNext()) { + Map.Entry<Field, FieldValue> field = fieldIter.next(); + setField(field.getKey().getName(), field.getValue()); + } + + // Assign an explicit document id field + setField("documentid", document.getId().toString()); + } + + public Document getDocument() { + return document; + } + + public int getIndex() { + return index; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java new file mode 100644 index 00000000000..cf5dfbcffc1 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.DocumentId; +import com.yahoo.search.result.Hit; + +public class DocumentRemoveHit extends Hit { + + private final DocumentId idOfRemovedDoc; + + public DocumentRemoveHit(DocumentId idOfRemovedDoc) { + super(idOfRemovedDoc.toString()); + this.idOfRemovedDoc = idOfRemovedDoc; + setField("documentid", idOfRemovedDoc.toString()); + } + + public DocumentId getIdOfRemovedDoc() { + return idOfRemovedDoc; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java new file mode 100755 index 00000000000..5266a2e1d09 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.log.LogLevel; +import com.yahoo.search.Result; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; +import com.yahoo.prelude.templates.Context; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.search.result.Hit; +import com.yahoo.text.XML; + +import java.io.IOException; +import java.io.Writer; +import java.util.logging.Logger; + +public class DocumentXMLTemplate extends UserTemplate<Writer> { + + private static final Logger log = Logger.getLogger(DocumentXMLTemplate.class.getName()); + + public DocumentXMLTemplate() { + super("vespa_xml"); + } + + public DocumentXMLTemplate(String mimeType, String encoding) { + super("vespa_xml", mimeType, encoding); + } + + private void writeErrorMessage(Writer writer, String type, int code, + String message, String detailedMessage) throws IOException { + writer.write("<error type=\"" + type + "\" code=\"" + code + "\" message=\""); + writer.write(XML.xmlEscape(message, true)); + if (detailedMessage != null) { + writer.write(": "); + writer.write(XML.xmlEscape(detailedMessage, true)); + } + writer.write("\"/>\n"); + } + + private void writeGenericErrorMessage(Writer writer, ErrorMessage message) throws IOException { + // A bit dirty, but we don't have to support many different types + if (message instanceof MessageBusErrorMessage) { + writeErrorMessage(writer, "messagebus", + ((MessageBusErrorMessage)message).getMessageBusCode(), + message.getMessage(), message.getDetailedMessage()); + } else { + writeErrorMessage(writer, "searcher", message.getCode(), + message.getMessage(), message.getDetailedMessage()); + } + } + + @Override + public void error(Context context, Writer writer) throws IOException { + writer.write("<errors>\n"); + // If the error contains no error hits, use a single error with the main + // code and description. Otherwise, use the error hits explicitly + ErrorHit errorHit = ((Result)context.get("result")).hits().getErrorHit(); + if (errorHit == null || errorHit.errors().isEmpty()) { + ErrorMessage message = ((Result)context.get("result")).hits().getError(); + writeGenericErrorMessage(writer, message); + } else { + for (ErrorMessage message : errorHit.errors()) { + writeGenericErrorMessage(writer, message); + } + } + writer.write("</errors>\n"); + } + + @Override + public void header(Context context, Writer writer) throws IOException { + writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); + writer.write("<result>\n"); + HitGroup rootGroup = ((Result) context.get("result")).hits(); + if (rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) != null) { + writer.write("<continuation>" + rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) + "</continuation>"); + } + } + + @Override + public void footer(Context context, Writer writer) throws IOException { + writer.write("</result>\n"); + } + + @Override + public void hit(Context context, Writer writer) throws IOException { + Hit hit = (Hit)context.get("hit"); + if (hit instanceof DocumentHit) { + DocumentHit docHit = (DocumentHit) hit; + if (docHit.getDocument() != null) { + writer.write(docHit.getDocument().toXML(" ")); + } + } else if (hit instanceof DocumentRemoveHit) { + writeDocumentRemoveHit(writer, (DocumentRemoveHit) hit); + } else { + log.log(LogLevel.WARNING, "Cannot render document XML; expected hit of type " + + "com.yahoo.storage.searcher.Document[Remove]Hit, got " + hit.getClass().getName() + + ". Is there another backend searcher present?"); + } + } + + private void writeDocumentRemoveHit(Writer writer, DocumentRemoveHit remove) throws IOException { + writer.write("<remove documentid=\""); + writer.write(XML.xmlEscape(remove.getIdOfRemovedDoc().toString())); + writer.write("\"/>\n"); + } + + @Override + public void hitFooter(Context context, Writer writer) throws IOException { + } + + @Override + public void noHits(Context context, Writer writer) throws IOException { + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java new file mode 100755 index 00000000000..661fcac6a64 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java @@ -0,0 +1,469 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.feedapi.*; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Reply; +import com.yahoo.search.Query; +import com.yahoo.search.query.Properties; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.DefaultErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.io.*; +import java.util.*; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; + +/** + * Searcher component to make GET requests to a content cluster. + * <p> + * Document ID must be given either as 1 "id=docid" query parameter + * for single-document GETs and 1-n "id[0]=docid_1&id[1]=...&id[n-1]=docid_n" + * parameters for multi-document GETs. + * + * <p> + * Standard gateway query parameters are implicitly supported: + * priority, timeout, route + * + * <p> + * The searcher also accepts the following (optional) query parameters: + * headersonly=true|false (default: false) + * For specifying whether or not to return only header fields. + * + * <p> + * field=string (default: no parameter specified) + * For getting a single document field. + * + * <p> + * contenttype=string (default: no content type specified) + * For specifiying the returned HTTP header content type for a returned + * document field's content. field must also be specified. + */ +public class GetSearcher extends Searcher { + + private static final Logger log = Logger.getLogger(GetSearcher.class.getName()); + + FeedContext context; + + private final long defaultTimeoutMillis; + + private class GetResponse implements SharedSender.ResultCallback { + + /** + * We have to maintain the same ordering of results as that + * given in the request. Do this by remembering the index of + * each requested document ID. + */ + private Map<String, Integer> ordering; + private List<DocumentHit> documentHits = new ArrayList<>(); + private List<DefaultErrorHit> errorHits = new ArrayList<>(); + + public GetResponse(List<String> documentIds) { + ordering = new HashMap<>(documentIds.size()); + for (int i = 0; i < documentIds.size(); ++i) { + ordering.put(documentIds.get(i), i); + } + } + + public boolean isAborted() { + return false; + } + + private String stackTraceFromException(Exception e) { + StringWriter sw = new StringWriter(); + PrintWriter ps = new PrintWriter(sw); + e.printStackTrace(ps); + ps.flush(); + return sw.toString(); + } + + public boolean handleReply(Reply reply, int numPending) { + if (!reply.hasErrors()) { + try { + addDocumentHit(reply); + } catch (Exception e) { + String msg = "Got exception of type " + e.getClass().getName() + + " during document deserialization: " + e.getMessage(); + errorHits.add(new DefaultErrorHit("GetSearcher", + ErrorMessage.createInternalServerError(msg))); + log.log(LogLevel.DEBUG, "Got exception during document deserialization: " + stackTraceFromException(e)); + } + } else { + errorHits.add(new DefaultErrorHit("GetSearcher", new MessageBusErrorMessage( + reply.getError(0).getCode(), 0, reply.getError(0).getMessage()))); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received error reply with message " + reply.getError(0).getMessage()); + } + } + + return (numPending > 0); + } + + private void addDocumentHit(Reply reply) { + Document doc = ((GetDocumentReply)reply).getDocument(); + GetDocumentMessage msg = (GetDocumentMessage)reply.getMessage(); + Integer index = ordering.get(msg.getDocumentId().toString()); + if (index == null) { // Shouldn't happen + throw new IllegalStateException("Received GetDocumentReply for unknown document: " + + doc.getId().toString()); + } + if (doc != null) { + documentHits.add(new DocumentHit(doc, index)); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received GetDocumentReply for " + + doc.getId().toString()); + } + } else { + // Don't add a hit for documents that can't be found + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received empty (not found) GetDocumentReply for " + + msg.getDocumentId().toString()); + } + } + } + + private class IndexComparator implements Comparator<DocumentHit> { + public int compare(DocumentHit o1, DocumentHit o2) { + return o1.getIndex() - o2.getIndex(); + } + } + + public void addHitsToResult(Result result, boolean populateHitFields) { + for (DefaultErrorHit hit : errorHits) { + result.hits().add(hit); + } + // Sort document hits according to their request index + Collections.sort(documentHits, new IndexComparator()); + for (DocumentHit hit : documentHits) { + if (populateHitFields) { + hit.populateHitFields(); + } + result.hits().add(hit); + } + result.setTotalHitCount(documentHits.size()); + } + + public List<DocumentHit> getDocumentHits() { + return documentHits; + } + + public List<DefaultErrorHit> getErrorHits() { + return errorHits; + } + } + + @Inject + public GetSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()), + (long)(feederConfig.timeout() * 1000)); + } + + GetSearcher(FeedContext context) throws Exception { + this.context = context; + this.defaultTimeoutMillis = context.getPropertyProcessor().getDefaultTimeoutMillis(); + } + + GetSearcher(FeedContext context, long defaultTimeoutMillis) throws Exception { + this.context = context; + this.defaultTimeoutMillis = defaultTimeoutMillis; + } + + private void postValidateDocumentIdParameters(Properties properties, int arrayIdsFound) throws Exception { + for (Map.Entry<String, Object> kv : properties.listProperties().entrySet()) { + if (!kv.getKey().startsWith("id[")) { + continue; + } + if (!kv.getKey().endsWith("]")) { + throw new IllegalArgumentException("Malformed document ID array parameter"); + } + String indexStr = kv.getKey().substring(3, kv.getKey().length() - 1); + int idx = Integer.parseInt(indexStr); + if (idx >= arrayIdsFound) { + throw new IllegalArgumentException("query contains document ID array " + + "that is not zero-based and/or linearly increasing"); + } + } + } + + private List<String> getDocumentIds(Query query) throws Exception { + Properties properties = query.properties(); + List<String> docIds = new ArrayList<>(); + + // First check for regular "id=XX" syntax. If found, return vector with that + // document id only + String singleId = properties.getString("id"); + + int index = 0; + if (singleId != null) { + docIds.add(singleId); + } else { + // Check for id[0]=XX&id[1]=YY...id[n]=ZZ syntax. Indices always start + // at 0 and are always increased by 1. + while (true) { + String docId = properties.getString("id[" + index + "]"); + if (docId == null) { + break; + } + docIds.add(docId); + ++index; + } + } + postValidateDocumentIdParameters(properties, index); + + handleData(query.getHttpRequest(), docIds); + return docIds; + } + + private void handleData(HttpRequest request, List<String> docIds) throws IOException { + if (request.getData() != null) { + InputStream input; + if ("gzip".equals(request.getHeader("Content-Encoding"))) { + input = new GZIPInputStream(request.getData()); + } else { + input = request.getData(); + } + InputStreamReader reader = new InputStreamReader(input, "UTF-8"); + BufferedReader lineReader = new BufferedReader(reader); + String line; + while ((line = lineReader.readLine()) != null) { + docIds.add(line); + } + } + } + + private void handleFieldFiltering(GetResponse response, Result result, + String fieldName, String contentType, + boolean headersOnly) { + + if (response.getDocumentHits().isEmpty()) { + result.hits().addError(ErrorMessage.createNotFound( + "Document not found, could not return field '" + fieldName + "'")); + return; + } + + if (result.hits().getErrorHit() == null) { + Document doc = response.getDocumentHits().get(0).getDocument(); + Field field = doc.getDataType().getField(fieldName); + boolean wrapXml = false; + + if (field == null) { + result.hits().addError(ErrorMessage.createIllegalQuery( + "Field '" + fieldName + "' not found in document type")); + return; + } + FieldValue value = doc.getFieldValue(field); + // If the field exists but hasn't been set in this document, the + // content will be null. We treat this as an error. + if (value == null) { + if (!field.isHeader() && headersOnly) { + // TODO(vekterli): make this work with field sets as well. + result.hits().addError(ErrorMessage.createInvalidQueryParameter( + "Field '" + fieldName + "' is located in document body, but headersonly " + + "prevents it from being retrieved in " + doc.getId().toString())); + } else { + result.hits().addError(ErrorMessage.createNotFound( + "Field '" + fieldName + "' found in document type, but had " + + "no content in " + doc.getId().toString())); + } + return; + } + String encoding = null; + if (field.getDataType() == DataType.RAW) { + if (contentType == null) { + contentType = "application/octet-stream"; + } + encoding = "ISO-8859-1"; + } else { + // By default, return field wrapped in a blanket of vespa XML + contentType = "text/xml"; + wrapXml = true; + } + if (encoding == null) { + // Encoding doesn't matter for binary content, since we're always + // writing directly to the byte buffer and not through a charset + // encoder. Presumably, the client is intelligent enough to not + // attempt to UTF-8 decode binary data. + encoding = "UTF-8"; + } + // Add hit now that we know there aren't any field errors. Otherwise, + // there would be both an error hit and a document hit in the result + response.addHitsToResult(result, false); + // Override Vespa XML template + result.getTemplating().setTemplates(new DocumentFieldTemplate(field, contentType, encoding, wrapXml)); + } + // else: return with error hit, invoking regular Vespa XML error template + } + + private void validateParameters(String fieldName, String contentType, + List<String> documentIds) { + // Content-type only makes sense for single document queries with a field + // set + if (contentType != null) { + if (documentIds.size() > 1) { + throw new IllegalArgumentException( + "contenttype parameter only valid for single document id query"); + } + if (fieldName == null) { + throw new IllegalArgumentException( + "contenttype set without document field being specified"); + } + } + if (fieldName != null && documentIds.size() > 1) { + throw new IllegalArgumentException( + "Field only valid for single document id query"); + } + } + + // For unit testing + protected MessagePropertyProcessor getMessagePropertyProcessor() { + return context.getPropertyProcessor(); + } + + private void doGetDocuments(Query query, Result result, List<String> documentIds) { + GetResponse response = new GetResponse(documentIds); + Properties properties = query.properties(); + + boolean headersOnly = properties.getBoolean("headersonly", false); + boolean populateHitFields = properties.getBoolean("populatehitfields", false); + String fieldSet = properties.getString("fieldset"); + String fieldName = properties.getString("field"); + String contentType = properties.getString("contenttype"); + long timeoutMillis = properties.getString("timeout") != null ? query.getTimeout() : defaultTimeoutMillis; + + if (fieldSet == null) { + fieldSet = headersOnly ? "[header]" : "[all]"; + } + + validateParameters(fieldName, contentType, documentIds); + + MessagePropertyProcessor.PropertySetter propertySetter; + propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); + + SingleSender sender = new SingleSender(response, context.getSharedSender(propertySetter.getRoute().toString())); + sender.addMessageProcessor(propertySetter); + + sendDocumentGetMessages(documentIds, fieldSet, sender); + // Twiddle thumbs until we've received a reply for all documents + sender.done(); + boolean completed = sender.waitForPending(timeoutMillis); + if ( ! completed) { + result.hits().addError(ErrorMessage.createTimeout( + "Timed out after waiting "+timeoutMillis+" ms for responses")); + } + if (fieldName != null) { + handleFieldFiltering(response, result, fieldName, contentType, headersOnly); + } else { + response.addHitsToResult(result, populateHitFields); + } + } + + private void sendDocumentGetMessages(List<String> documentIds, String fieldSet, SingleSender sender) { + for (String docIdStr : documentIds) { + DocumentId docId = new DocumentId(docIdStr); + GetDocumentMessage getMsg = new GetDocumentMessage(docId, fieldSet); + + sender.send(getMsg); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Sent GetDocumentMessage for " + + docId.toString()); + } + } + } + + boolean verifyBackendDocumentHitsOnly(Result result) { + if (result.hits().size() != 0) { + log.log(LogLevel.DEBUG, "Result had hits after being sent down"); + for (int i = 0; i < result.hits().size(); ++i) { + if (!(result.hits().get(i) instanceof DocumentHit)) { + log.log(LogLevel.WARNING, "Got hit from backend searcher which was " + + "not a com.yahoo.storage.searcher.DocumentHit instance: " + + result.hits().get(i).getClass().getName()); + return false; + } + } + } + return true; + } + + @Override + public Result search(Query query, Execution execution) { + // Pass through to next searcher + Result result = execution.search(query); + + List<String> documentIds; + try { + documentIds = getDocumentIds(query); + } catch (Exception e) { + setOutputFormat(query, result); + result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); + return result; + } + // Early-out for pass-through queries + if (documentIds.isEmpty()) { + return result; + } + // Make sure we don't try to combine non-document hits and document hits + // in the same result set. + if (!verifyBackendDocumentHitsOnly(result)) { + result = new Result(query); // Don't include unknown hits + setOutputFormat(query, result); + result.hits().addError(ErrorMessage.createInternalServerError( + "A backend searcher to com.yahoo.storage.searcher.GetSearcher " + + "returned a hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + + "Only DocumentHit instances are supported in the backend hit result set when doing " + + "queries that contain document identifier sets recognised by the Get Searcher.")); + return result; + } + setOutputFormat(query, result); + // Do not propagate exceptions back up, as we want to have all errors + // be reported using the proper template + try { + doGetDocuments(query, result, documentIds); + query.setHits(result.hits().size()); + } catch (IllegalArgumentException e) { + result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); + } catch (Exception e) { + result.hits().addError(ErrorMessage.createUnspecifiedError(e.getClass().getName() + ": " + e.getMessage())); + } + + return result; + } + + private static final CompoundName formatShortcut = new CompoundName("format"); + private static final CompoundName format = new CompoundName("presentation.format"); + + /** + * Use custom XML output format unless the default JSON renderer is specified in the request. + */ + @SuppressWarnings("deprecation") + static void setOutputFormat(Query query, Result result) { + if (getRequestProperty(formatShortcut, "", query).equals("JsonRenderer")) return; + if (getRequestProperty(format, "", query).equals("JsonRenderer")) return; + if (getRequestProperty(formatShortcut, "", query).equals("json")) return; + if (getRequestProperty(format, "", query).equals("json")) return; + result.getTemplating().setTemplates(new DocumentXMLTemplate()); + } + + private static String getRequestProperty(CompoundName propertyName, String defaultValue, Query query) { + String propertyValue = query.getHttpRequest().getProperty(propertyName.toString()); + if (propertyValue == null) return defaultValue; + return propertyValue; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java new file mode 100755 index 00000000000..7cc8a6514f7 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.search.result.ErrorMessage; + +/** + * Simple ErrorMessage extension that includes a message bus error code, not + * just the searcher error code (which isn't very useful for a Vespa XML consumer) + */ +public class MessageBusErrorMessage extends ErrorMessage { + + private int mbusCode; + + public MessageBusErrorMessage(int mbusCode, int qrsCode, String message) { + super(qrsCode, message); + this.mbusCode = mbusCode; + } + + public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage) { + super(qrsCode, message, detailedMessage); + this.mbusCode = mbusCode; + } + + public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage, Throwable cause) { + super(qrsCode, message, detailedMessage, cause); + this.mbusCode = mbusCode; + } + + public int getMessageBusCode() { + return mbusCode; + } + + public void setMessageBusCode(int code) { + this.mbusCode = code; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java new file mode 100644 index 00000000000..621ffcefbe1 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java @@ -0,0 +1,194 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vdslib.VisitorOrdering; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.config.FeederConfig; + +/** + * A searcher that allows you to iterate through a storage cluster using visiting. + */ +public class VisitSearcher extends Searcher { + + public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken"; + FeedContext context; + + public VisitSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric())); + } + + VisitSearcher(FeedContext context) throws Exception { + this.context = context; + } + + class HitDataHandler extends DumpVisitorDataHandler { + private final Result result; + private final boolean populateHits; + private final Object monitor = new Object(); + + HitDataHandler(Result result, boolean populateHits) { + this.result = result; + this.populateHits = populateHits; + } + + @Override + public void onDocument(Document document, long l) { + final DocumentHit hit = new DocumentHit(document, 0); + if (populateHits) { + hit.populateHitFields(); + } + synchronized (monitor) { + result.hits().add(hit); + } + } + + @Override + public void onRemove(DocumentId documentId) { + final DocumentRemoveHit hit = new DocumentRemoveHit(documentId); + synchronized (monitor) { + result.hits().add(hit); + } + } + } + + public VisitorParameters getVisitorParameters(Query query, Result result) throws Exception { + String documentSelection = query.properties().getString("visit.selection"); + if (documentSelection == null) { + documentSelection = ""; + } + + VisitorParameters params = new VisitorParameters(documentSelection); + params.setMaxBucketsPerVisitor(query.properties().getInteger("visit.maxBucketsPerVisitor", 1)); + params.setMaxPending(query.properties().getInteger("visit.maxPendingMessagesPerVisitor", 32)); + params.setMaxFirstPassHits(query.properties().getInteger("visit.approxMaxDocs", 1)); + params.setMaxTotalHits(query.properties().getInteger("visit.approxMaxDocs", 1)); + params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount( + query.properties().getInteger("visit.maxPendingVisitors", 1))); + params.setToTimestamp(query.properties().getLong("visit.toTimestamp", 0L)); + params.setFromTimestamp(query.properties().getLong("visit.fromTimestamp", 0L)); + + String pri = query.properties().getString("visit.priority"); + if (pri != null) { + params.setPriority(DocumentProtocol.Priority.valueOf(pri)); + } + + if (query.properties().getBoolean("visit.visitInconsistentBuckets")) { + params.visitInconsistentBuckets(true); + } + + String ordering = query.properties().getString("visit.order"); + if (!"ascending".equalsIgnoreCase(ordering)) { + params.setVisitorOrdering(VisitorOrdering.ASCENDING); + } else { + params.setVisitorOrdering(VisitorOrdering.DESCENDING); + } + + String remoteCluster = query.properties().getString("visit.dataHandler"); + if (remoteCluster != null) { + params.setRemoteDataHandler(remoteCluster); + } else { + params.setLocalDataHandler(new HitDataHandler( + result, query.properties().getBoolean("populatehitfields", false))); + } + + String fieldSet = query.properties().getString("visit.fieldSet"); + if (fieldSet != null) { + params.fieldSet(fieldSet); + } + + String continuation = query.properties().getString("visit.continuation"); + if (continuation != null) { + params.setResumeToken(ContinuationHit.getToken(continuation)); + } + + params.setVisitRemoves(query.properties().getBoolean("visit.visitRemoves")); + + MessagePropertyProcessor.PropertySetter propertySetter; + propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); + + propertySetter.process(params); + + if (context.getClusterList().getStorageClusters().size() == 0) { + throw new IllegalArgumentException("No content clusters have been defined"); + } + + String route = query.properties().getString("visit.cluster"); + ClusterDef found = null; + if (route != null) { + String names = ""; + for (ClusterDef c : context.getClusterList().getStorageClusters()) { + if (c.getName().equals(route)) { + found = c; + } + if (!names.isEmpty()) { + names += ", "; + } + names += c.getName(); + } + if (found == null) { + throw new IllegalArgumentException("Your vespa cluster contains the storage clusters " + names + ", not " + route + ". Please select a valid vespa cluster."); + } + } else if (context.getClusterList().getStorageClusters().size() == 1) { + found = context.getClusterList().getStorageClusters().get(0); + } else { + throw new IllegalArgumentException("Multiple content clusters are defined, select one using the \"visit.cluster\" option"); + } + + params.setRoute("[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]"); + return params; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + + VisitorParameters parameters; + + try { + parameters = getVisitorParameters(query, result); + } catch (Exception e) { + return new Result(query, ErrorMessage.createBadRequest("Illegal parameters: " + e.toString())); + } + + if (parameters != null) { + VisitorSession session = context.getSessionFactory().createVisitorSession(parameters); + + try { + if (!session.waitUntilDone(query.getTimeout())) { + return new Result(query, ErrorMessage.createTimeout("Visitor timed out")); + } + + ProgressToken token = session.getProgress(); + if (!token.isFinished()) { + final ContinuationHit continuation = new ContinuationHit(token); + result.hits().setField(VISITOR_CONTINUATION_TOKEN_FIELDNAME, continuation.getValue()); + } + } catch (InterruptedException e) { + } finally { + session.destroy(); + } + } + + GetSearcher.setOutputFormat(query, result); + query.setHits(result.hits().size()); + return result; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java new file mode 100644 index 00000000000..0aeaf714bf3 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.storage.searcher; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java new file mode 100644 index 00000000000..5f2952931f3 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -0,0 +1,326 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL; + +/** + * An instance of this class handles all requests from one client using VespaHttpClient. + * + * The implementation is based on the code from V2, but the object model is rewritten to simplify the logic and + * avoid using a threadpool that has no effect with all the extra that comes with it. V2 has one instance per thread + * on the client, while this is one instance for all threads. + */ +class ClientFeederV3 { + + protected static final Logger log = Logger.getLogger(ClientFeederV3.class.getName()); + // This is for all clients on this gateway, for load balancing from client. + private final static AtomicInteger outstandingOperations = new AtomicInteger(0); + private final BlockingQueue<OperationStatus> feedReplies = new LinkedBlockingQueue<>(); + private final ReferencedResource<SharedSourceSession> sourceSession; + private final String clientId; + private final ReplyHandler feedReplyHandler; + private final Metric metric; + private Instant prevOpsPerSecTime = Instant.now(); + private double operationsForOpsPerSec = 0d; + + private final Object monitor = new Object(); + private final StreamReaderV3 streamReaderV3; + private final AtomicInteger ongoingRequests = new AtomicInteger(0); + private String hostName; + private AtomicInteger threadsAvailableForFeeding; + + ClientFeederV3( + ReferencedResource<SharedSourceSession> sourceSession, + FeedReaderFactory feedReaderFactory, + DocumentTypeManager docTypeManager, + String clientId, + Metric metric, + ReplyHandler feedReplyHandler, + AtomicInteger threadsAvailableForFeeding) { + this.sourceSession = sourceSession; + this.clientId = clientId; + this.feedReplyHandler = feedReplyHandler; + this.metric = metric; + this.threadsAvailableForFeeding = threadsAvailableForFeeding; + this.streamReaderV3 = new StreamReaderV3(feedReaderFactory, docTypeManager); + try { + this.hostName = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + log.info("Could not find hostname, randomizing. " + e.getMessage()); + hostName = "UNKNOWNHOSTNAME" + System.nanoTime() % 10000; + } + } + + public boolean timedOut() { + synchronized (monitor) { + return Instant.now().isAfter(prevOpsPerSecTime.plusSeconds(6000)) && ongoingRequests.get() == 0; + } + } + + public void kill() { + // No new requests should be sent to this object, but there can be old one, even though this is very unlikely. + while (ongoingRequests.get() > 0) { + try { + ongoingRequests.wait(100); + } catch (InterruptedException e) { + break; + } + } + sourceSession.getReference().close(); + } + + private void transferPreviousRepliesToResponse(BlockingQueue<OperationStatus> operations) throws InterruptedException { + OperationStatus status = feedReplies.poll(); + while (status != null) { + outstandingOperations.decrementAndGet(); + operations.put(status); + status = feedReplies.poll(); + } + } + + public HttpResponse handleRequest(HttpRequest request) throws IOException { + threadsAvailableForFeeding.decrementAndGet(); + ongoingRequests.incrementAndGet(); + try { + InputStream inputStream = StreamReaderV3.unzipStreamIfNeeded(request); + final BlockingQueue<OperationStatus> replies = new LinkedBlockingQueue<>(); + FeederSettings feederSettings = new FeederSettings(request); + try { + feed(feederSettings, inputStream, replies, threadsAvailableForFeeding); + synchronized (monitor) { + // Handshake requests do not have DATA_FORMAT, we do not want to give responses to + // handshakes as it won't be processed by the client. + if (request.getJDiscRequest().headers().get(Headers.DATA_FORMAT) != null) { + transferPreviousRepliesToResponse(replies); + } + } + } catch (InterruptedException e) { + // NOP, just terminate + } catch (Throwable e) { + log.log(LogLevel.WARNING, "Unhandled exception while feeding: " + + Exceptions.toMessageString(e), e); + } finally { + try { + replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, null)); + } catch (InterruptedException e) { + // NOP, we are already exiting the thread + } + } + return new FeedResponse(200, replies, 3 /* protocol version */, clientId, outstandingOperations.get(), hostName); + } finally { + ongoingRequests.decrementAndGet(); + threadsAvailableForFeeding.incrementAndGet(); + } + } + + private Optional<DocumentOperationMessageV3> pullMessageFromRequest( + FeederSettings settings, InputStream requestInputStream, BlockingQueue<OperationStatus> repliesFromOldMessages) { + while (true) { + final Optional<String> operationId; + try { + operationId = streamReaderV3.getNextOperationId(requestInputStream); + } catch (IOException ioe) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); + } + return Optional.empty(); + } + if (! operationId.isPresent()) { + return Optional.empty(); + } + final DocumentOperationMessageV3 msg; + try { + msg = getNextMessage(operationId.get(), requestInputStream, settings); + } catch (Exception e) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); + } + repliesFromOldMessages.add(new OperationStatus( + Exceptions.toMessageString(e), operationId.get(), ErrorCode.ERROR, "")); + + continue; + } + setRoute(msg, settings); + return Optional.of(msg); + } + } + + private Result sendMessage( + FeederSettings settings, + DocumentOperationMessageV3 msg, + AtomicInteger threadsAvailableForFeeding) throws InterruptedException { + Result result = null; + while (result == null || result.getError().getCode() == SEND_QUEUE_FULL) { + msg.getMessage().pushHandler(feedReplyHandler); + + if (settings.denyIfBusy && threadsAvailableForFeeding.get() < 1) { + return sourceSession.getResource().sendMessage(msg.getMessage()); + } else { + result = sourceSession.getResource().sendMessageBlocking(msg.getMessage()); + } + if (result.isAccepted()) { + return result; + } + Thread.sleep(100); + } + return result; + } + + private void feed( + FeederSettings settings, + InputStream requestInputStream, + BlockingQueue<OperationStatus> repliesFromOldMessages, + AtomicInteger threadsAvailableForFeeding) throws InterruptedException { + while (true) { + + Optional<DocumentOperationMessageV3> msg = pullMessageFromRequest(settings, requestInputStream, repliesFromOldMessages); + + if (! msg.isPresent()) { + break; + } + setMessageParameters(msg.get(), settings); + + final Result result; + try { + result = sendMessage(settings, msg.get(), threadsAvailableForFeeding); + + } catch (RuntimeException e) { + repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), Exceptions.toMessageString(e), + ErrorCode.ERROR, msg.get().getMessage())); + continue; + } + + if (result.isAccepted()) { + outstandingOperations.incrementAndGet(); + updateOpsPerSec(); + log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.get().getOperationId()); + } else if (!result.getError().isFatal()) { + repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(), + ErrorCode.TRANSIENT_ERROR, msg.get().getMessage())); + continue; + } else { + // should probably not happen, but everybody knows stuff that + // shouldn't happen, happens all the time + repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(), + ErrorCode.ERROR, msg.get().getMessage())); + continue; + } + } + } + + private OperationStatus createOperationStatus(String id, String message, ErrorCode code, Message msg) + throws InterruptedException { + String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0 + ? msg.getTrace().toString() + : ""; + return new OperationStatus(message, id, code, traceMessage); + } + + // protected for mocking + protected DocumentOperationMessageV3 getNextMessage( + String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { + VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings); + + // This is a bit hard to set up while testing, so we accept that things are not perfect. + if (sourceSession.getResource().session() != null) { + metric.set( + MetricNames.PENDING, + Double.valueOf(sourceSession.getResource().session().getPendingCount()), + null); + } + + DocumentOperationMessageV3 msg = DocumentOperationMessageV3.create(operation, operationId, metric); + if (msg == null) { + // typical end of feed + return null; + } + metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); + log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.getOperationId()); + return msg; + } + + private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { + msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies)); + if (settings.traceLevel != null) { + msg.getMessage().getTrace().setLevel(settings.traceLevel); + } + if (settings.priority != null) { + try { + DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); + if (msg.getMessage() instanceof DocumentMessage) { + ((DocumentMessage) msg.getMessage()).setPriority(priority); + } + } + catch (IllegalArgumentException i) { + log.severe(i.getMessage()); + } + } + } + + private void setRoute(DocumentOperationMessageV3 msg, FeederSettings settings) { + if (settings.route != null) { + msg.getMessage().setRoute(settings.route); + } + } + + protected final void log(LogLevel level, Object... msgParts) { + StringBuilder s; + + if (!log.isLoggable(level)) { + return; + } + + s = new StringBuilder(); + for (Object part : msgParts) { + s.append(part.toString()); + } + + log.log(level, s.toString()); + } + + private void updateOpsPerSec() { + Instant now = Instant.now(); + synchronized (monitor) { + if (now.plusSeconds(1).isAfter(prevOpsPerSecTime)) { + Duration duration = Duration.between(now, prevOpsPerSecTime); + final double opsPerSec = operationsForOpsPerSec / (duration.toMillis() / 1000.); + metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, null /*metricContext*/); + operationsForOpsPerSec = 1.0d; + prevOpsPerSecTime = now; + } else { + operationsForOpsPerSec += 1.0d; + } + } + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java new file mode 100644 index 00000000000..240b02fabc7 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.vespa.http.client.core.OperationStatus; + +import java.util.concurrent.BlockingQueue; + +/** + * The state of a client session, used to save replies when client disconnects. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ClientState { + + public final int pending; + public final long creationTime; + public final BlockingQueue<OperationStatus> feedReplies; + public final ReferencedResource<SharedSourceSession> sourceSession; + public final Metric.Context metricContext; + + public final long prevOpsPerSecTime; // previous measurement time of OPS + public final double operationsForOpsPerSec; + + public ClientState(int pending, BlockingQueue<OperationStatus> feedReplies, + ReferencedResource<SharedSourceSession> sourceSession, Metric.Context metricContext, + long prevOpsPerSecTime, double operationsForOpsPerSec) { + super(); + this.pending = pending; + this.feedReplies = feedReplies; + this.sourceSession = sourceSession; + this.metricContext = metricContext; + creationTime = System.currentTimeMillis(); + this.prevOpsPerSecTime = prevOpsPerSecTime; + this.operationsForOpsPerSec = operationsForOpsPerSec; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java new file mode 100644 index 00000000000..cd61ae91dae --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.jdisc.Metric; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.routing.ErrorDirective; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.yolean.Exceptions; + +/** + * Keeps an operation with its message. + * + * This implementation is based on V2, but the code is restructured. + * + * @author dybdahl + */ +class DocumentOperationMessageV3 { + + private final String operationId; + private final Message message; + + private DocumentOperationMessageV3(String operationId, Message message) { + this.operationId = operationId; + this.message = message; + } + + Message getMessage() { + return message; + } + + String getOperationId() { + return operationId; + } + + static DocumentOperationMessageV3 newErrorMessage(String operationId, Exception exception) { + Message feedErrorMessageV3 = new FeedErrorMessage(operationId); + DocumentOperationMessageV3 msg = new DocumentOperationMessageV3(operationId, feedErrorMessageV3); + Hop hop = new Hop(); + hop.addDirective(new ErrorDirective(Exceptions.toMessageString(exception))); + Route route = new Route(); + route.addHop(hop); + feedErrorMessageV3.setRoute(route); + return msg; + } + + static DocumentOperationMessageV3 newUpdateMessage(VespaXMLFeedReader.Operation op, String operationId) { + DocumentUpdate update = op.getDocumentUpdate(); + update.setCondition(op.getCondition()); + Message msg = new UpdateDocumentMessage(update); + + String id = (operationId == null) ? update.getId().toString() : operationId; + return new DocumentOperationMessageV3(id, msg); + } + + static DocumentOperationMessageV3 newRemoveMessage(VespaXMLFeedReader.Operation op, String operationId) { + DocumentRemove remove = new DocumentRemove(op.getRemove()); + remove.setCondition(op.getCondition()); + Message msg = new RemoveDocumentMessage(remove); + + String id = (operationId == null) ? remove.getId().toString() : operationId; + return new DocumentOperationMessageV3(id, msg); + } + + static DocumentOperationMessageV3 newPutMessage(VespaXMLFeedReader.Operation op, String operationId) { + DocumentPut put = new DocumentPut(op.getDocument()); + put.setCondition(op.getCondition()); + Message msg = new PutDocumentMessage(put); + + String id = (operationId == null) ? put.getId().toString() : operationId; + return new DocumentOperationMessageV3(id, msg); + } + + static DocumentOperationMessageV3 create(VespaXMLFeedReader.Operation operation, String operationId, Metric metric) { + switch (operation.getType()) { + case DOCUMENT: + metric.add(MetricNames.NUM_PUTS, 1, null /*metricContext*/); + return newPutMessage(operation, operationId); + case REMOVE: + metric.add(MetricNames.NUM_REMOVES, 1, null /*metricContext*/); + return newRemoveMessage(operation, operationId); + case UPDATE: + metric.add(MetricNames.NUM_UPDATES, 1, null /*metricContext*/); + return newUpdateMessage(operation, operationId); + default: + // typical end of feed + return null; + } + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java new file mode 100644 index 00000000000..6f7c08148ed --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorHttpResponse.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.io.OutputStream; + +public class ErrorHttpResponse extends HttpResponse { + + private final String msg; + + public ErrorHttpResponse(final int statusCode, final String msg) { + super(statusCode); + this.msg = msg; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(Utf8.toBytes(msg)); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java new file mode 100644 index 00000000000..45336ea911a --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.document.DocumentId; +import com.yahoo.messagebus.Message; +import com.yahoo.text.Utf8String; + +import java.util.Arrays; + +public class FeedErrorMessage extends Message { + + private long sequenceId; + + public FeedErrorMessage(String operationId) { + try { + DocumentId id = new DocumentId(operationId); + sequenceId = Arrays.hashCode(id.getGlobalId()); + } catch (Exception e) { + sequenceId = 0; + } + } + + @Override + public Utf8String getProtocol() { + return new Utf8String("vespa-feed-handler-internal-bogus-protocol"); + } + + @Override + public int getType() { + return 1234; + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return sequenceId; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java new file mode 100644 index 00000000000..3c532d94b24 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -0,0 +1,354 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.collections.Tuple2; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.http.HttpResponse.Status; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.net.LinuxInetAddress; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.GZIPInputStream; + +/** + * Accept feeds from outside of the Vespa cluster. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1 + */ +public class FeedHandler extends LoggingRequestHandler { + + private final ExecutorService workers = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("feedhandler")); + private final DocumentTypeManager docTypeManager; + private final Map<String, ClientState> clients; + private final ScheduledThreadPoolExecutor cron; + private final SessionCache sessionCache; + protected final ReplyHandler feedReplyHandler; + private final AtomicLong sessionId; + private final Metric metric; + private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(2)); + private final String localHostname; + private final FeedHandlerV3 feedHandlerV3; + + public FeedHandler( + Executor executor, + DocumentmanagerConfig documentManagerConfig, + SessionCache sessionCache, + Metric metric, + AccessLog accessLog, + ThreadpoolConfig threadpoolConfig) throws Exception { + super(executor, accessLog); + feedHandlerV3 = new FeedHandlerV3(executor, documentManagerConfig, sessionCache, metric, accessLog, threadpoolConfig); + docTypeManager = createDocumentManager(documentManagerConfig); + clients = new HashMap<>(); + this.sessionCache = sessionCache; + sessionId = new AtomicLong(new Random(System.currentTimeMillis()).nextLong()); + feedReplyHandler = new FeedReplyReader(metric); + cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron")); + cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES); + this.metric = metric; + this.localHostname = resolveLocalHostname(); + } + + /** + * Exposed for creating mocks. + */ + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return new DocumentTypeManager(documentManagerConfig); + } + + private class CleanClients implements Runnable { + + @Override + public void run() { + List<ClientState> clientsToShutdown = new ArrayList<>(); + long now = System.currentTimeMillis(); + + synchronized (clients) { + for (Iterator<Map.Entry<String, ClientState>> i = clients + .entrySet().iterator(); i.hasNext();) { + ClientState client = i.next().getValue(); + + if (now - client.creationTime > 10 * 60 * 1000) { + clientsToShutdown.add(client); + i.remove(); + } + } + } + for (ClientState client : clientsToShutdown) { + client.sourceSession.getReference().close(); + } + } + } + + private Tuple2<HttpResponse, Integer> checkProtocolVersion(HttpRequest request) { + return doCheckProtocolVersion(request.getJDiscRequest().headers().get(Headers.VERSION)); + } + + static Tuple2<HttpResponse, Integer> doCheckProtocolVersion(List<String> clientSupportedVersions) { + List<String> washedClientVersions = splitVersions(clientSupportedVersions); + + if (washedClientVersions == null || washedClientVersions.isEmpty()) { + return new Tuple2<>(new ErrorHttpResponse( + Headers.HTTP_NOT_ACCEPTABLE, + "Request did not contain " + Headers.VERSION + + "header. Server supports protocol versions " + + serverSupportedVersions), -1); + } + + //select the highest version supported by both parties + //this could be extended when we support a gazillion versions - but right now: keep it simple. + int version; + if (washedClientVersions.contains("3")) { + version = 3; + } else if (washedClientVersions.contains("2")) { + version = 2; + } else { + return new Tuple2<>(new ErrorHttpResponse( + Headers.HTTP_NOT_ACCEPTABLE, + "Could not parse " + Headers.VERSION + + "header of request (values: " + washedClientVersions + + "). Server supports protocol versions " + + serverSupportedVersions), -1); + } + return new Tuple2<>(null, version); + } + + private static List<String> splitVersions(List<String> clientSupportedVersions) { + List<String> splittedVersions = new ArrayList<>(); + for (String v : clientSupportedVersions) { + if (v == null || v.trim().isEmpty()) { + continue; + } + if (!v.contains(",")) { + splittedVersions.add(v.trim()); + continue; + } + for (String part : v.split(",")) { + part = part.trim(); + if (!part.isEmpty()) { + splittedVersions.add(part); + } + } + } + return splittedVersions; + } + + @Override + public HttpResponse handle(HttpRequest request) { + Tuple2<HttpResponse, Integer> protocolVersion = checkProtocolVersion(request); + + if (protocolVersion.first != null) { + return protocolVersion.first; + } + if (3 == protocolVersion.second) { + return feedHandlerV3.handle(request); + } + final BlockingQueue<OperationStatus> operations = new LinkedBlockingQueue<>(); + Tuple2<String, Boolean> clientId; + clientId = sessionId(request); + + if (clientId.second != null && clientId.second) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received initial request from client with session ID " + + clientId.first + ", protocol version " + protocolVersion.second); + } + } + + final Feeder feeder; + try { + feeder = createFeeder( + request, request.getData(), operations, clientId.first, clientId.second, protocolVersion.second); + // the synchronous FeedResponse blocks draining the InputStream, letting + // the Feeder read it + workers.submit(feeder); + } catch (UnknownClientException uce) { + final String msg = Exceptions.toMessageString(uce); + log.log(LogLevel.WARNING, msg); + return new ErrorHttpResponse(Status.BAD_REQUEST, msg); + } catch (Exception e) { + final String msg = "Could not initialize document parsing: " + + Exceptions.toMessageString(e); + log.log(LogLevel.WARNING, msg); + return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, msg); + } + + try { + feeder.waitForRequestReceived(); + } catch (InterruptedException e) { + return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + + return new FeedResponse(200, operations, + protocolVersion.second.intValue(), + clientId.first); + } + + // Protected for testing + protected static InputStream unzipStreamIfNeeded(final InputStream inputStream, final HttpRequest httpRequest) + throws IOException { + final String contentEncodingHeader = httpRequest.getHeader("content-encoding"); + if ("gzip".equals(contentEncodingHeader)) { + return new GZIPInputStream(inputStream); + } else { + return inputStream; + } + } + + /** + * Exposed for creating mocks. + */ + protected Feeder createFeeder( + HttpRequest request, + InputStream requestInputStream, + final BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, + int protocolVersion) throws Exception { + switch (protocolVersion) { + case 2: + return new Feeder( + unzipStreamIfNeeded(requestInputStream, request), + new FeedReaderFactory(), + docTypeManager, + operations, + popClient(clientId), + new FeederSettings(request), + clientId, + sessionIdWasGeneratedJustNow, + sourceSessionParams(request), + sessionCache, + this, + metric, + feedReplyHandler, + localHostname); + default: + throw new IllegalStateException("BUG! Protocol version " + protocolVersion + " not supported."); + } + } + + private Tuple2<String, Boolean> sessionId(HttpRequest request) { + boolean sessionIdWasGeneratedJustNow = false; + String sessionId = request.getHeader(Headers.SESSION_ID); + if (sessionId == null) { + sessionId = Long.toString(this.sessionId.incrementAndGet()) + "-" + + remoteHostAddressAndPort(request.getJDiscRequest()) + "#" + + localHostname; + sessionIdWasGeneratedJustNow = true; + } + return new Tuple2<>(sessionId, sessionIdWasGeneratedJustNow); + } + + private static String remoteHostAddressAndPort(com.yahoo.jdisc.http.HttpRequest httpRequest) { + SocketAddress remoteAddress = httpRequest.getRemoteAddress(); + if (remoteAddress instanceof InetSocketAddress) { + InetSocketAddress isa = (InetSocketAddress) remoteAddress; + return isa.getAddress().getHostAddress() + "-" + isa.getPort(); + } + return ""; + } + + private static String resolveLocalHostname() { + try { + InetAddress inetAddress = LinuxInetAddress.getLocalHost(); + String hostname = inetAddress.getCanonicalHostName(); + if (hostname.equals("localhost")) { + return ""; + } + return inetAddress.getCanonicalHostName(); + } catch (UnknownHostException e) { + return ""; + } + } + + /** + * Exposed for use when creating mocks. + */ + protected SourceSessionParams sourceSessionParams(HttpRequest request) { + SourceSessionParams params = new SourceSessionParams(); + String timeout = request.getHeader(Headers.TIMEOUT); + + if (timeout != null) { + try { + params.setTimeout(Double.parseDouble(timeout)); + } catch (NumberFormatException e) { + // NOP + } + } + return params; + } + + @Override + protected void destroy() { + feedHandlerV3.destroy(); + // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. + // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this + // and started destructing something that required something only the messenger thread could provide. + Thread destroyer = new Thread(() -> { + internalDestroy(); + }); + destroyer.setDaemon(true); + destroyer.start(); + } + + private void internalDestroy() { + super.destroy(); + workers.shutdown(); + cron.shutdown(); + synchronized (clients) { + for (ClientState client : clients.values()) { + client.sourceSession.getReference().close(); + } + clients.clear(); + } + } + + void putClient(final String sessionId, final ClientState value) { + synchronized (clients) { + clients.put(sessionId, value); + } + } + + ClientState popClient(String sessionId) { + synchronized (clients) { + return clients.remove(sessionId); + } + } + + /** + * Guess what, testing only. + */ + void forceRunCleanClients() { + new CleanClients().run(); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java new file mode 100644 index 00000000000..4694563ecbe --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.yolean.Exceptions; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +/** + * This code is based on v2 code, however, in v3, one client has one ClientFeederV3 shared between all client threads. + * The new API has more logic for shutting down cleanly as the server is more likely to be upgraded. + * The code is restructured a bit. + * + * @author dybdahl + */ +public class FeedHandlerV3 extends LoggingRequestHandler { + + private DocumentTypeManager docTypeManager; + private final Map<String, ClientFeederV3> clientFeederByClientId = new HashMap<>(); + private final ScheduledThreadPoolExecutor cron; + private final SessionCache sessionCache; + protected final ReplyHandler feedReplyHandler; + private final Metric metric; + private final Object monitor = new Object(); + private final AtomicInteger threadsAvailableForFeeding; + private static final Logger log = Logger.getLogger(FeedHandlerV3.class.getName()); + + public FeedHandlerV3( + Executor executor, + DocumentmanagerConfig documentManagerConfig, + SessionCache sessionCache, + Metric metric, + AccessLog accessLog, + ThreadpoolConfig threadpoolConfig) throws Exception { + super(executor, accessLog); + docTypeManager = new DocumentTypeManager(documentManagerConfig); + this.sessionCache = sessionCache; + feedReplyHandler = new FeedReplyReader(metric); + cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron")); + cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES); + this.metric = metric; + // Half of the threads can be blocking on feeding before we deny requests. + if (threadpoolConfig != null) { + threadsAvailableForFeeding = new AtomicInteger(threadpoolConfig.maxthreads() / 2); + } else { + log.warning("No config for threadpool, using 250 for max blocking threads for feeding."); + threadsAvailableForFeeding = new AtomicInteger(250); + } + } + + public void injectDocumentManangerForTests(DocumentTypeManager docTypeManager) { + this.docTypeManager = docTypeManager; + } + + // TODO: If this is set up to run without first invoking the old FeedHandler code, we should + // verify the version header first. This is done in the old code. + @Override + public HttpResponse handle(HttpRequest request) { + final String clientId = clientId(request); + final ClientFeederV3 clientFeederV3; + synchronized (monitor) { + if (! clientFeederByClientId.containsKey(clientId)) { + SourceSessionParams sourceSessionParams = sourceSessionParams(request); + clientFeederByClientId.put( + clientId, + new ClientFeederV3( + retainSource(sessionCache, sourceSessionParams), + new FeedReaderFactory(), + docTypeManager, + clientId, + metric, + feedReplyHandler, + threadsAvailableForFeeding)); + } + clientFeederV3 = clientFeederByClientId.get(clientId); + } + try { + return clientFeederV3.handleRequest(request); + } catch (UnknownClientException uce) { + final String msg = Exceptions.toMessageString(uce); + log.log(LogLevel.WARNING, msg); + return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST, msg); + } catch (Exception e) { + final String msg = "Could not initialize document parsing: " + + Exceptions.toMessageString(e); + log.log(LogLevel.WARNING, msg); + return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.INTERNAL_SERVER_ERROR, msg); + } + } + + // SessionCache is final and no easy way to mock it so we need this to be able to do testing. + protected ReferencedResource<SharedSourceSession> retainSource(SessionCache sessionCache, SourceSessionParams params) { + return sessionCache.retainSource(params); + } + + @Override + protected void destroy() { + // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. + // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this + // and started destructing something that required something only the messenger thread could provide. + Thread destroyer = new Thread(() -> { + super.destroy(); + cron.shutdown(); + synchronized (monitor) { + for (ClientFeederV3 client : clientFeederByClientId.values()) { + client.kill(); + } + clientFeederByClientId.clear(); + } + }); + destroyer.setDaemon(true); + destroyer.start(); + } + + private String clientId(HttpRequest request) { + String clientDictatedId = request.getHeader(Headers.CLIENT_ID); + if (clientDictatedId == null || clientDictatedId.isEmpty()) { + throw new IllegalArgumentException("Did not get any CLIENT_ID header (" + Headers.CLIENT_ID + ")"); + } + return clientDictatedId; + } + + private SourceSessionParams sourceSessionParams(HttpRequest request) { + SourceSessionParams params = new SourceSessionParams(); + String timeout = request.getHeader(Headers.TIMEOUT); + + if (timeout != null) { + try { + params.setTimeout(Double.parseDouble(timeout)); + } catch (NumberFormatException e) { + // NOP + } + } + return params; + } + + private void removeOldClients() { + synchronized (monitor) { + for (Iterator<Map.Entry<String, ClientFeederV3>> iterator = clientFeederByClientId + .entrySet().iterator(); iterator.hasNext();) { + ClientFeederV3 client = iterator.next().getValue(); + if (client.timedOut()) { + client.kill(); + iterator.remove(); + } + } + } + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java new file mode 100644 index 00000000000..67d6acd926a --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.json.JsonFeedReader; +import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.io.InputStream; + +/** + * Class for creating FeedReader based on dataFormat. + * @author dybdahl + */ +public class FeedReaderFactory { + + /** + * Creates FeedReader + * @param inputStream source of feed data + * @param docTypeManager handles the parsing of the document + * @param dataFormat specifies the format + * @return a feedreader + */ + public FeedReader createReader( + InputStream inputStream, + DocumentTypeManager docTypeManager, + FeedParams.DataFormat dataFormat) { + switch (dataFormat) { + case XML_UTF8: + try { + return new VespaXMLFeedReader(inputStream, docTypeManager); + } catch (Exception e) { + throw new RuntimeException("Could not create VespaXMLFeedReader", e); + } + case JSON_UTF8: + return new JsonFeedReader(inputStream, docTypeManager); + default: + throw new IllegalStateException("Can not create feed reader for format: " + dataFormat); + } + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java new file mode 100644 index 00000000000..b7ecf4211a9 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.logging.Logger; + +import com.yahoo.jdisc.Metric; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Trace; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.OperationStatus; + +/** + * Catch message bus replies and make the available to a given session. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class FeedReplyReader implements ReplyHandler { + + private static final Logger log = Logger.getLogger(FeedReplyReader.class.getName()); + private final Metric metric; + + public FeedReplyReader(Metric metric) { + this.metric = metric; + } + + @Override + public void handleReply(Reply reply) { + Object o = reply.getContext(); + if (!(o instanceof ReplyContext)) { + return; + } + ReplyContext context = (ReplyContext) o; + metric.set( + MetricNames.LATENCY, + Double.valueOf((System.currentTimeMillis() - context.creationTime) / 1000.0d), + null); + if (reply.hasErrors()) { + metric.add(MetricNames.FAILED, 1, null); + enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, reply.getTrace()); + } else { + metric.add(MetricNames.SUCCEEDED, 1, null); + enqueue(context, "Document processed.", ErrorCode.OK, reply.getTrace()); + } + } + + private void enqueue(ReplyContext context, String message, ErrorCode status, Trace trace) { + try { + String traceMessage = (trace != null && trace.getLevel() > 0) ? trace.toString() : ""; + context.feedReplies.put(new OperationStatus(message, + context.docId.toString(), status, traceMessage)); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, + "Interrupted while enqueueing result from putting document with id: " + + context.docId.toString()); + Thread.currentThread().interrupt(); + } + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java new file mode 100644 index 00000000000..c6ad51242dc --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.OperationStatus; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.BlockingQueue; + +/** + * Reads feed responses from a queue and renders them continuously to the + * feeder. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1 + */ +public class FeedResponse extends HttpResponse { + + BlockingQueue<OperationStatus> operations; + + public FeedResponse( + int status, + BlockingQueue<OperationStatus> operations, + int protocolVersion, + String sessionId) { + super(status); + this.operations = operations; + headers().add(Headers.SESSION_ID, sessionId); + headers().add(Headers.VERSION, Integer.toString(protocolVersion)); + } + + // This is used by the V3 protocol. + public FeedResponse( + int status, + BlockingQueue<OperationStatus> operations, + int protocolVersion, + String sessionId, + int outstandingClientOperations, + String hostName) { + super(status); + this.operations = operations; + headers().add(Headers.SESSION_ID, sessionId); + headers().add(Headers.VERSION, Integer.toString(protocolVersion)); + headers().add(Headers.OUTSTANDING_REQUESTS, Integer.toString(outstandingClientOperations)); + headers().add(Headers.HOSTNAME, hostName); + } + + @Override + public void render(OutputStream output) throws IOException { + int i = 0; + OperationStatus status; + try { + status = operations.take(); + while (status.errorCode != ErrorCode.END_OF_FEED) { + output.write(toBytes(status.render())); + if (++i % 5 == 0) { + output.flush(); + } + status = operations.take(); + } + } catch (InterruptedException e) { + output.flush(); + } + } + + private byte[] toBytes(String s) { + byte[] b = new byte[s.length()]; + for (int i = 0; i < b.length; ++i) { + b[i] = (byte) s.charAt(i); // renderSingleStatus ensures ASCII only + } + return b; + } + + @Override + public String getContentType() { + return "text/plain"; + } + + @Override + public String getCharacterEncoding() { + return StandardCharsets.US_ASCII.name(); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java new file mode 100644 index 00000000000..106afe6ffe9 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java @@ -0,0 +1,542 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.routing.ErrorDirective; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.yolean.Exceptions; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.http.client.core.Encoder; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL; + +/** + * Read documents from client, and send them through message bus. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Feeder implements Runnable { + + protected static final Logger log = Logger.getLogger(Feeder.class.getName()); + + final InputStream requestInputStream; + final DocumentTypeManager docTypeManager; + final BlockingQueue<OperationStatus> operations; + final BlockingQueue<OperationStatus> feedReplies; + int numPending; + final FeederSettings settings; + final String clientId; + final ReferencedResource<SharedSourceSession> sourceSession; + final FeedHandler handler; + final Metric metric; + final Metric.Context metricContext; + private long prevOpsPerSecTime; // previous measurement time of OPS + private double operationsForOpsPerSec; + private final ReplyHandler feedReplyHandler; + protected final static String EOF = "End of stream"; + protected final boolean sessionIdWasGeneratedJustNow; + private final CountDownLatch requestReceived = new CountDownLatch(1); + private final FeedReaderFactory feedReaderFactory; + + // TODO refactor this perverse pile of constructor arguments + public Feeder(InputStream requestInputStream, + FeedReaderFactory feedReaderFactory, + DocumentTypeManager docTypeManager, + BlockingQueue<OperationStatus> operations, + ClientState storedState, + FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, + FeedHandler handler, Metric metric, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(); + this.feedReaderFactory = feedReaderFactory; + if (storedState == null) { + if (!sessionIdWasGeneratedJustNow) { + // We do not have a stored state, BUT the session ID came in with the request. + // Possible session timeout, server restart, server reconfig, or VIP usage. Examine. + examineClientId(clientId, localHostname); + } + numPending = 0; + feedReplies = new LinkedBlockingQueue<>(); + sourceSession = retainSession(sessionParams, sessionCache); + metricContext = createClientMetricContext(metric, clientId); + prevOpsPerSecTime = System.currentTimeMillis(); + operationsForOpsPerSec = 0.0; + } else { + //we have a stored state, and the session ID was obviously not generated now. All OK. + numPending = storedState.pending; + feedReplies = storedState.feedReplies; + sourceSession = storedState.sourceSession; + metricContext = storedState.metricContext; + prevOpsPerSecTime = storedState.prevOpsPerSecTime; + operationsForOpsPerSec = storedState.operationsForOpsPerSec; + } + this.clientId = clientId; + this.sessionIdWasGeneratedJustNow = sessionIdWasGeneratedJustNow; + this.requestInputStream = requestInputStream; + this.docTypeManager = docTypeManager; + this.operations = operations; + this.settings = settings; + this.handler = handler; + this.metric = metric; + this.feedReplyHandler = feedReplyHandler; + } + protected void examineClientId(String clientId, String localHostname) { + if (!clientId.contains("#")) { + throw new UnknownClientException("Got request from client with id '" + clientId + + "', but found no session for this client. " + + "Most probably this server is in VIP rotation, " + + "and a client session was rotated from one server to another. " + + "This must not happen. Configure VIP with persistence=enabled, " + + "or (preferably) do not use a VIP at all."); + } + int hashPos = clientId.indexOf("#"); + String supposedHostname = clientId.substring(hashPos + 1, clientId.length()); + if (supposedHostname.isEmpty()) { + throw new UnknownClientException("Got request from client with id '" + clientId + + "', but found no session for this client. Possible session " + + "timeout due to inactivity, server restart or reconfig, " + + "or bad VIP usage."); + } + + if (!supposedHostname.equals(localHostname)) { + throw new UnknownClientException("Got request from client with id '" + clientId + + "', but found no session for this client. " + + "Session was originally established towards host " + + supposedHostname + ", but our hostname is " + + localHostname + ". " + + "Most probably this server is in VIP rotation, " + + "and a session was rotated from one server to another. " + + "This should not happen. Configure VIP with persistence=enabled, " + + "or (preferably) do not use a VIP at all."); + } + log.log(LogLevel.DEBUG, "Client '" + clientId + "' reconnected after session inactivity, or server restart " + + "or reconfig. Re-establishing session."); + } + + + + private static Metric.Context createClientMetricContext(Metric metric, String clientId) { + // No real value in separate metric dimensions per client. + return null; + } + + /** + * Exposed for creating mocks. + */ + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + return sessionCache.retainSource(sessionParams); + } + + @Override + public void run() { + try { + if (handshake()) { + return; //will putClient in finally block below + } + flushResponseQueue(); + feed(); + requestReceived.countDown(); + drain(); + } catch (InterruptedException e) { + // NOP, just terminate + } catch (Exception e) { + log.log(LogLevel.WARNING, "Unhandled exception while feeding: " + + Exceptions.toMessageString(e), e); + } catch (Throwable e) { + log.log(LogLevel.WARNING, "Unhandled error while feeding: " + + Exceptions.toMessageString(e), e); + throw e; + } finally { + requestReceived.countDown(); + putClient(); + try { + enqueue("-", "-", ErrorCode.END_OF_FEED, null); + } catch (InterruptedException e) { + // NOP, we are already exiting the thread + } + } + } + + protected boolean handshake() throws IOException { + if (sessionIdWasGeneratedJustNow) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Handshake completed for client " + clientId + "."); + } + requestInputStream.close(); + return true; + } + return false; + } + + void feed() throws InterruptedException { + while (true) { + Result result; + String operationId; + try { + operationId = getNextOperationId(); + } catch (IOException ioe) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); + } + break; + } + + //noinspection StringEquality + if (operationId == EOF) { + break; + } + + Tuple2<String, Message> msg; + try { + msg = getNextMessage(operationId); + setRoute(msg); + } catch (Exception e) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); + } + //noinspection StringEquality + if (operationId != null) { //v1 always returns null, all others return something useful, or throw an exception above + msg = newErrorMessage(operationId, e); + } else { + break; + } + } + + if (msg == null) { + break; + } + + setMessageParameters(msg); + + while (true) { + try { + msg.second.pushHandler(feedReplyHandler); + if (settings.denyIfBusy) { + result = sourceSession.getResource().sendMessage(msg.second); + } else { + result = sourceSession.getResource().sendMessageBlocking(msg.second); + } + } catch (RuntimeException e) { + enqueue(msg.first, Exceptions.toMessageString(e), + ErrorCode.ERROR, msg.second); + return; + } + if (result.isAccepted() || result.getError().getCode() != SEND_QUEUE_FULL) { + break; + } + if (settings.denyIfBusy) { + break; + } else { + //This will never happen + Thread.sleep(100); + } + } + + if (result.isAccepted()) { + ++numPending; + updateMetrics(msg.second); + updateOpsPerSec(); + log(LogLevel.DEBUG, "Sent message successfully, document id: ", + msg.first); + } else if (!result.getError().isFatal()) { + enqueue(msg.first, result.getError().getMessage(), + ErrorCode.TRANSIENT_ERROR, msg.second); + break; + } else { + // should probably not happen, but everybody knows stuff that + // shouldn't happen, happens all the time + enqueue(msg.first, result.getError().getMessage(), + ErrorCode.ERROR, msg.second); + break; + } + } + } + + private Tuple2<String, Message> newErrorMessage(String operationId, Exception e) { + Message m = new FeedErrorMessage(operationId); + Tuple2<String, Message> msg = new Tuple2<>(operationId, m); + Hop hop = new Hop(); + hop.addDirective(new ErrorDirective(Exceptions.toMessageString(e))); + Route route = new Route(); + route.addHop(hop); + m.setRoute(route); + return msg; + } + + private void updateMetrics(Message m) { + metric.set( + MetricNames.PENDING, + Double.valueOf(sourceSession.getResource().session().getPendingCount()), + null); + + metric.add(MetricNames.NUM_OPERATIONS, 1, metricContext); + + if (m instanceof PutDocumentMessage) { + metric.add(MetricNames.NUM_PUTS, 1, metricContext); + } else if (m instanceof RemoveDocumentMessage) { + metric.add(MetricNames.NUM_REMOVES, 1, metricContext); + } else if (m instanceof UpdateDocumentMessage) { + metric.add(MetricNames.NUM_UPDATES, 1, metricContext); + } + } + + private void updateOpsPerSec() { + long now = System.currentTimeMillis(); + if ((now - prevOpsPerSecTime) >= 1000) { //every second + double ms = (double) (now - prevOpsPerSecTime); + final double opsPerSec = operationsForOpsPerSec / (ms / 1000); + metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, metricContext); + operationsForOpsPerSec = 1.0d; + prevOpsPerSecTime = now; + } else { + operationsForOpsPerSec += 1.0d; + } + } + + private Tuple2<String, Message> getNextMessage(String operationId) throws Exception { + VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + Tuple2<String, Message> msg; + + getNextOperation(op); + + switch (op.getType()) { + case DOCUMENT: + msg = newPutMessage(op, operationId); + break; + case REMOVE: + msg = newRemoveMessage(op, operationId); + break; + case UPDATE: + msg = newUpdateMessage(op, operationId); + break; + default: + // typical end of feed + return null; + } + log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.first); + return msg; + } + + private void setMessageParameters(Tuple2<String, Message> msg) { + msg.second.setContext(new ReplyContext(msg.first, feedReplies)); + if (settings.traceLevel != null) { + msg.second.getTrace().setLevel(settings.traceLevel); + } + if (settings.priority != null) { + try { + DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); + if (msg.second instanceof DocumentMessage) { + ((DocumentMessage) msg.second).setPriority(priority); + } + } + catch (IllegalArgumentException i) { + log.severe(i.getMessage()); + } + } + } + + private void setRoute(Tuple2<String, Message> msg) { + if (settings.route != null) { + msg.second.setRoute(settings.route); + } + } + + protected void getNextOperation(VespaXMLFeedReader.Operation op) throws Exception { + int length = readByteLength(); + + try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ + FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); + reader.read(op); + } + } + + protected String getNextOperationId() throws IOException { + return readOperationId(); + } + + private String readOperationId() throws IOException { + StringBuilder idBuf = new StringBuilder(100); + int c; + while ((c = requestInputStream.read()) != -1) { + if (c == 32) { + break; + } + idBuf.append((char) c); //it's ASCII + } + if (idBuf.length() == 0) { + return EOF; + } + return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); + } + + private int readByteLength() throws IOException { + StringBuilder lenBuf = new StringBuilder(8); + int c; + while ((c = requestInputStream.read()) != -1) { + if (c == 10) { + break; + } + lenBuf.append((char) c); //it's ASCII + } + if (lenBuf.length() == 0) { + throw new IllegalStateException("Operation length missing."); + } + return Integer.valueOf(lenBuf.toString(), 16); + } + + protected final void log(LogLevel level, Object... msgParts) { + StringBuilder s; + + if (!log.isLoggable(level)) { + return; + } + + s = new StringBuilder(); + for (Object part : msgParts) { + s.append(part.toString()); + } + + log.log(level, s.toString()); + } + + private Tuple2<String, Message> newUpdateMessage(Operation op, String operationId) { + DocumentUpdate update = op.getDocumentUpdate(); + update.setCondition(op.getCondition()); + Message msg = new UpdateDocumentMessage(update); + + String id = (operationId == null) ? update.getId().toString() : operationId; + return new Tuple2<>(id, msg); + } + + private Tuple2<String, Message> newRemoveMessage(Operation op, String operationId) { + DocumentRemove remove = new DocumentRemove(op.getRemove()); + remove.setCondition(op.getCondition()); + Message msg = new RemoveDocumentMessage(remove); + + String id = (operationId == null) ? remove.getId().toString() : operationId; + return new Tuple2<>(id, msg); + } + + private Tuple2<String, Message> newPutMessage(Operation op, String operationId) { + DocumentPut put = new DocumentPut(op.getDocument()); + put.setCondition(op.getCondition()); + Message msg = new PutDocumentMessage(put); + + String id = (operationId == null) ? put.getId().toString() : operationId; + return new Tuple2<>(id, msg); + } + + + void flushResponseQueue() throws InterruptedException { + OperationStatus status = feedReplies.poll(); + while (status != null) { + decreasePending(status); + status = feedReplies.poll(); + } + } + + void putClient() { + handler.putClient(clientId, + new ClientState(numPending, + feedReplies, sourceSession, metricContext, + prevOpsPerSecTime, operationsForOpsPerSec)); + } + + void drain() throws InterruptedException { + if (settings.drain) { + while (numPending > 0) { + OperationStatus o = feedReplies.take(); + decreasePending(o); + } + } + } + + private void decreasePending(OperationStatus o) throws InterruptedException { + operations.put(o); + --numPending; + } + + private void enqueue(String id, String message, ErrorCode code, Message msg) + throws InterruptedException { + String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0 + ? msg.getTrace().toString() + : ""; + operations.put(new OperationStatus(message, id, code, traceMessage)); + } + + public void waitForRequestReceived() throws InterruptedException { + requestReceived.await(1, TimeUnit.HOURS); + } + + public class FeedErrorMessage extends Message { + private long sequenceId; + + private FeedErrorMessage(String operationId) { + try { + DocumentId id = new DocumentId(operationId); + sequenceId = Arrays.hashCode(id.getGlobalId()); + } catch (Exception e) { + sequenceId = 0; + } + } + + @Override + public Utf8String getProtocol() { + return new Utf8String("vespa-feed-handler-internal-bogus-protocol"); + } + + @Override + public int getType() { + return 1234; + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return sequenceId; + } + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java new file mode 100644 index 00000000000..64bc687d425 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; +import com.yahoo.vespa.http.client.core.Headers; + +/** + * Wrapper for the feed feederSettings read from HTTP request. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class FeederSettings { + + private static final Route DEFAULT_ROUTE = Route.parse("default"); + public final boolean drain; + public final Route route; + public final boolean denyIfBusy; + public final DataFormat dataFormat; + public final String priority; + public final Integer traceLevel; + + public FeederSettings(HttpRequest request) { + { + String tmpDrain = request.getHeader(Headers.DRAIN); + if (tmpDrain != null) { + drain = Boolean.parseBoolean(tmpDrain); + } else { + drain = false; + } + } + { + String tmpRoute = request.getHeader(Headers.ROUTE); + if (tmpRoute != null) { + route = Route.parse(tmpRoute); + } else { + route = DEFAULT_ROUTE; + } + } + { + String tmpDenyIfBusy = request.getHeader(Headers.DENY_IF_BUSY); + if (tmpDenyIfBusy != null) { + denyIfBusy = Boolean.parseBoolean(tmpDenyIfBusy); + } else { + denyIfBusy = false; + } + } + { + String tmpDataFormat = request.getHeader(Headers.DATA_FORMAT); + if (tmpDataFormat != null) { + dataFormat = DataFormat.valueOf(tmpDataFormat); + } else { + dataFormat = DataFormat.XML_UTF8; + } + } + { + String tmpDataFormat = request.getHeader(Headers.PRIORITY); + if (tmpDataFormat != null) { + priority = tmpDataFormat; + } else { + priority = null; + } + } + { + String tmpDataFormat = request.getHeader(Headers.TRACE_LEVEL); + if (tmpDataFormat != null) { + traceLevel = Integer.valueOf(tmpDataFormat); + } else { + traceLevel = null; + } + } + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java new file mode 100644 index 00000000000..087cb11c350 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * Place to store the metric names so where the metrics are logged can be found + * more easily in an IDE. + * + * @author steinar + */ +public final class MetricNames { + + private static final String PREFIX = "httpapi_"; + + public static final String NUM_OPERATIONS = PREFIX + "num_operations"; + public static final String NUM_PUTS = PREFIX + "num_puts"; + public static final String NUM_REMOVES = PREFIX + "num_removes"; + public static final String NUM_UPDATES = PREFIX + "num_updates"; + public static final String OPERATIONS_PER_SEC = PREFIX + "ops_per_sec"; + public static final String LATENCY = PREFIX + "latency"; + public static final String FAILED = PREFIX + "failed"; + public static final String SUCCEEDED = PREFIX + "succeeded"; + public static final String PENDING = PREFIX + "pending"; + + private MetricNames() { + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java new file mode 100644 index 00000000000..30d8b04131d --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.vespa.http.client.core.OperationStatus; + +import java.util.concurrent.BlockingQueue; + +/** + * Mapping between document ID and client session. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ReplyContext { + + public final String docId; + public final BlockingQueue<OperationStatus> feedReplies; + public final long creationTime; + + public ReplyContext(String docId, BlockingQueue<OperationStatus> feedReplies) { + this.docId = docId; + this.feedReplies = feedReplies; + this.creationTime = System.currentTimeMillis(); + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java new file mode 100644 index 00000000000..0453d41fab8 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.vespa.http.client.core.Encoder; +import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; + +/** + * This code is based on v2 code, but restructured so stream reading code is in one dedicated class. + * @author dybdahl + */ +public class StreamReaderV3 { + + protected static final Logger log = Logger.getLogger(StreamReaderV3.class.getName()); + + private final FeedReaderFactory feedReaderFactory; + private final DocumentTypeManager docTypeManager; + + public StreamReaderV3(FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager) { + this.feedReaderFactory = feedReaderFactory; + this.docTypeManager = docTypeManager; + } + + public VespaXMLFeedReader.Operation getNextOperation( + InputStream requestInputStream, FeederSettings settings) throws Exception { + VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + + int length = readByteLength(requestInputStream); + + try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ + FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); + reader.read(op); + } + return op; + } + + public Optional<String> getNextOperationId(InputStream requestInputStream) throws IOException { + StringBuilder idBuf = new StringBuilder(100); + int c; + while ((c = requestInputStream.read()) != -1) { + if (c == 32) { + break; + } + idBuf.append((char) c); //it's ASCII + } + if (idBuf.length() == 0) { + return Optional.empty(); + } + return Optional.of(Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString()); + } + + private int readByteLength(InputStream requestInputStream) throws IOException { + StringBuilder lenBuf = new StringBuilder(8); + int c; + while ((c = requestInputStream.read()) != -1) { + if (c == 10) { + break; + } + lenBuf.append((char) c); //it's ASCII + } + if (lenBuf.length() == 0) { + throw new IllegalStateException("Operation length missing."); + } + return Integer.valueOf(lenBuf.toString(), 16); + } + + public static InputStream unzipStreamIfNeeded(final HttpRequest httpRequest) + throws IOException { + final String contentEncodingHeader = httpRequest.getHeader("content-encoding"); + if ("gzip".equals(contentEncodingHeader)) { + return new GZIPInputStream(httpRequest.getData()); + } else { + return httpRequest.getData(); + } + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java new file mode 100644 index 00000000000..ce605a82dbc --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.5.0 + */ +public class UnknownClientException extends RuntimeException { + + public UnknownClientException(String message) { + super(message); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java new file mode 100644 index 00000000000..1707069871b --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Server side of programmatic API for feeding into Vespa from outside of the + * clusters. Not a public API, not meant for direct use. + */ +@com.yahoo.api.annotations.PackageMarker +package com.yahoo.vespa.http.server; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java new file mode 100644 index 00000000000..a720611c91e --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server.util; + +import java.io.IOException; +import java.io.InputStream; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * + * @since 5.1.23 + */ +public class ByteLimitedInputStream extends InputStream { + + private final InputStream wrappedStream; + private int remaining; + + public ByteLimitedInputStream(InputStream wrappedStream, int limit) { + this.wrappedStream = wrappedStream; + if (limit < 0) { + throw new IllegalArgumentException("limit cannot be 0"); + } + this.remaining = limit; + } + + @Override + public int read() throws IOException { + if (remaining <= 0) { + return -1; + } + int retval = wrappedStream.read(); + if (retval < 0) { + remaining = 0; + } else { + --remaining; + } + return retval; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + if (remaining <= 0) { + return -1; + } + + int bytesToRead = Math.min(remaining, len); + int retval = wrappedStream.read(b, off, bytesToRead); + + if (retval < 0) { + //end of underlying stream was reached, and nothing was read. + remaining = 0; + } else { + remaining -= retval; + } + return retval; + } + + @Override + public int available() throws IOException { + return remaining; + } + + @Override + public void close() throws IOException { + //we will never close the underlying stream + if (remaining <= 0) { + return; + } + while (remaining > 0) { + skip(remaining); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/application/services.xml b/vespaclient-container-plugin/src/test/application/services.xml new file mode 100644 index 00000000000..df178e109c3 --- /dev/null +++ b/vespaclient-container-plugin/src/test/application/services.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<jdisc version="1.0" jetty="true"> + + <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test"> + <binding>http://*/document/v1/*</binding> + </handler> + + <component id="injected" class="com.yahoo.document.restapi.resource.MockedOperationHandler" bundle="integration-test"> + </component> + + + <http> + <!-- This indicates that we want JDisc to allocate a port for us --> + <server id="mainServer" port="0" /> + </http> +</jdisc> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg new file mode 100644 index 00000000000..6878ddffb79 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg @@ -0,0 +1,113 @@ +enablecompression false +datatype[10] +datatype[0].id 1002 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 2 +datatype[0].weightedsettype[0] +datatype[0].structtype[0] +datatype[0].documenttype[0] +datatype[1].id 1000 +datatype[1].arraytype[1] +datatype[1].arraytype[0].datatype 0 +datatype[1].weightedsettype[0] +datatype[1].structtype[0] +datatype[1].documenttype[0] +datatype[2].id 1004 +datatype[2].arraytype[1] +datatype[2].arraytype[0].datatype 4 +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[0] +datatype[3].id 1016 +datatype[3].arraytype[1] +datatype[3].arraytype[0].datatype 16 +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[4].id 1001 +datatype[4].arraytype[1] +datatype[4].arraytype[0].datatype 1 +datatype[4].weightedsettype[0] +datatype[4].structtype[0] +datatype[4].documenttype[0] +datatype[5].id 2001 +datatype[5].arraytype[0] +datatype[5].weightedsettype[1] +datatype[5].weightedsettype[0].datatype 0 +datatype[5].weightedsettype[0].createifnonexistant false +datatype[5].weightedsettype[0].removeifzero false +datatype[5].structtype[0] +datatype[5].documenttype[0] +datatype[6].id 2002 +datatype[6].arraytype[0] +datatype[6].weightedsettype[1] +datatype[6].weightedsettype[0].datatype 2 +datatype[6].weightedsettype[0].createifnonexistant false +datatype[6].weightedsettype[0].removeifzero false +datatype[6].structtype[0] +datatype[6].documenttype[0] +datatype[7].id -628990518 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name news.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[6] +datatype[7].structtype[0].field[0].name url +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype 10 +datatype[7].structtype[0].field[1].name title +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype 2 +datatype[7].structtype[0].field[2].name last_downloaded +datatype[7].structtype[0].field[2].id[0] +datatype[7].structtype[0].field[2].datatype 0 +datatype[7].structtype[0].field[3].name value_long +datatype[7].structtype[0].field[3].id[0] +datatype[7].structtype[0].field[3].datatype 4 +datatype[7].structtype[0].field[4].name value_content +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 3 +datatype[7].structtype[0].field[5].name stringarr +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 1002 +datatype[7].documenttype[0] +datatype[8].id 538588767 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name news.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[7] +datatype[8].structtype[0].field[0].name intarr +datatype[8].structtype[0].field[0].id[0] +datatype[8].structtype[0].field[0].datatype 1000 +datatype[8].structtype[0].field[1].name longarr +datatype[8].structtype[0].field[1].id[0] +datatype[8].structtype[0].field[1].datatype 1004 +datatype[8].structtype[0].field[2].name bytearr +datatype[8].structtype[0].field[2].id[0] +datatype[8].structtype[0].field[2].datatype 1016 +datatype[8].structtype[0].field[3].name floatarr +datatype[8].structtype[0].field[3].id[0] +datatype[8].structtype[0].field[3].datatype 1001 +datatype[8].structtype[0].field[4].name weightedsetint +datatype[8].structtype[0].field[4].id[0] +datatype[8].structtype[0].field[4].datatype 2001 +datatype[8].structtype[0].field[5].name weightedsetstring +datatype[8].structtype[0].field[5].id[0] +datatype[8].structtype[0].field[5].datatype 2002 +datatype[8].structtype[0].field[6].name content +datatype[8].structtype[0].field[6].id[0] +datatype[8].structtype[0].field[6].datatype 3 +datatype[8].documenttype[0] +datatype[9].id -1048827947 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name news +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -628990518 +datatype[9].documenttype[0].bodystruct 538588767 diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml new file mode 100644 index 00000000000..52739246b1f --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml @@ -0,0 +1,91 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<!-- + Document : test10.xml + Created on : July 27, 2007, 11:37 AM + Author : alimf + Description: + this feed contains both documents, updates and removes. +--> + +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="doc:news:http://news10b"> + <url>testUrl2</url> + </document> + + <update documenttype="news" documentid="doc:news:http://news10c"> + <add field="stringarr"> + <item>addString1</item> + <item>addString2</item> + </add> + <add field="longarr"> + <item>5</item> + </add> + <add field="longarr">6</add> + <add field="weightedsetint"> + <item weight="11">11</item> + <item weight="12">12</item> + </add> + <add field="weightedsetstring"> + <item>add13</item> + </add> + <add field="weightedsetstring">add14</add> + </update> + + <update documenttype="news" documentid="doc:news:http://news10d"> + <assign field="url">assignUrl</assign> + <assign field="value_long">2</assign> + <assign field="stringarr"> + <item>assignString1</item> + <item>assignString2</item> + </assign> + <assign field="intarr"> + <item>3</item> + <item>4</item> + </assign> + <assign field="weightedsetint"> + <item weight="11">11</item> + <item weight="12">12</item> + </assign> + </update> + + <remove documentid="doc:news:http://news10e"/> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml new file mode 100644 index 00000000000..44762f594e0 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<!-- + Document : test10.xml + Created on : July 27, 2007, 11:37 AM + Author : alimf + Description: + this feed contains both documents, updates and removes. +--> + +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="doc:news:http://news10b"> + <url>testUrl2</url> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml new file mode 100644 index 00000000000..a62df0c645a --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="foobar:news:http://news12"> + <url>testUrl2</url> + </document> + + <document documenttype="news" documentid="doc:news:ok"> + <url>testUrl2</url> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml new file mode 100755 index 00000000000..434eaa7789c --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + <document documenttype="news" documentid="foobar:news:http://news12"> + <url>testUrl2</url> + </document> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml new file mode 100755 index 00000000000..f0b36ea3bc9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype=news documentid=foobar:news:http://news12 + <url>testUrl2</url> + </document> +</vespafed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_removes b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes new file mode 100755 index 00000000000..0232970ed4f --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes @@ -0,0 +1,2 @@ +doc:test:remove1 +doc:test:remove2 diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java new file mode 100644 index 00000000000..87360fcf998 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.yahoo.vespaclient.ClusterDef; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + + +public class OperationHandlerImplTest { + + @Test(expected = IllegalArgumentException.class) + public void missingClusterDef() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + OperationHandlerImpl.resolveClusterRoute(Optional.empty(), clusterDef); + } + + @Test(expected = IllegalArgumentException.class) + public void missingClusterDefSpecifiedCluster() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef); + } + + @Test(expected = RestApiException.class) + public void oneClusterPresentNotMatching() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo", "configId")); + OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef); + } + + @Test() + public void oneClusterMatching() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo", "configId")); + assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef), + is("[Storage:cluster=foo;clusterconfigid=configId]")); + } + + @Test() + public void oneClusterMatchingManyAvailable() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo2", "configId2")); + clusterDef.add(new ClusterDef("foo", "configId")); + clusterDef.add(new ClusterDef("foo3", "configId2")); + assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef), + is("[Storage:cluster=foo;clusterconfigid=configId]")); + } + + @Test() + public void checkErrorMessage() throws RestApiException, IOException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo2", "configId2")); + clusterDef.add(new ClusterDef("foo", "configId")); + clusterDef.add(new ClusterDef("foo3", "configId2")); + try { + OperationHandlerImpl.resolveClusterRoute(Optional.of("wrong"), clusterDef); + } catch(RestApiException e) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + e.getResponse().render(stream); + String errorMsg = new String( stream.toByteArray()); + assertThat(errorMsg, is("{\"errors\":[\"Your vespa cluster contains the content clusters foo2 " + + "(configId2), foo (configId), foo3 (configId2), not wrong. Please select a valid vespa cluster.\"]}")); + return; + } + fail("Expected exception"); + } +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java new file mode 100644 index 00000000000..4bb7a264aba --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class RestUriTest { + + URI createUri(String path, String query) throws URISyntaxException { + return new URIBuilder() + .addParameter("foo", "bar") + .setHost("host") + .setScheme("http") + .setPort(666) + .setPath(path) + .setCustomQuery(query) + .setFragment("fargment").build(); + } + + @Rule + public ExpectedException thrown= ExpectedException.none(); + + @Test + public void testBasic() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid", "query")); + assertThat(restUri.getDocId(), is("myid")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::myid")); + } + + @Test + public void encodingSlashes() throws Exception { + // Try with slashes encoded. + final String id = " !\"øæåp/:;&,.:;'1Q"; + String encodedId = URLEncoder.encode(id, StandardCharsets.UTF_8.name()); + RestUri restUri = new RestUri(URI.create("/document/v1/namespace/doctype/docid/" + encodedId)); + assertThat(restUri.getDocId(), is(id)); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); + } + + @Test + public void encodingSlashes2() throws Exception { + // This will decode the slashes. + final String id = " !\"øæåp/:;&,.:;'1Q "; + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/" + id, "query")); + assertThat(restUri.getDocId(), is(id)); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); + } + + + @Test + public void testVisit() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/", "query")); + assertThat(restUri.getDocId(), is("")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::")); + } + + @Test + public void testOneSlashTooMuchWhichIsFine() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid:342:23/wrong", "")); + assertThat(restUri.getDocId(), is("myid:342:23/wrong")); + } + + @Test + public void testGroupG() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group/myid", "")); + assertThat(restUri.getDocId(), is("myid")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getGroup().get().name, is('g')); + assertThat(restUri.getGroup().get().value, is("group")); + assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group:myid")); + } + + @Test + public void testGroupN() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", "")); + assertThat(restUri.getGroup().get().name, is('n')); + assertThat(restUri.getGroup().get().value, is("group")); + } + + @Test + public void testGroupUnknown() throws Exception { + thrown.expect(RestApiException.class); + new RestUri(createUri("/document/v1/namespace/doctype/Q/myid", "")); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java new file mode 100644 index 00000000000..ab5b36c74d1 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.document.restapi.OperationHandler; +import com.yahoo.document.restapi.Response; +import com.yahoo.document.restapi.RestApiException; +import com.yahoo.document.restapi.RestUri; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.util.Optional; + +/** + * Mock that collects info about operation and returns them on second delete. + */ +public class MockedOperationHandler implements OperationHandler { + + StringBuilder log = new StringBuilder(); + int deleteCount = 0; + + @Override + public VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException { + return new VisitResult(Optional.of("token"), "List of json docs, cont token " + continuation.map(a->a).orElse("not set") + ", doc selection: '" + + documentSelection + "'"); + } + + @Override + public void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + log.append("PUT: " + data.getDocument().getId()); + log.append(data.getDocument().getBody().toString()); + } + + @Override + public void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + log.append("UPDATE: " + data.getDocumentUpdate().getId()); + log.append(data.getDocumentUpdate().getFieldUpdates().toString()); + if (data.getDocumentUpdate().getCreateIfNonExistent()) { + log.append("[CREATE IF NON EXISTENT IS TRUE]"); + } + } + + @Override + public void delete(RestUri restUri, String condition) throws RestApiException { + deleteCount++; + if (deleteCount == 2) { + String theLog = log.toString(); + log = new StringBuilder(); + deleteCount = 0; + throw new RestApiException(Response.createErrorResponse(666, theLog)); + } + log.append("DELETE: " + restUri.generateFullId()); + } + + @Override + public Optional<String> get(RestUri restUri) throws RestApiException { + log.append("GET: " + restUri.generateFullId()); + return Optional.empty(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java new file mode 100644 index 00000000000..17a36c142ea --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.restapi.OperationHandler; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +public class RestApiMaxThreadTest { + final CountDownLatch latch = new CountDownLatch(1); + final AtomicInteger requestsInFlight = new AtomicInteger(0); + private class RestApiMocked extends RestApi { + + public RestApiMocked() { + super(mock(Executor.class), null, (OperationHandler)null); + } + + @Override + protected HttpResponse handleInternal(HttpRequest request) { + requestsInFlight.incrementAndGet(); + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + } + } + + @Test + public void testCallsAreThrottled() throws InterruptedException { + RestApiMocked restApiMocked = new RestApiMocked(); + // Fire lots of requests. + for (int x = 0; x < 30; x++) { + new Thread(() -> restApiMocked.handle(null)).start(); + } + // Wait for all threads to be used + while (requestsInFlight.get() != 19) { + Thread.sleep(1); + } + // A new request should be blocked. + final HttpResponse response = restApiMocked.handle(null); + assertThat(response.getStatus(), is(429)); + latch.countDown(); + } +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java new file mode 100644 index 00000000000..036dc63ad34 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -0,0 +1,298 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import com.yahoo.application.container.handler.Request; +import com.yahoo.container.Container; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNot.not; +import static org.hamcrest.core.StringContains.containsString; +import static org.hamcrest.core.StringStartsWith.startsWith; +import static org.junit.Assert.assertThat; + +public class RestApiTest { + Application application; + + @Before + public void setup() throws Exception { + application = Application.fromApplicationPackage(Paths.get("src/test/application"), Networking.enable); + } + + @After + public void tearDown() throws Exception { + application.close(); + } + + String post_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String post_test_doc = "{\n" + + "\"foo\" : \"bar\"," + + "\"fields\": {\n" + + "\"title\": \"This is the title\",\n" + + "\"body\": \"This is the body\"" + + "}" + + "}"; + String post_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + // Run this test to manually do request against the REST-API with backend mock. + @Ignore + @Test + public void blockingTest() throws Exception { + System.out.println("Running on port " + getFirstListenPort()); + Thread.sleep(Integer.MAX_VALUE); + } + + @Test + public void testbasicPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity(post_test_doc, ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, is(post_test_response)); + } + + String post_test_uri_cond = "/document/v1/namespace/testdocument/docid/c?condition=foo"; + String post_test_doc_cond = "{\n" + + "\"foo\" : \"bar\"," + + "\"fields\": {\n" + + "\"title\": \"This is the title\",\n" + + "\"body\": \"This is the body\"" + + "}" + + "}"; + String post_test_response_cond = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testConditionalPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri_cond); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity(post_test_doc_cond, ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, is(post_test_response_cond)); + } + + String post_test_empty_response = "{\"errors\":[\"Could not read document, no document?\"]"; + @Test + public void testEmptyPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity("", ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, startsWith(post_test_empty_response)); + } + + String update_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String update_test_doc = "{\n" + + "\t\"fields\": {\n" + + "\"title\": {\n" + + "\"assign\": \"Oh lala\"\n" + + "}\n" + + "}\n" + + "}\n"; + + String update_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testbasicUpdate() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_response)); + assertThat(getLog(), not(containsString("CREATE IF NON EXISTING IS TRUE"))); + } + + @Test + public void testbasicUpdateCreateTrue() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri + "?create=true"); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_response)); + assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); + } + + String update_test_create_if_non_existient_uri = "/document/v1/namespace/testdocument/docid/c"; + String update_test_create_if_non_existient_doc = "{\n" + + "\"create\":true," + + "\t\"fields\": {\n" + + "\"title\": {\n" + + "\"assign\": \"Oh lala\"\n" + + "}\n" + + "}\n" + + "}\n"; + + String update_test_create_if_non_existing_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testCreateIfNonExistingUpdateInDocTrue() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response)); + assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); + + } + + @Test + public void testCreateIfNonExistingUpdateInDocTrueButQueryParamsFalse() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri + "?create=false"); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response)); + assertThat(getLog(), not(containsString("CREATE IF NON EXISTENT IS TRUE"))); + + } + + // Get logs through some hackish fetch method. Logs is something the mocked backend write. + String getLog() throws IOException { + // The mocked backend will throw a runtime exception wtih a log if delete is called three times.. + Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); + HttpDelete delete = new HttpDelete(request.getUri()); + doRest(delete); + return doRest(delete); + } + + + String remove_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String remove_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testbasicRemove() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); + HttpDelete delete = new HttpDelete(request.getUri()); + assertThat(doRest(delete), is(remove_test_response)); + } + + String get_test_uri = "/document/v1/namespace/document-type/docid/c"; + String get_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/c\""; + String get_response_part2 = "\"id\":\"id:namespace:document-type::c\""; + + + @Test + public void testbasicGet() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_response_part1)); + assertThat(rest, containsString(get_response_part2)); + } + + String id_test_uri = "/document/v1/namespace/document-type/docid/f/u/n/n/y/!"; + String id_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/f/u/n/n/y/!\""; + String id_response_part2 = "\"id\":\"id:namespace:document-type::f/u/n/n/y/!\""; + + @Test + public void testSlashesInId() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + id_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(id_response_part1)); + assertThat(rest, containsString(id_response_part2)); + } + + + String get_enc_id = "!\":æøå@/& Q1+"; + // Space encoded as %20, not encoding ! + String get_enc_id_encoded_v1 = "!%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26%20Q1%2B"; + // Space encoded as + + String get_enc_id_encoded_v2 = "%21%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26+Q1%2B"; + String get_enc_test_uri_v1 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1; + String get_enc_test_uri_v2 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2; + String get_enc_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1 + "\""; + String get_enc_response_part1_v2 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2 + "\""; + + // JSON encode " as \" + String get_enc_response_part2 = "\"id\":\"id:namespace:document-type::" + get_enc_id.replace("\"", "\\\"") + "\""; + + + @Test + public void testbasicEncodingV1() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v1); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_enc_response_part1)); + assertThat(rest, containsString(get_enc_response_part2)); + } + + @Test + public void testbasicEncodingV2() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v2); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_enc_response_part1_v2)); + assertThat(rest, containsString(get_enc_response_part2)); + } + + String visit_test_uri = "/document/v1/namespace/document-type/docid/?continuation=abc"; + String visit_response_part1 = "\"documents\":[List of json docs, cont token abc, doc selection: '']"; + String visit_response_part2 = "\"continuation\":\"token\""; + String visit_response_part3 = "\"pathId\":\"/document/v1/namespace/document-type/docid/\""; + + + @Test + public void testbasicVisit() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(visit_response_part1)); + assertThat(rest, containsString(visit_response_part2)); + assertThat(rest, containsString(visit_response_part3)); + } + + String visit_test_bad_uri = "/document/v1/namespace/document-type/group/abc?continuation=abc"; + String visit_test_bad_response = "Visiting does not support setting value for group/value,"; + + + @Test + public void testBadVisit() throws Exception { + final Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_bad_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(visit_test_bad_response)); + } + + private String doRest(HttpRequestBase request) throws IOException { + HttpClient client = HttpClientBuilder.create().build(); + HttpResponse response = client.execute(request); + HttpEntity entity = response.getEntity(); + return EntityUtils.toString(entity); + } + + private String getFirstListenPort() { + JettyHttpServer serverProvider = + (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0); + return Integer.toString(serverProvider.getListenPort()); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java new file mode 100644 index 00000000000..cfb120d9891 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.restapi.OperationHandler; + +import java.util.concurrent.Executor; + +/** + * For setting up RestApi with a simple document type manager. + * + * @author dybdahl + */ +public class RestApiWithTestDocumentHandler extends RestApi{ + + private DocumentTypeManager docTypeManager = new DocumentTypeManager(); + + public RestApiWithTestDocumentHandler( + Executor executor, + AccessLog accessLog, + OperationHandler operationHandler) { + super(executor, accessLog, operationHandler); + + DocumentType documentType = new DocumentType("testdocument"); + + documentType.addField("title", DataType.STRING); + documentType.addField("body", DataType.STRING); + docTypeManager.registerDocumentType(documentType); + + setDocTypeManagerForTests(docTypeManager); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java new file mode 100644 index 00000000000..6d0ee59c68d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Metric; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespa.http.server.FeedHandler; +import com.yahoo.vespa.http.server.Feeder; +import org.junit.Test; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit test for FeedHandler class. + * + * @author dybdahl + */ +public class FeedHandlerTest { + + /** + * This class extends FeedHandler and allows to create a custom Feeder. + */ + static class TestFeedHandler extends FeedHandler { + private final CountDownLatch countDownLatch = new CountDownLatch(1); + + public TestFeedHandler() throws Exception { + super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null); + } + + /** + * Builds a feeder that blocks until countDownLatch is stepped down. + */ + @Override + protected Feeder createFeeder( + com.yahoo.container.jdisc.HttpRequest request, + InputStream requestInputStream, + final BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, + int protocolVersion) throws Exception { + Feeder feeder = mock(Feeder.class); + doAnswer(invocation -> { + try { + countDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + }).when(feeder).waitForRequestReceived(); + return feeder; + } + } + + /** + * nginx require that a post is finished before the server ack with a response. This behaviour is verified + * in this test + */ + @Test + public void testResponseIsSentAfterWaitForRequestReceivedReturns() throws Exception { + HttpRequest request = mock(HttpRequest.class); + + // Create a request with valid version. + com.yahoo.jdisc.http.HttpRequest jdiscRequest = mock(com.yahoo.jdisc.http.HttpRequest.class); + HeaderFields headerFields = mock(HeaderFields.class); + List<String> version = new ArrayList<>(); + version.add("2"); + when(headerFields.get(Headers.VERSION)).thenReturn(version); + when(jdiscRequest.headers()).thenReturn(headerFields); + when(request.getJDiscRequest()).thenReturn(jdiscRequest); + + TestFeedHandler feedHandler = new TestFeedHandler(); + // After a short period, make the feed finish. + new Thread(() -> { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + feedHandler.countDownLatch.countDown(); + }).start(); + // This should not return before countdown latch is stepped down. + feedHandler.handle(request); + // This should always returns after the countDownLatch has become zero. This might cause false positive, + // but not false negatives. This is fine. + assertThat(feedHandler.countDownLatch.getCount(), is(0L)); + + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java new file mode 100755 index 00000000000..646bcb805f6 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java @@ -0,0 +1,1015 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.CallStack; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.messagebus.*; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.container.Container; +import com.yahoo.docproc.*; +import com.yahoo.docproc.jdisc.DocumentProcessingHandler; +import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; +import org.junit.After; +import org.junit.Test; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class VespaFeedHandlerTestCase { + + private VespaFeedHandler feedHandler; + private VespaFeedHandlerRemove removeHandler; + private VespaFeedHandlerStatus statusHandler; + private VespaFeedHandlerRemoveLocation removeLocationHandler; + private FeedContext context; + + private DummySessionFactory factory; + private final String xmlFilesPath = "src/test/files/feedhandler/"; + + public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg, + boolean autoReply, + DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { + DocumentTypeManager docMan = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg"); + + if (autoReply) { + if (autoReplyFactory != null) { + factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory); + } else { + factory = DummySessionFactory.createWithErrorAutoReply(e); + } + } else { + factory = DummySessionFactory.createDefault(); + } + + context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric()); + + Executor threadPool = Executors.newCachedThreadPool(); + feedHandler = new VespaFeedHandler(context, threadPool); + removeHandler = new VespaFeedHandlerRemove(context, threadPool); + statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool); + removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool); + + CallStack dpCallstack = new CallStack("bar"); + dpCallstack.addLast(new TestDocProc()); + dpCallstack.addLast(new TestLaterDocProc()); + + DocprocService myservice = new DocprocService("bar"); + myservice.setCallStack(dpCallstack); + myservice.setInService(true); + + ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>(); + registry.register(new ComponentId(myservice.getName()), myservice); + + DocumentProcessingHandler handler = new DocumentProcessingHandler(registry, + new ComponentRegistry<>(), new ComponentRegistry<>(), + new DocumentProcessingHandlerParameters()); + + Container container = Container.get(); + ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>(); + requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler); + container.setRequestHandlerRegistry(requestHandlerComponentRegistry); + } + + public void setup(com.yahoo.messagebus.Error e) throws Exception { + setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null); + } + + public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory); + } + + public void setup() throws Exception { + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null); + } + + @After + public void resetContainer() { + Container.resetInstance(); + } + + + @Test + public void testLoadTypes() throws Exception { + List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>(); + typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW")); + typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3")); + + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null); + + { + Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType()); + assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority()); + } + } + + @Test + public void testPostXML() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + assertTrue(res.output.contains("count=\"2\"")); + assertTrue(res.error == null); + } + + @Test + public void testPostXMLAsync() throws Exception { + setup(); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + // Should not have metrics at this point. + assertTrue(!res.output.contains("count=\"2\"")); + assertTrue(res.error == null); + } + + + @Test + public void testPostGZIPedXML() throws Exception { + setup(null); + Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + assertTrue(res.error == null); + } + + @Test + public void testDocProc() throws Exception { + setup(null); + + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); + + assertEquals("doc:news:http://news10a", d.getId().toString()); + assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); + + assertEquals("doc:news:http://news10b", d.getId().toString()); + assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); + } + } + + @Test + public void testPostXMLVariousTypes() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10.xml", "feed?"); + + assertEquals(5, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + { + Message m = res.messages.get(2); + assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); + DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); + assertEquals("doc:news:http://news10c", d.toString()); + } + { + Message m = res.messages.get(3); + assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); + DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); + assertEquals("doc:news:http://news10d", d.toString()); + } + { + Message m = res.messages.get(4); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:news:http://news10e", d.toString()); + } + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + "\n" + + "</result>\n", val); + } + + @Test + public void testStatusPage() throws Exception { + setup(null); + + testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + testFeed(xmlFilesPath + "test10.xml", "feed?"); + testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); + testFeed(xmlFilesPath + "test_removes", "remove?"); + + assertEquals(2, factory.sessionsCreated()); + Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); + + assertEquals("<status>\n" + + "\n" + + " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" + + " <routes>\n" + + " <route name=\"total\">\n" + + " <total>\n" + + " <latency count=\"14\"/>\n" + + " <count count=\"14\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"6\"/>\n" + + " <count count=\"6\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <latency count=\"9\"/>\n" + + " <count count=\"9\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"3\"/>\n" + + " <count count=\"3\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"storage\">\n" + + " <total>\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " </routes>\n" + + " </snapshot>\n" + + "\n" + + "</status>\n", val); + } + + @Test + public void testStatusPage2() throws Exception { + setup(null); + + testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + testFeed(xmlFilesPath + "test10.xml", "feed?"); + testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); + testFeed(xmlFilesPath + "test_removes", "remove?"); + + assertEquals(2, factory.sessionsCreated()); + Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); + + assertEquals("<status>\n" + + "\n" + + " <routes>\n" + + " <route name=\"total\" description=\"Messages sent to all routes\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"14\"/>\n" + + " <count count=\"14\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"6\"/>\n" + + " <count count=\"6\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"default\" description=\"Messages sent to the named route\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"9\"/>\n" + + " <count count=\"9\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"3\"/>\n" + + " <count count=\"3\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"storage\" description=\"Messages sent to the named route\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " </routes>\n" + + "\n" + + "</status>\n", val); + } + + @Test + public void testMetricForIgnoredDocumentsIsIncreased() throws Exception { + DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() { + @Override + public Reply createReply(Message m) { + return new DocumentIgnoredReply(); + } + }; + setupWithReplyFactory(replyFactory); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + assertEquals(2, res.messages.size()); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <ignored count=\"2\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <ignored count=\"2\"/>\n" + + " </putdocument>\n" + + " </route>\n" + + "\n" + + "</result>\n", val); + } + + @Test + public void testPostXMLWithMBusFailureAllowed() throws Exception { + setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>")); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); + System.out.println(val); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"2\"/>\n" + + " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + + " </errors>\n" + + " </total>\n" + + " <putdocument>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"2\"/>\n" + + " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + + " </errors>\n" + + " </putdocument>\n" + + " </route>\n\n" + + " <errors count=\"2\">\n" + + " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in <document>\"/>\n" + + " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in <document>\"/>\n" + + " </errors>\n" + + "\n" + + "</result>\n", val); + + assertTrue(res.error != null); + assertTrue(res.errorCount > 0); + } + + @Test + public void testPostXMLWithMBusFailure() throws Exception { + setup(new com.yahoo.messagebus.Error(32, "Hello world")); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + + String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"1\"/>\n" + + " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + + " </errors>\n" + + " </total>\n" + + " <putdocument>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"1\"/>\n" + + " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + + " </errors>\n" + + " </putdocument>\n" + + " </route>\n\n" + + " <errors count=\"1\">\n" + + " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" + + " </errors>\n" + + "\n" + + "</result>\n", val); + + assertTrue(res.error != null); + assertTrue(res.errorCount > 0); + } + + @Test + public void testPostXMLWithIllegalDocId() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:ok", d.toString()); + } + } + + @Test + public void testPostUnparseableXML() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testOverrides() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2"); + + assertEquals(2, res.messages.size()); + + for (Message m : res.messages) { + assertEquals(2222, m.getTimeRemaining()); + assertEquals(Route.parse("storage"), m.getRoute()); + assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority()); + } + } + + @Test + public void testBogusPriority() throws Exception { + try { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS"); + assertTrue(false); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testPostXMLWithIllegalDocIdFirst() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?"); + + assertEquals(0, res.messages.size()); + } + + @Test + public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testSimpleRemove() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + } + + @Test + public void testRemoveUser() throws Exception { + setup(null); + + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("storage", m.getRoute().toString()); + assertEquals("id.user=1234", selection); + } + } + + @Test + public void testRemoveGroup() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("storage", m.getRoute().toString()); + assertEquals("id.group=\"foo\"", selection); + } + } + + @Test + public void testRemoveBadSyntax() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("Exactly one of")); + } + + @Test + public void testRemoveGroupMultipleClusters() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1")); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("More than one")); + } + + @Test + public void testRemoveGroupNoClusters() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("No storage clusters")); + } + + @Test + public void testRemoveSelection() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("id.user=1234", selection); + } + } + + @Test + public void testSimpleRemoveIndex() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + } + + @Test + public void testPostRemove() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_removes", "remove?"); + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove1", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove2", d.toString()); + } + } + + @Test + public void testRemoveBogusId() throws Exception { + try { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertTrue(false); + } catch (Exception e) { + } + } + + @Test + public void testMultiRemove() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(3, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove2", d.toString()); + } + + { + Message m = res.messages.get(2); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove3", d.toString()); + } + } + + @Test + public void testMultiRemoveSameDoc() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + } + } + + @Test + public void testFeedHandlerStatusCreation() throws Exception { + VespaFeedHandlerStatus status = new VespaFeedHandlerStatus( + new FeedContext(new MessagePropertyProcessor( + new FeederConfig(new FeederConfig.Builder()), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, null, new ClusterList(), new NullFeedMetric()), + true, true, + Executors.newCachedThreadPool()); + } + + private class TestDocProc extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document document = ((DocumentPut)op).getDocument(); + document.setFieldValue("last_downloaded", new IntegerFieldValue(1234)); + } + } + return Progress.DONE; + } + } + + private class TestLaterDocProc extends DocumentProcessor { + private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName()); + + private int counter = 0; + @Override + public Progress process(Processing processing) { + synchronized (this) { + counter++; + if (counter % 2 == 1) { + log.info("Returning LATER."); + return Progress.LATER; + } + log.info("Returning DONE."); + return Progress.DONE; + } + } + } + + private Result testRequest(HttpRequest req) throws Exception { + HttpResponse response = null; + String feedPrefix = "feed"; + String removePrefix = "remove"; + String feedStatusPrefix = "feedstatus"; + String removeLocationPrefix = "removelocation"; + + if (req.getUri().getPath().startsWith(feedPrefix)) { + response = feedHandler.handle(req); + } + if (req.getUri().getPath().startsWith(removePrefix)) { + response = removeHandler.handle(req); + } + if (req.getUri().getPath().startsWith(feedStatusPrefix)) { + response = statusHandler.handle(req); + } + if (req.getUri().getPath().startsWith(removeLocationPrefix)) { + response = removeLocationHandler.handle(req); + } + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + response.render(output); + + Result res = new Result(); + res.messages = factory.messages; + res.output = new String(output.toByteArray()); + + if (response instanceof FeedResponse) { + FeedResponse feedResponse = (FeedResponse)response; + res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0); + res.errorCount = feedResponse.getErrorMessageList().size(); + assertTrue(feedResponse.isSuccess() == (res.errorCount == 0)); + } + return res; + } + + private Result testFeed(String xmlFile, String request) throws Exception { + return testRequest(new FileRequest(new File(xmlFile), request).toRequest()); + } + + private Result testFeedGZIP(String xmlFile, String request) throws Exception { + return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest()); + } + + private class FileRequest { + + private final String req; + private final File f; + private boolean gzip = false; + + FileRequest(File f, String req) { + this.req = req; + this.f = f; + } + + FileRequest(File f, String req, boolean gzip) { + this.f = f; + this.req = req; + this.gzip = gzip; + } + + public InputStream getData() { + try { + InputStream fileStream = new FileInputStream(f); + if (gzip) { + // Not exactly pretty, but in lack of an elegant way of transcoding + ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); + GZIPOutputStream compressed = new GZIPOutputStream(rawOut); + byte[] buffer = new byte[1024]; + int read = -1; + while (true) { + read = fileStream.read(buffer); + if (read == -1) break; + compressed.write(buffer, 0, read); + } + compressed.finish(); + compressed.flush(); + rawOut.flush(); + return new ByteArrayInputStream(rawOut.toByteArray()); + } + return fileStream; + } catch (Exception e) { + return null; + } + } + + public void addHeaders(HeaderFields headers) { + headers.add("Content-Type", "image/jpeg"); + if (gzip) + headers.add("Content-Encoding", "gzip"); + } + + public HttpRequest toRequest() { + HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); + addHeaders(request.getJDiscRequest().headers()); + return request; + } + + } + + private class Result { + private List<Message> messages; + private String output; + private com.yahoo.processing.request.ErrorMessage error; + private int errorCount; + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java new file mode 100644 index 00000000000..708a9ed7a6b --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler.v3; + +import com.google.common.base.Splitter; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespa.http.server.ReplyContext; +import com.yahoo.vespa.http.server.FeedHandlerV3; +import org.junit.Test; +import com.yahoo.container.jdisc.HttpRequest; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.yahoo.messagebus.Result; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +public class FeedTesterV3 { + + @Test + public void feedOneDocument() throws Exception { + final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(); + HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1)); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + httpResponse.render(outStream); + assertThat(httpResponse.getContentType(), is("text/plain")); + assertThat(Utf8.toString(outStream.toByteArray()), is("1230 OK message trace\n")); + + } + + @Test + public void feedManyDocument() throws Exception { + final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(); + HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100)); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + httpResponse.render(outStream); + assertThat(httpResponse.getContentType(), is("text/plain")); + String result = Utf8.toString(outStream.toByteArray()); + assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); + } + + DocumentTypeManager createDoctypeManager() { + DocumentTypeManager docTypeManager = new DocumentTypeManager(); + DocumentType documentType = new DocumentType("testdocument"); + documentType.addField("title", DataType.STRING); + documentType.addField("body", DataType.STRING); + docTypeManager.registerDocumentType(documentType); + return docTypeManager; + } + + HttpRequest createRequest(int numberOfDocs) { + String clientId = "client123"; + StringBuilder wireData = new StringBuilder(); + for (int x = 0; x < numberOfDocs; x++) { + String docData = "[{\"put\": \"id:testdocument:testdocument::c\", \"fields\": { \"title\": \"fooKey\", \"body\": \"value\"}}]"; + String operationId = "123" + x; + wireData.append(operationId + " " + Integer.toHexString(docData.length()) + "\n" + docData); + } + InputStream inputStream = new ByteArrayInputStream(wireData.toString().getBytes()); + HttpRequest request = HttpRequest.createTestRequest( + "http://dummyhostname:19020/reserved-for-internal-use/feedapi", + com.yahoo.jdisc.http.HttpRequest.Method.POST, + inputStream); + request.getJDiscRequest().headers().add(Headers.VERSION, "3"); + request.getJDiscRequest().headers().add(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name()); + request.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + request.getJDiscRequest().headers().add(Headers.CLIENT_ID, clientId); + request.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + request.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + request.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + return request; + } + + FeedHandlerV3 setupFeederHandler() throws Exception { + Executor threadPool = Executors.newCachedThreadPool(); + DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)); + FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( + threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null) { + @Override + protected ReferencedResource<SharedSourceSession> retainSource( + SessionCache sessionCache, SourceSessionParams sessionParams) { + SharedSourceSession sharedSourceSession = mock(SharedSourceSession.class); + + try { + Mockito.stub(sharedSourceSession.sendMessageBlocking(anyObject())).toAnswer((Answer) invocation -> { + Object[] args = invocation.getArguments(); + PutDocumentMessage putDocumentMessage = (PutDocumentMessage) args[0]; + ReplyContext replyContext = (ReplyContext)putDocumentMessage.getContext(); + replyContext.feedReplies.add(new OperationStatus("message", replyContext.docId, ErrorCode.OK, "trace")); + Result result = mock(Result.class); + when(result.isAccepted()).thenReturn(true); + return result; + }); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + Result result = mock(Result.class); + when(result.isAccepted()).thenReturn(true); + ReferencedResource<SharedSourceSession> refSharedSessopn = + new ReferencedResource<>(sharedSourceSession, () -> {}); + return refSharedSessopn; + } + }; + feedHandlerV3.injectDocumentManangerForTests(createDoctypeManager()); + return feedHandlerV3; + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java new file mode 100644 index 00000000000..8b991909cd8 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorIterator; +import org.junit.Test; + +import java.util.Set; +import java.util.TreeSet; + +import static org.junit.Assert.*; + +public class ContinuationHitTest { + + private static final String SINGLE_BUCKET_URL_SAFE_BASE64 + = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA"; + private static final String MULTI_BUCKET_URL_SAFE_BASE64 + = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" + + "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA="; + + @Test + public void continuationTokensAreUrlSafeBase64Encoded() throws Exception { + ContinuationHit hit = new ContinuationHit(createSingleBucketProgress()); + // We want -_ instead of +/ + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue()); + } + + @Test + public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception { + ContinuationHit hit = new ContinuationHit(createMultiBucketProgress()); + assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits. + assertFalse(hit.getValue().contains("\n")); + } + + @Test + public void decodingAcceptsUrlSafeTokens() throws Exception { + final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64); + // Roundtrip should yield identical results. + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(token).getValue()); + } + + /** + * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era. + */ + @Test + public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception { + final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64); + final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64); + + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(legacyToken).getValue()); + } + + /** + * Legacy Base64 encoder would happily output line breaks after each MIME line + * boundary. Ensure we handle these gracefully. + */ + @Test + public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception { + final String multiBucketLegacyToken = + "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + + "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA="; + final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken); + + assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(legacyToken).getValue()); + } + + /** + * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit) + */ + private ProgressToken createSingleBucketProgress() { + ProgressToken token = new ProgressToken(16); + // Use explicit bucket set so we can better control the binary representation + // of the buckets, and thus the values written as base 64. + Set<BucketId> buckets = new TreeSet<>(); + // This particular bucket ID will contain +/ chars when output as non-URL safe base 64. + buckets.add(new BucketId(58, 0x123456789abcfeffL)); + VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. + return token; + } + + /** + * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit) + */ + private ProgressToken createMultiBucketProgress() { + ProgressToken token = new ProgressToken(16); + Set<BucketId> buckets = new TreeSet<>(); + buckets.add(new BucketId(58, 0x123456789abcfeffL)); + buckets.add(new BucketId(58, 0x123456789abcfefaL)); + buckets.add(new BucketId(58, 0x123456789abcfefbL)); + VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. + return token; + } + + private String convertedToMimeBase64Chars(String token) { + // Doesn't split on MIME line boundaries, so not fully MIME compliant. + return token.replace('-', '+').replace('_', '/'); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java new file mode 100755 index 00000000000..ea88128aaa9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.SendSession; +import com.yahoo.jdisc.Metric; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +/** + * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage + */ +public class DocumentSessionFactory extends DummySessionFactory { + + private DocumentType docType; + private Error error; + // Reply instances are shared between the two collections. + List<GetDocumentReply> autoReplies; + Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>(); + boolean autoReply = true; + boolean nullReply = false; + private int sessionsCreated = 0; + + private class DocumentReplySession extends SendSession { + + ReplyHandler handler; + Error e; + DummySessionFactory owner; + + public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) { + this.handler = handler; + this.e = e; + this.owner = owner; + } + + protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { + if (!(m instanceof GetDocumentMessage)) { + throw new IllegalArgumentException("Expected GetDocumentMessage"); + } + GetDocumentMessage gm = (GetDocumentMessage)m; + owner.messages.add(m); + if (autoReply) { + Document replyDoc; + if (!nullReply) { + replyDoc = new Document(docType, gm.getDocumentId()); + } else { + replyDoc = null; + } + Reply r = new GetDocumentReply(replyDoc); + r.setMessage(m); + r.setContext(m.getContext()); + if (e != null) { + r.addError(e); + } + handler.handleReply(r); + } else if (owner.messages.size() == autoReplies.size()) { + // Pair up all replies with their messages + for (Message msg : owner.messages) { + GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId()); + reply.setMessage(msg); + reply.setContext(msg.getContext()); + if (e != null) { + reply.addError(e); + } + } + // Now send them in the correct order. Instances are shared, so source + // messages and contexts are properly set + for (Reply reply : autoReplies) { + handler.handleReply(reply); + } + } + + return Result.ACCEPTED; + } + + public void close() { + } + } + + public DocumentSessionFactory(DocumentType docType) { + this.docType = docType; + this.error = null; + } + + public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) { + this.docType = docType; + this.error = error; + this.autoReplies = Arrays.asList(autoReplies); + for (GetDocumentReply reply : autoReplies) { + this.autoReplyLookup.put(reply.getDocument().getId(), reply); + } + this.autoReply = autoReply; + } + + public boolean isNullReply() { + return nullReply; + } + + public void setNullReply(boolean nullReply) { + this.nullReply = nullReply; + } + + public int getSessionsCreated() { + return sessionsCreated; + } + + public SendSession createSendSession(ReplyHandler r, Metric metric) { + ++sessionsCreated; + return new DocumentReplySession(r, error, this); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters p) { + return new DummyVisitorSession(p, docType); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java new file mode 100644 index 00000000000..d1c55c0968f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Trace; + +import java.util.ArrayList; +import java.util.List; + +/** + * Stub to test visitors. + */ +public class DummyVisitorSession implements VisitorSession { + + final VisitorParameters parameters; + final DocumentType documentType; + final List<Message> autoReplyMessages = new ArrayList<>(); + + DummyVisitorSession(VisitorParameters p, DocumentType documentType) { + parameters = p; + this.documentType = documentType; + p.getLocalDataHandler().setSession(this); + addDefaultReplyMessages(); + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public ProgressToken getProgress() { + return new ProgressToken(12); + } + + @Override + public Trace getTrace() { + return null; + } + + public void addDocumentReply(String docId) { + Document replyDoc = new Document(documentType, docId); + autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc))); + } + + public void addRemoveReply(String docId) { + autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId))); + } + + public void addDefaultReplyMessages() { + addDocumentReply("userdoc:foo:1234:bar"); + if (parameters.visitRemoves()) { + addRemoveReply("userdoc:foo:1234:removed"); + } + } + + public void clearAutoReplyMessages() { + autoReplyMessages.clear(); + } + + @Override + public boolean waitUntilDone(long l) throws InterruptedException { + for (Message msg : autoReplyMessages) { + parameters.getLocalDataHandler().onMessage(msg, new AckToken(this)); + } + return true; + } + + @Override + public void ack(AckToken ackToken) { + } + + @Override + public void abort() { + } + + @Override + public VisitorResponse getNext() { + return null; + } + + @Override + public VisitorResponse getNext(int i) throws InterruptedException { + return null; + } + + @Override + public void destroy() { + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java new file mode 100755 index 00000000000..1e39b9766b1 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java @@ -0,0 +1,1090 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.component.chain.Chain; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.java7compat.Util; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; + +import org.junit.Test; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.*; + +public class GetSearcherTestCase { + + private DocumentTypeManager docMan = null; + private DocumentType docType; + private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder()); + private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder()); + + @org.junit.Before + public void setUp() { + docMan = new DocumentTypeManager(); + docType = new DocumentType("kittens"); + docType.addHeaderField("name", DataType.STRING); + docType.addField("description", DataType.STRING); + docType.addField("image", DataType.STRING); + docType.addField("fluffiness", DataType.INT); + docType.addField("foo", DataType.RAW); + docMan.registerDocumentType(docType); + } + + @org.junit.After + public void tearDown() { + docMan = null; + docType = null; + } + + private void assertHits(HitGroup hits, String... wantedHits) { + assertEquals(wantedHits.length, hits.size()); + for (int i = 0; i < wantedHits.length; ++i) { + assertTrue(hits.get(i) instanceof DocumentHit); + DocumentHit hit = (DocumentHit)hits.get(i); + assertEquals(wantedHits[i], hit.getDocument().getId().toString()); + } + } + + @Test + public void testGetSingleDocumentQuery() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2")); + System.out.println("HTTP request is " + result.getQuery().getHttpRequest()); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1:2"); + // By default, document hit should not have its hit fields set + DocumentHit hit = (DocumentHit)result.hits().get(0); + assertEquals(0, hit.fieldKeys().size()); + } + + @Test + public void testGetMultipleDocumentsQuery() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + assertEquals(2, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4"); + assertEquals(2, query.getHits()); + } + + // Test that you can use both query string and POSTed IDs + @Test + public void testGetMultipleDocumentsQueryAndPOST() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; + MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); + Query query = new Query(request.toRequest()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(5, factory.messages.size()); + assertEquals(5, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", + "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); + } + + @Test + public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; + + // Create with automatic GZIP encoding + MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true); + Query query = new Query(request.toRequest()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(5, factory.messages.size()); + assertEquals(5, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", + "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); + } + + /* Test that a query without any ids is passed through to the next chain */ + @Test + public void testQueryPassThrough() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + HitGroup hits = new HitGroup("mock"); + hits.add(new Hit("blernsball")); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern")); + + assertEquals(0, factory.messages.size()); + assertEquals(1, result.hits().size()); + assertNotNull(result.hits().get("blernsball")); + } + + /* Test that a query will contain both document hits and hits from a searcher + * further down the chain, iff the searcher returns a DocumentHit. + */ + @Test + public void testQueryPassThroughAndGet() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); + doc1.setFieldValue("name", new StringFieldValue("megacat")); + doc1.setFieldValue("description", new StringFieldValue("supercat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo")); + + assertEquals(1, factory.messages.size()); + assertEquals(2, result.hits().size()); + assertNotNull(result.hits().get("userdoc:kittens:5678:bar")); + assertNotNull(result.hits().get("userdoc:kittens:1234:foo")); + } + + @Test + public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + HitGroup hits = new HitGroup("mock"); + hits.add(new Hit("blernsball")); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa")); + + assertEquals(0, factory.messages.size()); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + + "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " + + "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + + "Only DocumentHit instances are supported in the backend hit result set " + + "when doing queries that contain document identifier sets recognised by the " + + "Get Searcher.\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testConfig() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("route66").retryenabled(false)), defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:batman:dahnahnahnah", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("route66"), gdm.getRoute()); + assertFalse(gdm.getRetryEnabled()); + assertEquals(458000, gdm.getTimeRemaining()); + } + } + + @Test + public void testConfigChanges() throws Exception { + String config = "raw:timeout 458\nroute \"riksveg18\"\nretryenabled true"; + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), + defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); + + assertEquals(1, factory.messages.size()); + assertEquals(1, factory.getSessionsCreated()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:batman:dahnahnahnah", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("riksveg18"), gdm.getRoute()); + assertTrue(gdm.getRetryEnabled()); + assertEquals(458000, gdm.getTimeRemaining()); + } + + factory.messages.clear(); + + FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder() + .timeout(123) + .route("e6") + .retryenabled(false) + ); + searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg); + + new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=doc:spiderman:does_whatever_a_spider_can")); + + // riksveg18 is created again, and e6 is created as well. + assertEquals(3, factory.getSessionsCreated()); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("e6"), gdm.getRoute()); + assertFalse(gdm.getRetryEnabled()); + assertEquals(123000, gdm.getTimeRemaining()); + } + } + + @Test + public void testQueryOverridesDefaults() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=458")); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(458000, gdm.getTimeRemaining()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(458000, gdm.getTimeRemaining()); + } + } + + @Test + public void testQueryOverridesConfig() throws Exception { + String config = "raw:timeout 458\nroute \"route66\""; + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123")); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(123000, gdm.getTimeRemaining()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(123000, gdm.getTimeRemaining()); + } + } + + @Test + public void testBadPriorityValue() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: No enum const" + + (Util.isJava7Compatible() ? "ant " : " class ") + + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" + + (Util.isJava7Compatible() ? "." : "$") + + "Priority.onkel_jubalon\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testMultiIdBadArrayIndex() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[1]=userdoc:kittens:1:2")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&id[1]=userdoc:kittens:2:3")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0=userdoc:kittens:1:2")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + } + + @Test + public void testLegacyHeadersOnly() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&headersonly=true")); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[header]", gdm.getFieldSet()); + } + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1:2"); + } + + @Test + public void testFieldSet() throws Exception { + } + + @Test + public void testConsistentResultOrdering() throws Exception { + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))), + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))), + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123"))) + }; + + // Use a predefined reply list to ensure messages are answered out of order + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8")); + + assertEquals(3, factory.messages.size()); + assertEquals(3, result.hits().size()); + // Hits must be in the same order as their document IDs in the query + assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8"); + + assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex()); + assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex()); + assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex()); + } + + @Test + public void testResultWithSingleError() throws Exception { + com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly"); + DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testResultWithMultipleErrors() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111")); + GetDocumentReply errorReply1 = new GetDocumentReply(doc1); + errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); + GetDocumentReply errorReply2 = new GetDocumentReply(doc2); + errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains.")); + GetDocumentReply[] replies = new GetDocumentReply[] { + errorReply1, + errorReply2 + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111")); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + + "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testResultWithNullDocument() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); + factory.setNullReply(true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:55:bad_document_id")); + // Document not found does not produce any hit at all, error or otherwise + assertNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "</result>\n", result); + } + + @Test + public void testDefaultDocumentHitRendering() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); + doc1.setFieldValue("name", new StringFieldValue("mittens")); + doc1.setFieldValue("description", new StringFieldValue("it's a cat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(8)); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); + doc2.setFieldValue("name", new StringFieldValue("garfield")); + doc2.setFieldValue("description", + new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised")); + doc2.setFieldValue("fluffiness", new IntegerFieldValue(2)); + Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + GetDocumentReply errorReply = new GetDocumentReply(doc3); + errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + new GetDocumentReply(doc2), + errorReply + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + + "</errors>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + + " <name>garfield</name>\n" + + " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + + " <fluffiness>2</fluffiness>\n" + + "</document>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + + " <name>mittens</name>\n" + + " <description>it's a cat</description>\n" + + " <fluffiness>8</fluffiness>\n" + + "</document>\n" + + "</result>\n", xmlResult); + } + + @Test + public void testDocumentFieldNoContentType() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "derrick"); + doc1.setFieldValue("description", "kommisar katze"); + doc1.setFieldValue("fluffiness", 0); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); + assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>kommisar katze</result>\n", result); + } + + @Test + public void testDocumentFieldEscapeXML() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asfd"); + doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>"); + doc1.setFieldValue("fluffiness", 0); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); + assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result><script type=\"evil/madness\">horror & screams</script></result>\n", result); + } + + @Test + public void testDocumentFieldRawContent() throws Exception { + byte[] contentBytes = new byte[] { 0, -128, 127 }; + + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); + doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:123:456&field=foo")); + + assertNull(result.hits().getErrorHit()); + assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + + byte[] resultBytes = stream.toByteArray(); + assertEquals(contentBytes.length, resultBytes.length); + for (int i = 0; i < resultBytes.length; ++i) { + assertEquals(contentBytes[i], resultBytes[i]); + } + } + + @Test + public void testDocumentFieldRawWithContentOverride() throws Exception { + byte[] contentBytes = new byte[] { 0, -128, 127 }; + + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); + doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + + byte[] resultBytes = stream.toByteArray(); + assertEquals(contentBytes.length, resultBytes.length); + for (int i = 0; i < resultBytes.length; ++i) { + assertEquals(contentBytes[i], resultBytes[i]); + } + } + + @Test + public void testDocumentFieldWithMultipleIDs() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentFieldNotSet() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asdf"); + doc1.setFieldValue("description", "fdsafsdf"); + doc1.setFieldValue("fluffiness", 10); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=image")); + + assertNotNull(result.hits().getErrorHit()); + assertEquals(1, result.hits().size()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + + "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + + @Test + public void testDocumentFieldWithDocumentNotFound() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); + factory.setNullReply(true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&field=name")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + + "Document not found, could not return field 'name'\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asdf"); + // don't set body fields + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true")); + + assertNotNull(result.hits().getErrorHit()); + assertEquals(1, result.hits().size()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " + + "Field 'description' is located in document body, but headersonly " + + "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testVespaXMLTemplate() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); + doc1.setFieldValue("name", "mittens"); + doc1.setFieldValue("description", "it's a cat"); + doc1.setFieldValue("fluffiness", 8); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); + doc2.setFieldValue("name", "garfield"); + doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised"); + doc2.setFieldValue("fluffiness", 2); + Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + GetDocumentReply errorReply = new GetDocumentReply(doc3); + errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + new GetDocumentReply(doc2), + errorReply + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO! + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a <ni!>"shrubbery"</ni!>\"/>\n"+ + "</errors>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + + " <name>garfield</name>\n" + + " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + + " <fluffiness>2</fluffiness>\n" + + "</document>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + + " <name>mittens</name>\n" + + " <description>it's a cat</description>\n" + + " <fluffiness>8</fluffiness>\n" + + "</document>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentHitWithPopulatedHitFields() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); + doc1.setFieldValue("name", new StringFieldValue("megacat")); + doc1.setFieldValue("description", new StringFieldValue("supercat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true")); + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1234:foo"); + + DocumentHit hit = (DocumentHit)result.hits().get(0); + Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator(); + Set<String> fieldSet = new TreeSet<>(); + while (iter.hasNext()) { + Map.Entry<String, Object> kv = iter.next(); + StringBuilder field = new StringBuilder(); + field.append(kv.getKey()).append(" -> ").append(kv.getValue()); + fieldSet.add(field.toString()); + } + StringBuilder fields = new StringBuilder(); + for (String s : fieldSet) { + fields.append(s).append("\n"); + } + assertEquals( + "description -> supercat\n" + + "documentid -> userdoc:kittens:1234:foo\n" + + "fluffiness -> 10000\n" + + "name -> megacat\n", + fields.toString()); + } + + @Test + public void deserializationExceptionsAreHandledGracefully() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new MockFailingGetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1")); + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + + "Got exception of type java.lang.RuntimeException during document " + + "deserialization: epic dragon attack\"/>\n"+ + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testJsonRendererSetting() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Query query = newQuery("?id=userdoc:kittens:1:2&format=json"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate); + } + + private Query newQuery(String queryString) { + return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + } + + private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + return new Chain<>(searcher); + } + + private static class MockFailingGetDocumentReply extends GetDocumentReply { + private int countdown = 2; + + private MockFailingGetDocumentReply(Document doc) { + super(doc); + } + + @Override + public Document getDocument() { + // Reason for countdown is that the test DocumentSessionFactory calls + // getDocument once internally before code can ever reach handleReply. + if (--countdown == 0) { + throw new RuntimeException("epic dragon attack"); + } + return super.getDocument(); + } + } + + private static class MockBackend extends Searcher { + private Hit hitToReturn; + + public MockBackend(Hit hitToReturn) { + this.hitToReturn = hitToReturn; + } + + public @Override Result search(Query query, Execution execution) { + Result result = new Result(query); + result.hits().add(hitToReturn); + return result; + } + } + + private class MockHttpRequest { + + private final String req; + private byte[] data; + private boolean gzip = false; + + MockHttpRequest(byte[] data, String req) { + this.req = req; + this.data = data; + } + + MockHttpRequest(byte[] data, String req, boolean gzip) { + this.data = data; + this.req = req; + this.gzip = gzip; + } + + public InputStream getData() { + if (gzip) { + try { + ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); + GZIPOutputStream compressed = new GZIPOutputStream(rawOut); + compressed.write(data, 0, data.length); + compressed.finish(); + compressed.flush(); + rawOut.flush(); + return new ByteArrayInputStream(rawOut.toByteArray()); + } catch (Exception e) { + return null; + } + } + return new ByteArrayInputStream(data); + } + + public void addHeaders(HeaderFields headers) { + headers.add("Content-Type", "text/plain;encoding=UTF-8"); + if (gzip) + headers.add("Content-Encoding", "gzip"); + } + + public com.yahoo.container.jdisc.HttpRequest toRequest() { + com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); + addHeaders(request.getJDiscRequest().headers()); + return request; + } + + } + + public static void assertRendered(String expected,Result result) throws Exception { + assertRendered(expected,result,true); + } + + public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception { + if (checkFullEquality) + assertEquals(expected, ResultRenderingUtil.getRendered(result)); + else + assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java new file mode 100644 index 00000000000..21d39089d2a --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.processing.rendering.Renderer; +import com.yahoo.search.Result; +import com.yahoo.search.rendering.DefaultRenderer; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; + +public class ResultRenderingUtil { + + public static String getRendered(Result result) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + Charset cs = Charset.forName("utf-8"); + CharsetDecoder decoder = cs.newDecoder(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java new file mode 100644 index 00000000000..820f7f56e2f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java @@ -0,0 +1,248 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.component.chain.Chain; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.protect.Error; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.prelude.templates.DefaultTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vdslib.VisitorOrdering; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class VisitorSearcherTestCase { + + private DocumentTypeManager docMan = null; + private DocumentType docType; + DocumentSessionFactory factory; + + @org.junit.Before + public void setUp() { + docMan = new DocumentTypeManager(); + docType = new DocumentType("kittens"); + docType.addHeaderField("name", DataType.STRING); + docType.addField("description", DataType.STRING); + docType.addField("image", DataType.RAW); + docType.addField("fluffiness", DataType.INT); + docType.addField("foo", DataType.RAW); + docMan.registerDocumentType(docType); + factory = new DocumentSessionFactory(docType); + } + + public VisitSearcher create() throws Exception { + ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); + ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); + ClusterList clusterList = new ClusterList(); + clusterList.configure(clusterListCfg); + return new VisitSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, docMan, clusterList, new NullFeedMetric())); + } + + @Test + public void testQueryParameters() throws Exception { + VisitSearcher searcher = create(); + VisitorParameters params = searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" + + "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" + + "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" + + "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null); + + assertEquals("id.user=1234", params.getDocumentSelection()); + assertEquals(7, params.getMaxPending()); + assertEquals(2, params.getMaxBucketsPerVisitor()); + assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString()); + assertEquals("othercluster", params.getRemoteDataHandler()); + assertEquals("[header]", params.fieldSet()); + assertEquals(112, params.getFromTimestamp()); + assertEquals(224, params.getToTimestamp()); + assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering()); + assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); + assertEquals(7, params.getTraceLevel()); + assertEquals(true, params.visitInconsistentBuckets()); + assertEquals(true, params.visitRemoves()); + } + + @Test + public void timestampQueryParametersAreParsedAsLongs() throws Exception { + VisitorParameters params = create().getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&" + + "visit.fromTimestamp=1419021596000000&" + + "visit.toTimestamp=1419021597000000"), null); + assertEquals(1419021596000000L, params.getFromTimestamp()); + assertEquals(1419021597000000L, params.getToTimestamp()); + } + + @Test + public void testQueryParametersDefaults() throws Exception { + VisitSearcher searcher = create(); + VisitorParameters params = searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&hits=100"), null); + + assertEquals("id.user=1234", params.getDocumentSelection()); + assertEquals(1, params.getMaxBucketsPerVisitor()); + assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + assertEquals(1, params.getMaxFirstPassHits()); + assertEquals(1, params.getMaxTotalHits()); + assertEquals(32, params.getMaxPending()); + assertEquals(false, params.visitInconsistentBuckets()); + } + + @Test + public void testWrongCluster() throws Exception { + VisitSearcher searcher = create(); + + try { + searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null); + + assertTrue(false); + } catch (Exception e) { + // e.printStackTrace(); + } + } + + + @Test(expected = IllegalArgumentException.class) + public void testNoClusterParamWhenSeveralClusters() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); + ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); + ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); + ClusterList clusterList = new ClusterList(); + clusterList.configure(clusterListCfg); + VisitSearcher searcher = new VisitSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, docMan, clusterList, new NullFeedMetric())); + + searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234"), null); + } + + @Test + public void testSimple() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100")); + assertEquals(1, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + + "</result>\n", result); + } + + private Result invokeVisitRemovesSearchChain() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + return new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true")); + } + + @Test + public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception { + Result result = invokeVisitRemovesSearchChain(); + assertEquals(2, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + + "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" + + "</result>\n", result); + } + + @Test + public void removedDocumentIdsAreXmlEscaped() throws Exception { + factory = mock(DocumentSessionFactory.class); + when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> { + VisitorParameters params = (VisitorParameters)p.getArguments()[0]; + DummyVisitorSession session = new DummyVisitorSession(params, docType); + session.clearAutoReplyMessages(); + session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>"); + return session; + }); + Result result = invokeVisitRemovesSearchChain(); + assertEquals(1, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<remove documentid=\"userdoc:foo:1234:<rem"o"ved&stuff>\"/>\n" + + "</result>\n", result); + } + + private Result invokeSearcherWithUserQuery() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + return new Execution(searchChain, Execution.Context.createContextStub()) + .search(new Query("visit?visit.selection=id.user=1234&hits=100")); + } + + @Test + public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception { + VisitorSession session = mock(VisitorSession.class); + when(session.waitUntilDone(anyLong())).thenReturn(false); + factory = mock(DocumentSessionFactory.class); + when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session); + + Result result = invokeSearcherWithUserQuery(); + assertNotNull(result.hits().getErrorHit()); + assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode()); + } + + @Test + public void testRendererWiring() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); + } + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); + } + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass()); + } + } + + public static void assertRendered(String expected, Result result) throws Exception { + assertEquals(expected, ResultRenderingUtil.getRendered(result)); + } + + private Query newQuery(String queryString) { + return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java new file mode 100644 index 00000000000..d19560ee553 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.jdisc.Metric; + +import java.util.Map; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.20 + */ +class DummyMetric implements Metric { + + @Override + public void set(String key, Number val, Context ctx) { + } + + @Override + public void add(String key, Number val, Context ctx) { + } + + @Override + public Context createContext(Map<String, ?> properties) { + return DummyContext.INSTANCE; + } + + private static class DummyContext implements Context { + private static final DummyContext INSTANCE = new DummyContext(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java new file mode 100644 index 00000000000..2cabed13a7b --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FeedHandlerCompressionTest { + + public static byte[] compress(final String dataToBrCompressed) throws IOException { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(dataToBrCompressed.length()); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream); + gzipOutputStream.write(dataToBrCompressed.getBytes()); + gzipOutputStream.close(); + byte[] compressedBytes = byteArrayOutputStream.toByteArray(); + byteArrayOutputStream.close(); + return compressedBytes; + } + + @Test + public void testUnzipStreamIfNeeded() throws Exception { + final String testData = "foo bar"; + InputStream inputStream = new ByteArrayInputStream(compress(testData)); + HttpRequest httpRequest = mock(HttpRequest.class); + when(httpRequest.getHeader("content-encoding")).thenReturn("gzip"); + InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest); + final StringBuilder processedInput = new StringBuilder(); + while (true) { + int readValue = decompressedStream.read(); + if (readValue < 0) { + break; + } + processedInput.append((char)readValue); + } + assertThat(processedInput.toString(), is(testData)); + } + + /** + * Test by setting encoding, but not compressing data. + * @throws Exception + */ + @Test + public void testUnzipFails() throws Exception { + final String testData = "foo bar"; + InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); + HttpRequest httpRequest = mock(HttpRequest.class); + when(httpRequest.getHeader("Content-Encoding")).thenReturn("gzip"); + InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest); + final StringBuilder processedInput = new StringBuilder(); + while (true) { + int readValue = decompressedStream.read(); + if (readValue < 0) { + break; + } + processedInput.append((char)readValue); + } + assertThat(processedInput.toString(), is(testData)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java new file mode 100644 index 00000000000..ae96a25fad3 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.text.Utf8; + +import java.io.ByteArrayInputStream; + +/** + * Stream with extra data outside the actual input stream. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class MetaStream extends ByteArrayInputStream { + + private byte[] operations; + int i; + + public MetaStream(byte[] buf) { + super(createPayload(buf)); + this.operations = buf; + i = 0; + } + + private static final byte[] createPayload(byte[] buf) { + StringBuilder bu = new StringBuilder(); + for (byte b : buf) { + bu.append("id:banana:banana::doc1 0\n"); + } + return Utf8.toBytes(bu.toString()); + } + + public byte getNextOperation() { + if (i >= operations.length) { + return 0; + } + return operations[i++]; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java new file mode 100644 index 00000000000..1208ce334d9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.List; + +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.network.NetworkOwner; +import com.yahoo.messagebus.routing.RoutingNode; + +/** + * NOP-network. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MockNetwork implements Network { + + @Override + public boolean waitUntilReady(double seconds) { + return true; + } + + @Override + public void attach(NetworkOwner owner) { + } + + @Override + public void registerSession(String session) { + } + + @Override + public void unregisterSession(String session) { + + } + + @Override + public boolean allocServiceAddress(RoutingNode recipient) { + return true; + } + + @Override + public void freeServiceAddress(RoutingNode recipient) { + + } + + @Override + public void send(Message msg, List<RoutingNode> recipients) { + } + + @Override + public void sync() { + } + + @Override + public void shutdown() { + } + + @Override + public String getConnectionSpec() { + return null; + } + + @Override + public IMirror getMirror() { + return null; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java new file mode 100644 index 00000000000..298c925d032 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.messagebus.Reply; +import com.yahoo.text.Utf8String; + +/** + * Minimal reply simulator. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MockReply extends Reply { + + Object context; + + public MockReply(Object context) { + this.context = context; + } + + @Override + public Utf8String getProtocol() { + return null; + } + + @Override + public int getType() { + return 0; + } + + @Override + public Object getContext() { + return context; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java new file mode 100644 index 00000000000..2e88c440b12 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java @@ -0,0 +1,236 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2ErrorsInResultTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + } + + private static class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + + @Override + protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + int count; + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + count = 0; + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + Result r; + ReplyHandler handler = msg.popHandler(); + + switch (count++) { + case 0: + r = new Result(ErrorCode.FATAL_ERROR, + "boom"); + break; + case 1: + r = new Result(ErrorCode.TRANSIENT_ERROR, + "transient boom"); + break; + case 2: + final FailedReply reply = new FailedReply(msg.getContext()); + reply.addError(new Error( + ErrorCode.FATAL_ERROR, + "bad mojo, dude")); + handler.handleReply(reply); + r = Result.ACCEPTED; + break; + default: + handler.handleReply(new MockReply(msg.getContext())); + r = Result.ACCEPTED; + } + return r; + } + + } + + private static class FailedReply extends Reply { + Object context; + + public FailedReply(Object context) { + this.context = context; + } + + @Override + public Utf8String getProtocol() { + return null; + } + + @Override + public int getType() { + return 0; + } + + @Override + public Object getContext() { + return context; + } + } + + private static class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream stream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR boom \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR transient{20}boom \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR bad{20}mojo,{20}dude \n", + Utf8.toString(out.toByteArray())); + } + + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java new file mode 100644 index 00000000000..74ed844d69f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java @@ -0,0 +1,535 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2ExternalFeedTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + Level logLevel; + Logger logger; + boolean initUseParentHandlers; + LogBuffer logChecker; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + logger = Logger.getLogger(Feeder.class.getName()); + logLevel = logger.getLevel(); + logger.setLevel(Level.ALL); + initUseParentHandlers = logger.getUseParentHandlers(); + logChecker = new LogBuffer(); + logger.setUseParentHandlers(false); + logger.addHandler(logChecker); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + logger.setLevel(logLevel); + logger.removeHandler(logChecker); + logger.setUseParentHandlers(initUseParentHandlers); + } + + private static class LogBuffer extends Handler { + public final BlockingQueue<LogRecord> records = new LinkedBlockingQueue<>(); + + @Override + public void publish(LogRecord record) { + try { + records.put(record); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + } + + private static class LessConfiguredHandler extends FeedHandler { + volatile DataFormat lastFormatSeen; + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + @Override + protected Feeder createFeeder(HttpRequest request, + InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + LessConfiguredFeeder f = new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, "ourHostname"); + lastFormatSeen = f.settings.dataFormat; + return f; + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + MockReply mockReply = new MockReply(msg.getContext()); + if (msg instanceof Feeder.FeedErrorMessage) { + mockReply.addError(new com.yahoo.messagebus.Error(123, "Could not feed this")); + } + if (msg instanceof PutDocumentMessage) { + assert(msg.getTrace().getLevel() == 4); + assert(((PutDocumentMessage) msg).getPriority().name().equals("LOWEST")); + } + handler.handleReply(mockReply); + return Result.ACCEPTED; + } + + } + + private static class LessConfiguredFeeder extends Feeder { + public LessConfiguredFeeder(InputStream stream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException, InterruptedException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + assertEquals(7, logChecker.records.size()); + String actualHandshake = logChecker.records.take().getMessage(); + assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); + assertEquals("Successfully deserialized document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertEquals("Sent message successfully, document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + } + + //test session ID without #, i.e. something fishy related to VIPs is going on + sessionId = "something"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "2"); + + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something', but found no session for this client. " + + "Most probably this server is in VIP rotation, and a client session was rotated from one " + + "server to another. This must not happen. Configure VIP with persistence=enabled, or " + + "(preferably) do not use a VIP at all."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # but no hostname + sessionId = "something#"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something#', but found no session for this " + + "client. Possible session timeout due to inactivity, server restart or " + + "reconfig, or bad VIP usage."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # and some unknown hostname at the end + sessionId = "something#thisHostnameDoesNotExistAnywhere"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something#thisHostnameDoesNotExistAnywhere', " + + "but found no session for this client. Session was originally established " + + "towards host thisHostnameDoesNotExistAnywhere, but our hostname is " + + "ourHostname. Most probably this server is in VIP rotation, and a session " + + "was rotated from one server to another. This should not happen. Configure VIP " + + "with persistence=enabled, or (preferably) do not use a VIP at all."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # and some unknown hostname at the end + sessionId = "something#ourHostname"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + Thread.sleep(1000); + } + } + + @Test + public final void testFailedReading() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 4 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testCleaningDoesNotBlowUp() throws IOException { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + handler.forceRunCleanClients(); + } + + @Test + public final void testMockNetworkDoesNotBlowUp() { + Network n = new MockNetwork(); + n.registerSession(null); + n.unregisterSession(null); + assertTrue(n.allocServiceAddress(null)); + n.freeServiceAddress(null); + n.send(null, null); + assertNull(n.getConnectionSpec()); + assertNull(n.getMirror()); + } + + @Test + public final void testMockReplyDoesNotBlowUp() { + MockReply r = new MockReply(null); + assertNull(r.getProtocol()); + assertEquals(0, r.getType()); + assertFalse(r.hasFatalErrors()); + } + + @Test + public final void testFlush() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1, 1, 1, 1, 1, 1, 1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testIllegalVersion() throws IOException { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers() + .add(Headers.VERSION, Integer.toString(Integer.MAX_VALUE)); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals(Headers.HTTP_NOT_ACCEPTABLE, r.getStatus()); + } + + @Test + public final void testSettings() { + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.ROUTE, "bamse brakar"); + nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + FeederSettings settings = new FeederSettings(nalle); + assertEquals(false, settings.drain); + assertEquals(2, settings.route.getNumHops()); + assertEquals(true, settings.denyIfBusy); + } + + @Test + public final void testJsonInputFormat() throws IOException, InterruptedException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DATA_FORMAT, DataFormat.JSON_UTF8.name()); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + assertEquals(7, logChecker.records.size()); + String actualHandshake = logChecker.records.take().getMessage(); + assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); + assertEquals("Successfully deserialized document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertEquals("Sent message successfully, document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertSame(DataFormat.JSON_UTF8, handler.lastFormatSeen); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java new file mode 100644 index 00000000000..4e6ca17abef --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2FailingMessagebusTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + int mbus; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + mbus = 0; + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + mbus = 0; + } + + private class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + @Override + protected Feeder createFeeder(HttpRequest request, + InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + + switch (mbus) { + case 0: + throw new RuntimeException("boom"); + case 1: + Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "tralala"); + mbus = 2; + return r; + case 2: + handler.handleReply(new MockReply(msg.getContext())); + return Result.ACCEPTED; + default: + throw new IllegalStateException("WTF?!"); + } + } + } + + private class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream inputStream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(inputStream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void testFailingMbus() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR boom \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testBusyMbus() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 2; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 1; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers() + .add(Headers.DENY_IF_BUSY, "false"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 1; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR tralala \n", + Utf8.toString(out.toByteArray())); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java new file mode 100644 index 00000000000..1b2d855bdf2 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java @@ -0,0 +1,162 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2NoXmlReaderTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + } + + private static class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + + @Override + protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + MockReply mockReply = new MockReply(msg.getContext()); + if (msg instanceof Feeder.FeedErrorMessage) { + mockReply.addError(new Error(123, "Could not feed this")); + } + handler.handleReply(mockReply); + return Result.ACCEPTED; + } + + } + + private static class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream inputStream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(inputStream, null, null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + //This is different from v1. If we cannot parse XML, we will still get response code 200, but with a sensible + //error message in the response. + assertEquals(200, r.getStatus()); + assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", + Utf8.toString(out.toByteArray())); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java new file mode 100644 index 00000000000..73095a3efe9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; + + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +public class V3CongestionTestCase { + AtomicInteger threadsAvail = new AtomicInteger(10); + AtomicInteger requests = new AtomicInteger(0); + + + static class ClientFeederWithMocks extends ClientFeederV3 { + + private final DocumentOperationMessageV3 docOp; + + ClientFeederWithMocks(ReferencedResource<SharedSourceSession> sourceSession, FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager, String clientId, Metric metric, ReplyHandler feedReplyHandler, AtomicInteger threadsAvailableForFeeding) { + super(sourceSession, feedReaderFactory, docTypeManager, clientId, metric, feedReplyHandler, threadsAvailableForFeeding); + // The operation to return from the client feeder. + VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + docOp = DocumentOperationMessageV3.newRemoveMessage(op, "operation id"); + + } + + @Override + protected DocumentOperationMessageV3 getNextMessage( + String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { + while (true) { + int data = requestInputStream.read(); + if (data == -1 || data == (char)'\n') { + break; + } + } + return docOp; + } + } + + final static int NUMBER_OF_QUEUE_FULL_RESPONSES = 5; + + ClientFeederV3 clientFeederV3; + HttpRequest request; + + @Before + public void setup() { + // Set up a request to be used from the tests. + InputStream in = new MetaStream(new byte[] { 1 }); + request = HttpRequest + .createTestRequest( + "http://foo.bar:19020/reserved-for-internal-use/feedapi", + com.yahoo.jdisc.http.HttpRequest.Method.POST, in); + request.getJDiscRequest().headers().add(Headers.VERSION, "3"); + request.getJDiscRequest().headers().add(Headers.CLIENT_ID, "clientId"); + + + // Create a mock that does not parse the message, only reads the rest of the line. Makes it easier + // to write tests. It uses a mock for message bus. + clientFeederV3 = new ClientFeederWithMocks( + retainMockSession(new SourceSessionParams(), requests), + new MockFeedReaderFactory(), + null /*DocTypeManager*/, + "clientID", + null/*metric*/, + new FeedReplyReader(null/*metric*/), + threadsAvail); + } + + // A mock for message bus that can simulate blocking requests. + private static class MockSharedSession extends SharedSourceSession { + boolean queuFull = true; + AtomicInteger requests; + + public MockSharedSession(SourceSessionParams params, AtomicInteger requests) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + this.requests = requests; + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + if (queuFull) { + requests.incrementAndGet(); + // Disable queue full after some attempts + if (requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES) { + queuFull = false; + } + Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "queue full"); + return r; + } + + handler.handleReply(new MockReply(msg.getContext())); + return Result.ACCEPTED; + } + } + + ReferencedResource<SharedSourceSession> retainMockSession( + SourceSessionParams sessionParams, + AtomicInteger requests) { + final SharedSourceSession session = new MockSharedSession(sessionParams, requests); + return new ReferencedResource<>(session, References.fromResource(session)); + } + + @Test + public void testRetriesWhenThreadsAvailable() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + threadsAvail.set(10); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES); + } + + @Test + public void testNoRetriesWhenNoThreadsAvailable() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + threadsAvail.set(0); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == 1); + } + + @Test + public void testRetriesWhenNoThreadsAvailableButNoDenyIfBusy() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "false"); + threadsAvail.set(0); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES); + } +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java new file mode 100644 index 00000000000..df1de5d75ed --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.HttpResponse; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.7.0 + */ +public class VersionsTestCase { + + private static final List<String> EMPTY = Collections.emptyList(); + private static final List<String> ONE_TWO = Arrays.asList("1", "2"); + private static final List<String> TWO_THREE = Arrays.asList("3", "2"); + private static final List<String> ONE_NULL_TWO = Arrays.asList("1", null, "2"); + private static final List<String> ONE_COMMA_TWO = Collections.singletonList("1, 2"); + private static final List<String> ONE_EMPTY_TWO = Arrays.asList("1", "", "2"); + private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000"); + private static final List<String> TWO_TOO_LARGE_NUMBER = Arrays.asList("2", "1000000000"); + private static final List<String> TWO_COMMA_TOO_LARGE_NUMBER = Arrays.asList("2,1000000000"); + private static final List<String> GARBAGE = Collections.singletonList("garbage"); + + @Test + public void testEmpty() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(EMPTY); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testOneTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTwoThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_THREE); + assertThat(v.first, nullValue()); + assertThat(v.second, is(3)); + } + + @Test + public void testOneNullTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testOneCommaTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testOneEmptyTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TOO_LARGE_NUMBER); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testTwoTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_TOO_LARGE_NUMBER); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTwoCommaTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_COMMA_TOO_LARGE_NUMBER); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testGarbage() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(GARBAGE); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java new file mode 100644 index 00000000000..b134b5c34cd --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server.util; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.23 + */ +public class ByteLimitedInputStreamTestCase { + + private static ByteLimitedInputStream create(byte[] source, int limit) { + if (limit > source.length) { + throw new IllegalArgumentException("Limit is greater than length of source buffer."); + } + InputStream wrappedStream = new ByteArrayInputStream(source); + return new ByteLimitedInputStream(wrappedStream, limit); + } + + @Test + public void requireThatBasicsWork() throws IOException { + ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9); + + assertThat(stream.available(), is(9)); + assertThat(stream.read(), is(97)); + assertThat(stream.available(), is(8)); + assertThat(stream.read(), is(98)); + assertThat(stream.available(), is(7)); + assertThat(stream.read(), is(99)); + assertThat(stream.available(), is(6)); + assertThat(stream.read(), is(100)); + assertThat(stream.available(), is(5)); + assertThat(stream.read(), is(101)); + assertThat(stream.available(), is(4)); + assertThat(stream.read(), is(102)); + assertThat(stream.available(), is(3)); + assertThat(stream.read(), is(103)); + assertThat(stream.available(), is(2)); + assertThat(stream.read(), is(104)); + assertThat(stream.available(), is(1)); + assertThat(stream.read(), is(105)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + } + + @Test + public void requireThatChunkedReadWorks() throws IOException { + ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9); + + assertThat(stream.available(), is(9)); + byte[] toBuf = new byte[4]; + assertThat(stream.read(toBuf), is(4)); + assertThat(toBuf[0], is((byte) 97)); + assertThat(toBuf[1], is((byte) 98)); + assertThat(toBuf[2], is((byte) 99)); + assertThat(toBuf[3], is((byte) 100)); + assertThat(stream.available(), is(5)); + + assertThat(stream.read(toBuf), is(4)); + assertThat(toBuf[0], is((byte) 101)); + assertThat(toBuf[1], is((byte) 102)); + assertThat(toBuf[2], is((byte) 103)); + assertThat(toBuf[3], is((byte) 104)); + assertThat(stream.available(), is(1)); + + assertThat(stream.read(toBuf), is(1)); + assertThat(toBuf[0], is((byte) 105)); + assertThat(stream.available(), is(0)); + + assertThat(stream.read(toBuf), is(-1)); + assertThat(stream.available(), is(0)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java new file mode 100644 index 00000000000..c245490b1d7 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.server.FeedReaderFactory; + +import java.io.InputStream; + +/** + * For creating MockReader of innput stream. + * @author dybdahl + */ +public class MockFeedReaderFactory extends FeedReaderFactory { + + @Override + public FeedReader createReader( + InputStream inputStream, + DocumentTypeManager docTypeManager, + FeedParams.DataFormat dataFormat) { + try { + return new MockReader(inputStream); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java new file mode 100644 index 00000000000..a4ac0f4fdaf --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespa.http.server.MetaStream; +import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; + +import java.io.InputStream; +import java.lang.reflect.Field; + +/** + * Mock for ExternalFeedTestCase which had to override package private methods. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class MockReader implements FeedReader { + + MetaStream stream; + boolean finished = false; + + public MockReader(InputStream stream) throws Exception { + this.stream = getMetaStream(stream); + } + + private static MetaStream getMetaStream(InputStream stream) { + if (stream instanceof MetaStream) { + return (MetaStream) stream; + } + if (!(stream instanceof ByteLimitedInputStream)) { + throw new IllegalStateException("Given unknown stream type."); + } + //Ooooooo this is so ugly + try { + ByteLimitedInputStream byteLimitedInputStream = (ByteLimitedInputStream) stream; + Field f = byteLimitedInputStream.getClass().getDeclaredField("wrappedStream"); //NoSuchFieldException + f.setAccessible(true); + return (MetaStream) f.get(byteLimitedInputStream); + } catch (Exception e) { + throw new IllegalStateException("Implementation of ByteLimitedInputStream has changed.", e); + } + } + + @Override + public void read(Operation operation) throws Exception { + if (finished) { + return; + } + + byte whatToDo = stream.getNextOperation(); + DocumentId id = new DocumentId("id:banana:banana::doc1"); + DocumentType docType = new DocumentType("banana"); + switch (whatToDo) { + case 0: + finished = true; + break; + case 1: + Document doc = new Document(docType, id); + operation.setDocument(doc); + break; + case 2: + operation.setRemove(id); + break; + case 3: + operation.setDocumentUpdate(new DocumentUpdate(docType, id)); + break; + case 4: + throw new RuntimeException("boom"); + } + } + +}
\ No newline at end of file |